#go away
User-agent: Directcrawler
Disallow: /
User-agent: VoilaBot
Disallow: /
User-agent: Java/1.5.0_11
Disallow: /
User-agent: Java/1.4.1_04
Disallow: /
User-agent: gsa-crawler
Disallow: /
#User-agent: ia_archiver-web.archive.org
#Disallow: /
User-agent: ShopWiki
Disallow: /
User-agent: *
Disallow: /702show*
Disallow: /accounts*
Disallow: /accounts/login*
Disallow: /blogs/robin-leachs-las-vegas-celebrity-watch*
Disallow: /comments*
Disallow: /dossier*
Disallow: /mailfriend*
Disallow: /*rawhtml*
Disallow: /sunbin*
Disallow: /sun/dossier*
Disallow: *reminder/
Disallow: /ufc-sn/
Disallow: /ufc-video-sn/
Disallow: /mma-sn/
Disallow: /wec-sn/
Disallow: *ufcsn
#to stop scraping of old events
Disallow: /events/search/?category=*
#User-agent: msnbot
Crawl-delay: 120