Search

User-agent: *
Disallow: /cgi-bin
Disallow: /syadm
Disallow: /sysadmfiles
Disallow: /pascalfsnotes
Disallow: /c2009notes
Disallow: /files
Disallow: /altmalta/dokuwiki
Disallow: /c2011notesdis
Disallow: /test

# htdig is a bad bot
User-agent: htdig
Disallow: /

User-agent: NPBot
Disallow: /

User-agent: EmailSiphon
Disallow: /

# Rover is a bad dog 
User-agent: Roverbot
Disallow: /

User-agent: NetMechanic
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: grub
Disallow: /

User-agent: WebZip
User-agent: larbin
User-agent: b2w/0.1
User-agent: Copernic
User-agent: TurnitinBot
Disallow: /

User-agent: Zao
Disallow: /

User-agent: NutchOrg
Disallow:  /

User-agent: Nutch
Disallow: /

User-agent: Baidu
Disallow: /

User-agent: Jetbot/1.0
Disallow: /

User-agent: Jetbot
Disallow: /

User-agent: LinkextractorPro
Disallow: /

User-agent: NationalDirectory
User-agent: LNSpiderguy
User-agent: Teleport
User-agent: MIIxpc
User-agent: asterias
User-agent: lwp-trivial
User-agent: LinkWalker
User-agent: cosmos
User-agent: MSIECrawler
User-agent: sitecheck.internetseer.com
User-agent: pompos
User-agent: Generic
User-agent: WebSearchBench
User-agent: almaden
User-agent: k2spider
User-agent: curl
User-agent: Wget
User-agent: UbiCrawler
Disallow: /

User-agent: Wget/1.6
Disallow: /

User-agent: Wget/1.5.3
Disallow: /

User-agent: Wget
Disallow: /

User-agent: Microsoft URL Control - 6.00.8169
Disallow: /

User-agent: Microsoft URL Control
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Dynamic Submission v7.0
Disallow: /

User-agent: Dart Communications PowerTCP
Disallow: /

User-agent: surveybot
Disallow: /

User-agent: LinkLint-checkonly
Disallow: /