Search

User-agent: PetalBot
Disallow: /
User-agent: MJ12bot
Disallow: /
#User-agent: Blekkobot
#Disallow: /
User-agent: dotbot
Disallow: /
#User-agent: Exabot
#Disallow: /
#User-agent: YodaoBot
#Disallow: /
User-agent: AhrefsBot
Disallow: /
#User-agent: Yandex 
#Disallow: /
User-agent: SemrushBot
Disallow: /
#User-agent: Baiduspider
#Disallow: /
User-agent: BLEXBot
Disallow: /
#User-agent: SEOkicks-Robot
#Disallow: /
#User-agent: Twitterbot 
#Disallow: /
#User-agent: Riddler
#Disallow: /
User-agent: Sogou web spider
Disallow: /
User-agent: XoviBot
Disallow: /
User-agent: *
Crawl-Delay: 240
Disallow: /cgi-bin/itkcvsweb.cgi/
Disallow: /gitweb
Disallow: /cgi-bin/
Disallow: /cgi-bin/cvsweb.cgi/
Disallow: /cgi-bin/cvsweb.cgi/
Disallow: /cgi-bin/viewcvs.cgi/
Disallow: /cgi-bin/viewcvs2.cgi/
Disallow: /CMake/Testing/
Disallow: /CMake/Testing/HTML/TestingResults/
Disallow: /cmake/help/git-next
Disallow: /cmake/help/git-master
Disallow: /Insight/Testing/
Disallow: /Insight/Testing/HTML/TestingResults/
Disallow: /Paraview/Testing/
Disallow: /Paraview/Testing/HTML/TestingResults/
Disallow: /VTK/doc/release/3/
Disallow: /vtk/quality/
Disallow: /VTK/Testing/
Disallow: /HTML/Download.htm
Disallow: /doc/release/5.2/vtkDocHtml-5.2.0.chm
Disallow: /doc/release/5.2/vtkDocHtml-5.2.0.tar.gz
Disallow: /files/
Disallow: /doc/nightly/vtkNightlyDocHtml.tar.gz
Disallow: /IGSTKWIKI/images/
Disallow: /Training/
#Disallow: /download/
Disallow: /archive/
Disallow: /PublicBeta
Disallow: /svn/
Disallow: /Bug/api/soap/
Disallow: /mailman/options/
Disallow: www.itk.org/HTML/Download.htm
Disallow: *.pdf
Disallow: *.gz
Disallow: *.zip