Search

# robots.txt for https://ftp.gnu.org/

User-agent: *
Crawl-delay: 60
Disallow: /norobotsnorhumansshouldevervisithispage/
Allow: /

# Majestic - SEO
User-agent: MJ12bot
Disallow: /

# DataForSeo - SEO
User-agent: DataForSeoBot
Disallow: /

# webmeup - SEO
User-agent: BLEXBot
Disallow: /

# Ahrefs - SEO
User-agent: AhrefsBot
Disallow: /

# babbar - SEO
User-agent: barkrowler
Disallow: /

# Screamingfrog - SEO
User-agent: Screaming Frog SEO Spider
Disallow: /

# Seozoom - SEO
User-Agent: ZoomBot
Disallow: /

# Brandwatch - SEO
User-agent: magpie-crawler
Disallow: /

# Begin Moz - SEO
# Not to be confused with Mozilla.
User-agent: DotBot
Disallow: /

User-agent: rogerbot
Disallow: /
# End Moz - SEO

# Begin Semrush - SEO
User-agent: SemrushBot
Disallow: /

User-agent: SiteAuditBot
Disallow: /

User-agent: SemrushBot-BA
Disallow: /

User-agent: SemrushBot-SI
Disallow: /

User-agent: SemrushBot-SWA
Disallow: /

User-agent: SplitSignalBot
Disallow: /

User-agent: SemrushBot-OCOB
Disallow: /
# End Semrush - SEO

# cognitiveSEO - SEO
User-agent: JamesBOT
Disallow: /

# oncrawl - SEO
User-agent: Oncrawl
Disallow: /

# BEGIN Awario - Marketing
User-agent: AwarioRssBot
Disallow: /

User-agent: AwarioSmartBot
Disallow: /

User-agent: AwarioBot
Disallow: /
# END Awario - Marketing

# SERPSTAT - SEO
User-agent: serpstatbot
Disallow: /

# website-datenbank.de - Search engine?
User-agent: netEstate NE Crawler
Disallow: /

# Ignores crawl-delay and does not help us.
User-Agent: panscient.com
Disallow: /

# Aggressive Latvian Academic Integrity bot that does not help us.
User-agent: AcademicBotRTU          
Disallow: /

# Timpi NFT
User-Agent: Timpibot
Disallow: /