Search

# Robots.txt for Insight.com
# Use specialized blocks only if rules differ from the global policy.

User-agent: *
# Allow specific parameters first
Allow: /*?qtype=
Allow: /*?pq=
Allow: /*?identifier=shopping
Allow: /*?partnermessage
Allow: /insightweb/*.css$
Allow: /*.html
Allow: /*/shop/product/
Allow: /*%23*

# Block all other parameters and system folders
Disallow: /*?*
Disallow: /*/search*.html
Disallow: /insightweb/
Disallow: /flytrap/
Disallow: /content/dam/insight-web/*/solutions/service-provider/microsite/assets/
Disallow: /content/dam/insight-web/*/pdfs/
Disallow: /content/dam/insight/
Disallow: /content/dam/global/*/pdfs/
Disallow: /content/insight-web/*/help/*
Disallow: /content/insight-web/*/client/*
Disallow: /content/insight-web/*/Sandbox/*
Disallow: /content/insight-web/*/sandbox/*

############################
# BLOCKED CRAWLERS
############################
User-agent: CCBot
User-agent: FacebookBot
User-agent: NeevaAI
User-agent: Bytespider
User-agent: Firecrawl
User-agent: Kadoa
User-agent: ImagesiftBot
Disallow: /


Sitemap: https://www.insight.com/sitemap.xml