Search

# ==============================================================================
# Google for Education - robots.txt Configuration
# Goal: Balance AI search visibility (referral traffic) vs. LLM training (scraping)
# ==============================================================================

# ------------------------------------------------------------------------------
# 1. ALLOWED CRAWLERS (Traditional Search & real-time generative answers)
# ------------------------------------------------------------------------------

# OpenAI ChatGPT Search Crawler
User-agent: OAI-SearchBot
Allow: /

# Perplexity Search Crawler
User-agent: PerplexityBot
Allow: /

# Apple Spotlight/Spotlight/Siri Crawler
User-agent: Applebot
Allow: /

# Google Standard Search Crawler (Powers standard results & AI Overviews)
User-agent: Googlebot
Allow: /

# Bing Standard Search Crawler (Powers standard results & Bing Copilot)
User-agent: Bingbot
Allow: /

# ------------------------------------------------------------------------------
# 2. BLOCKED BOTS (Bulk LLM Training & Non-Attributed Scraping)
# ------------------------------------------------------------------------------

# Google Gemini Training Engine
User-agent: Google-Extended
Disallow: /

# OpenAI GPT Training Engine
User-agent: GPTBot
Disallow: /

# Anthropic Claude Training Engines
User-agent: ClaudeBot
Disallow: /

User-agent: Anthropic-AI
Disallow: /

# Apple Intelligence Training Engine
User-agent: Applebot-Extended
Disallow: /

# Cohere AI Training Engine
User-agent: Cohere-ai
Disallow: /

# Meta AI Training Engines
User-agent: FacebookBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

# Third-Party LLM Scraping Platforms
User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

# ------------------------------------------------------------------------------
# 3. DEFAULT GLOBAL & EXPERIMENT CRAWL POLICIES
# ------------------------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /exp/ # Block experiment/staging environments

# Sitemap Link
Sitemap: https://edu.google.com/sitemap.xml