Search

# DevOps Shield robots.txt
# Posture (Phase 6 — Answer-Engine Distribution):
#   * Default: allow all general crawlers to index the public site.
#   * Search + answer-engine crawlers (build a live index / answer with citation):
#     explicitly allowed.
#   * Training-only crawlers (scrape corpora for model training, no live index
#     or citation back): disallowed.
#
# AI-crawler ingestion is MEASURED from server/access logs by matching these
# user-agents; review periodically to gauge answer-engine visibility:
#   - GPTBot           (OpenAI training crawler)      -> disallowed below
#   - OAI-SearchBot    (OpenAI ChatGPT search)        -> allowed below
#   - PerplexityBot    (Perplexity search)            -> allowed below
#   - ClaudeBot        (Anthropic training crawler)   -> disallowed below
#   - Google-Extended  (Google Gemini training)       -> disallowed below
#
# Machine-readable docs index: https://www.devopsshield.com/llms.txt

# --- General crawlers: allow full indexing ---
User-agent: *
Allow: /

# --- Search / answer engines: explicitly allowed (live index + citation) ---
User-agent: Googlebot
Allow: /

User-agent: bingbot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

# --- Training-only crawlers: disallowed (no live index / no citation) ---
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

Sitemap: https://www.devopsshield.com/sitemap.xml