Search

# --------------------------------------------------------
# BLOCK AI TRAINING BOTS (Prevents scraping for model training)
# --------------------------------------------------------

# Block Google from using content to train Gemini/Vertex AI
User-agent: Google-Extended
Disallow: /

# Block OpenAI from using content to train GPT models
User-agent: GPTBot
Disallow: /

# Block Anthropic from using content to train Claude
User-agent: ClaudeBot
Disallow: /

# Block Apple from using content to train Apple Intelligence
User-agent: Applebot-Extended
Disallow: /

# Block Common Crawl (The massive dataset many AIs use for training)
User-agent: CCBot
Disallow: /

# --------------------------------------------------------
# ALLOW REAL-TIME AI SEARCH BOTS (For visibility & citations)
# --------------------------------------------------------

# OpenAI's bot that fetches web pages when a user asks a real-time question
User-agent: ChatGPT-User
Allow: /

# OpenAI's SearchGPT crawler
User-agent: OAI-SearchBot
Allow: /

# Perplexity's real-time search bot
User-agent: PerplexityBot
Allow: /

# Anthropic's real-time search bot (Unverified but safe to include)
User-agent: Claude-Web
Allow: /

# You.com AI Search Engine
User-agent: YouBot
Allow: /

# Brave Search AI (Brave Leo)
User-agent: BraveBot
Allow: /

# --------------------------------------------------------
# STANDARD CRAWLERS & SITEMAP
# --------------------------------------------------------

# (Standard Googlebot and Bingbot are allowed by default here)

User-agent: *
Allow: /

Host: www.hacktiv8.com

Sitemap: https://www.hacktiv8.com/sitemap.xml