Search

# CarAPI.dev API Documentation Robots.txt
# Updated to allow AI crawlers for improved visibility.

# ============================================
# AI Crawlers - Allowed with full access
# ============================================

# OpenAI GPTBot
User-agent: GPTBot
Allow: /
Allow: /endpoints/
Allow: /llms.txt

# OpenAI ChatGPT-User (browsing mode)
User-agent: ChatGPT-User
Allow: /
Allow: /endpoints/
Allow: /llms.txt

# Google AI (Gemini)
User-agent: Google-Extended
Allow: /
Allow: /endpoints/
Allow: /llms.txt

# Anthropic Claude
User-agent: ClaudeBot
Allow: /
Allow: /endpoints/
Allow: /llms.txt

User-agent: Claude-Web
Allow: /
Allow: /endpoints/
Allow: /llms.txt

User-agent: anthropic-ai
Allow: /
Allow: /endpoints/
Allow: /llms.txt

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Allow: /endpoints/
Allow: /llms.txt

# Cohere AI
User-agent: cohere-ai
Allow: /
Allow: /endpoints/
Allow: /llms.txt

# Meta AI
User-agent: FacebookBot
Allow: /
Allow: /endpoints/
Allow: /llms.txt

User-agent: Meta-ExternalAgent
Allow: /
Allow: /endpoints/
Allow: /llms.txt

# Microsoft/Bing AI (Copilot)
User-agent: Bingbot
Allow: /

# You.com AI
User-agent: YouBot
Allow: /
Allow: /endpoints/
Allow: /llms.txt

# OpenAI SearchBot (search feature)
User-agent: OAI-SearchBot
Allow: /
Allow: /endpoints/
Allow: /llms.txt

# Amazon Alexa/AWS AI
User-agent: Amazonbot
Allow: /
Allow: /endpoints/
Allow: /llms.txt

# ByteDance/TikTok AI
User-agent: Bytespider
Allow: /
Allow: /endpoints/
Allow: /llms.txt

# Diffbot AI data extraction
User-agent: Diffbot
Allow: /
Allow: /endpoints/
Allow: /llms.txt

# ImagesiftBot
User-agent: ImagesiftBot
Allow: /
Allow: /endpoints/
Allow: /llms.txt

# ============================================
# Traditional Search Engines - Allowed
# ============================================

User-agent: Googlebot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Applebot
Allow: /

# ============================================
# Social Media Bots - Allowed for previews
# ============================================

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# ============================================
# SEO/Marketing Bots - Blocked
# ============================================

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SiteAuditBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: VoilaBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: CrawlBot
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: spbot
Disallow: /

User-agent: SentiBot
Disallow: /

User-agent: Cliqzbot
Disallow: /

User-agent: MojeekBot
Disallow: /

User-agent: SafeSearch
Disallow: /

# ============================================
# Default rule - Allow all
# ============================================

User-agent: *
Allow: /

# ============================================
# Sitemap & AI Resources
# ============================================

Sitemap: https://docs.carapi.dev/sitemap.xml

# AI-specific resources (direct links for AI crawlers)
LLMs-Txt: https://docs.carapi.dev/llms.txt
OpenAPI: https://api.carapi.dev/openapi.json

# Crawl delay for all bots
Crawl-delay: 1