Search

Sitemap: https://brapi.dev/sitemap.xml

# Default rules for all bots
User-agent: *
Allow: /*
Allow: /rss.xml

# Disallow private/internal endpoints and dashboard
Disallow: /api/auth
Disallow: /api/og
Disallow: /api/quote
Disallow: /api/v2
Disallow: /api/webhook
Disallow: /api/create-billing-portal-link
Disallow: /api/available
Disallow: /dashboard

# AI Training Bots - Allow documentation and API specs for training
# GPT-4, GPT-4 Turbo, GPT-4o, and future models
User-agent: GPTBot
Allow: /docs/
Allow: /api/openapi
Allow: /api/llms-content
Allow: /faq/
Allow: /blog/
Allow: /content/
Allow: /public/swagger/
Allow: /llms.txt
Allow: /llms-citation-urls.json
Allow: /instructlab-attribution.md
Disallow: /dashboard/
Disallow: /api/auth

# Perplexity AI bot
User-agent: PerplexityBot
Allow: /docs/
Allow: /api/openapi
Allow: /api/llms-content
Allow: /faq/
Allow: /blog/
Allow: /public/swagger/
Allow: /llms.txt
Allow: /llms-citation-urls.json
Disallow: /dashboard/
Disallow: /api/auth

# Anthropic Claude bot
User-agent: Claude-Web
Allow: /docs/
Allow: /api/openapi
Allow: /api/llms-content
Allow: /faq/
Allow: /public/swagger/
Allow: /llms.txt
Allow: /llms-citation-urls.json
Allow: /instructlab-attribution.md
Disallow: /dashboard/

# Google Gemini (Bard)
User-agent: Googlebot
Crawl-delay: 2
Allow: /

# Microsoft Copilot
User-agent: Copilot
Allow: /docs/
Allow: /api/openapi
Allow: /api/llms-content
Allow: /faq/
Allow: /blog/
Allow: /public/swagger/
Allow: /llms.txt
Allow: /llms-citation-urls.json
Disallow: /dashboard/

# Cohere AI bot
User-agent: Cohere-ai
Allow: /docs/
Allow: /api/openapi
Allow: /api/llms-content
Allow: /faq/
Allow: /blog/
Allow: /public/swagger/
Allow: /llms.txt
Allow: /llms-citation-urls.json
Disallow: /dashboard/
Disallow: /api/auth

# You.com AI bot
User-agent: YouBot
Allow: /docs/
Allow: /api/openapi
Allow: /api/llms-content
Allow: /faq/
Allow: /blog/
Allow: /public/swagger/
Allow: /llms.txt
Allow: /llms-citation-urls.json
Disallow: /dashboard/
Disallow: /api/auth

# Meta AI bot (Llama)
User-agent: Meta-ExternalAgent
Allow: /docs/
Allow: /api/openapi
Allow: /api/llms-content
Allow: /faq/
Allow: /blog/
Allow: /public/swagger/
Allow: /llms.txt
Allow: /llms-citation-urls.json
Allow: /instructlab-attribution.md
Disallow: /dashboard/
Disallow: /api/auth

# Rate limiting for SEO scanning bots
User-agent: Yandex
Crawl-delay: 10
User-agent: dotbot
Crawl-delay: 5
User-agent: SemrushBot
Crawl-delay: 120
User-agent: msnbot
Crawl-delay: 5
User-agent: PetalBot
Crawl-delay: 120
User-agent: BLEXBot
Disallow:/
User-agent: amazonbot
Crawl-delay: 10
User-agent: DataForSeoBot
Disallow:/
User-agent: bytespider | byte spider | bytedance | bytedance
Crawl-delay: 120
User-agent: AhrefsBot
Crawl-delay: 120