Search

# 這是web3.bitget.com的robots.txt

# -----------------------------------------------------
# Allow specific popular AI/LLM crawlers (Overseas)
# 这些爬虫的数据可能用于训练AI模型或支持AI相关功能

User-agent: Anthropic-AI
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: Claude-Web
Allow: /
User-agent: cohere-ai
Allow: /
User-agent: GPTBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: DeepseekBot
Allow: /
User-agent: Google-Extended
Allow: /
User-agent: Googlebot
Allow: /
User-agent: FacebookBot
Allow: /
User-agent: meta-externalagent
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: YouBot
Allow: /
User-agent: Applebot
Allow: /
User-agent: Applebot-Extended
Allow: /
User-agent: LinkedInBot
Allow: /

# -----------------------------------------------------
# China
User-agent: Baiduspider
Allow: /
User-agent: Bytespider
Allow: /
User-agent: Kimi-Bot
Allow: /
User-agent: Sogou web spider
Allow: /
User-agent: Yisouspider
Allow: /
User-agent: UCBrowser
Allow: /

# -----------------------------------------------------
# 搜索引擎通用规则
User-Agent: *
Disallow:/*/dapp/browsinghistory
Disallow:/*/dapp/collect
Disallow:/*/dapp/searchresults

# -----------------------------------------------------
# Sitemaps
Sitemap: https://web3.bitget.com/com-all-sitemap.xml