Search

# OneHack.st — MAXIMUM AI DISCOVERY robots.txt
# Updated: 2026-01-17
# Philosophy: In the AI era, let ALL bots discover ALL content
# Only block security-sensitive paths, open EVERYTHING else

# Sitemaps (AI bots, read these!)
Sitemap: https://onehack.st/sitemap.xml
Sitemap: https://onehack.st/sitemap_recent.xml
Sitemap: https://onehack.st/sitemap_1.xml
Sitemap: https://onehack.st/sitemap_2.xml
Sitemap: https://onehack.st/sitemap_3.xml
Sitemap: https://onehack.st/sitemap_4.xml

# =============================================================================
# DEFAULT: ALLOW EVERYTHING, block only security paths
# =============================================================================
User-agent: *
# Security-only blocks (MUST keep these for safety)
Disallow: /admin/
Disallow: /auth/
Disallow: /email/
Disallow: /user-api-key
Disallow: /session
Disallow: /login-preferences
Disallow: /my/
Disallow: /u/*/preferences
Disallow: /u/*/private-messages
Disallow: /u/*/notifications
Disallow: /u/*/invited
Disallow: /draft/
Disallow: /*?api_key*
Disallow: /*.json$
# ALLOW EVERYTHING ELSE - including user profiles, search, tags, groups!
Allow: /u/
Allow: /search
Allow: /tag/
Allow: /tags/
Allow: /g/
Allow: /groups/
Allow: /badges/
Allow: /categories
Allow: /latest
Allow: /top
Allow: /new
Allow: /unread
Allow: /c/
Allow: /t/
Allow: /

# =============================================================================
# AI CHATBOTS - WELCOME! Discover everything!
# =============================================================================

# OpenAI (ChatGPT, GPT-4, SearchGPT)
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# Anthropic (Claude)
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: anthropic-ai
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# Google (Search + Gemini AI)
User-agent: Googlebot
User-agent: Google-Extended
User-agent: Googlebot-Image
User-agent: Storebot-Google
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# Microsoft (Bing + Copilot)
User-agent: bingbot
User-agent: msnbot
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# Apple (Siri, Spotlight)
User-agent: Applebot
User-agent: Applebot-Extended
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# Meta AI (Llama, Meta AI assistant)
User-agent: meta-externalagent
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
User-agent: facebookexternalhit
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# Amazon (Alexa, Q)
User-agent: Amazonbot
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# Common Crawl (trains many AI models!)
User-agent: CCBot
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# You.com AI
User-agent: YouBot
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# Cohere AI
User-agent: cohere-ai
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# AI21 Labs
User-agent: AI2Bot
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# Mistral AI
User-agent: MistralBot
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# xAI (Grok)
User-agent: Grokbot
User-agent: xAI-bot
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# Brave Search AI
User-agent: BraveBot
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# Mojeek
User-agent: MojeekBot
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# =============================================================================
# INTERNATIONAL SEARCH & AI
# =============================================================================

# Yandex (Russia)
User-agent: YandexBot
User-agent: YandexImages
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# Baidu (China)
User-agent: Baiduspider
User-agent: Baiduspider-image
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# Sogou (China)
User-agent: Sogou web spider
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# 360 Search (China)
User-agent: 360Spider
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# ByteDance (TikTok, Douyin)
User-agent: Bytespider
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# Naver (Korea)
User-agent: Yeti
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# Seznam (Czech)
User-agent: SeznamBot
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# Qwant (Europe)
User-agent: Qwantify
User-agent: Qwantbot
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# Coc Coc (Vietnam)
User-agent: coccocbot
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# =============================================================================
# SOCIAL & PREVIEW BOTS
# =============================================================================

User-agent: LinkedInBot
User-agent: Twitterbot
User-agent: Slackbot
User-agent: Discordbot
User-agent: TelegramBot
User-agent: WhatsApp
User-agent: Pinterest
User-agent: Slurp
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# =============================================================================
# ARCHIVE BOTS (preserve our content!)
# =============================================================================

User-agent: ia_archiver
User-agent: archive.org_bot
Disallow: /admin/
Disallow: /auth/
Disallow: /session
Disallow: /*?api_key*
Allow: /

# =============================================================================
# End of robots.txt - OneHack.st welcomes ALL AI bots!
# Questions? Contact @CyanoMeme on X/Twitter
# =============================================================================