Search

# robots.txt for https://rye.dev
# Last updated: May 2026

# Allow all crawlers full access to the site
User-agent: *
# Content Signals — declare AI usage preferences (contentsignals.org,
# draft-romm-aipref-contentsignals). ai-train=no: don't train on this
# content. search=yes: index for search. ai-input=yes: allow agent
# retrieval at query time (RAG, summarisation, MCP).
Content-Signal: ai-train=no, search=yes, ai-input=yes
Allow: /

# Sitemap location
Sitemap: https://rye.dev/sitemap-index.xml

# LLM-friendly documentation (llms.txt specification)
# See: https://llmstxt.org/
# llms.txt: Concise overview with links to key resources
# llms-full.txt: Complete content inlined for large context windows

# Disallow crawling of build artifacts and internal paths
Disallow: /_astro/
Disallow: /pagefind/
Disallow: /api/
Disallow: /admin/
Disallow: /admin
Disallow: /login

# Public agent-discovery + tool endpoints — explicitly re-allowed inside
# the /api/ disallow so RFC 9727 catalog entries remain reachable.
Allow: /api/health
Allow: /api/contact
Allow: /api/newsletter
Allow: /api/reactions
Allow: /api/mentions
Allow: /api/pulse
Allow: /api/agent-observatory
Allow: /api/webmention
Allow: /mcp
Allow: /openapi.json

# Allow search engine image crawlers
User-agent: Googlebot-Image
Allow: /images/
Allow: /og/
Allow: /screenshots/

# Allow social media crawlers for rich previews
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

# Crawl-delay for less aggressive bots (optional, not all bots respect this)
# User-agent: *
# Crawl-delay: 1

# Security Policy for rye.dev
# Following RFC 9116 (https://www.rfc-editor.org/rfc/rfc9116)

# Contact for security vulnerability reports
# Please use the about page for responsible disclosure
Contact: https://rye.dev/about/

# Canonical location of this security.txt file
Canonical: https://rye.dev/.well-known/security.txt

# Preferred languages for security communications
Preferred-Languages: en

# Expiration date for this security.txt (1 year from creation)
Expires: 2026-12-01T00:00:00.000Z

# Policy URL (links to about page which contains contact info)
Policy: https://rye.dev/about/

# Acknowledgments page (optional - can be added later if needed)
# Acknowledgments: https://rye.dev/security/acknowledgments/