Search

# =========================================================
# robots.txt — angeo.dev
# Optimized for Google Search + AI/AEO visibility (2026)
# =========================================================

# =========================
# GLOBAL RULES
# =========================
User-agent: *
Allow: /

# =========================
# WORDPRESS SYSTEM
# (genuine no-value / admin paths only)
# =========================
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /license.txt

# =========================
# DUPLICATE / INTERNAL URLS
# NOTE: tracking params (utm, fbclid, etc.) are intentionally
# NOT blocked here — let rel=canonical dedupe them so the
# underlying pages stay crawlable and indexable.
# =========================
Disallow: /*?replytocom=
Disallow: /*?attachment_id=
Disallow: /*?s=
Disallow: /search/

# =========================
# FEEDS / COMMENTS / TRACKBACKS
# =========================
Disallow: */feed/
Disallow: */comments/
Disallow: */trackback/
Disallow: /wp-comments-post.php

# =========================
# THEME & PLUGIN INTERNALS
# NOTE: /wp-content/plugins/ and /wp-content/themes/ are
# intentionally NOT blocked. Blocking them prevents Googlebot
# from fetching your CSS/JS, breaks rendering-based indexing,
# and is a documented cause of "not mobile-friendly" flags in
# Search Console. Cache stays blocked since it carries no
# unique renderable assets.
# =========================
Disallow: /wp-content/cache/
Allow: /wp-content/uploads/

# =========================
# REST API
# NOTE: /wp-json/ is intentionally left open at the root.
# oEmbed discovery (/wp-json/oembed/) is used by other sites
# and services (Slack, Twitter, other WP installs) to generate
# rich embeds of your pages — blocking the whole namespace
# breaks that. Some Gutenberg dynamic blocks (Interactivity
# API, live Query Loop filters, etc.) also fetch from
# /wp-json/wp/v2/... client-side, and Googlebot executes JS,
# so a blanket block can leave those blocks empty in what
# Google actually renders. Only the contact-form endpoint,
# which is POST-only and has no crawl value, is blocked below.
# =========================
Disallow: /wp-json/hostinger-reach/
Allow: /wp-json/oembed/

# NOTE: /tag/ is intentionally allowed (removed the old Disallow).
# Your Yoast sitemap submits post_tag-sitemap.xml, so blocking /tag/
# created a "Blocked by robots.txt" contradiction in Search Console.
# Tag archives also build topical clusters that help SEO + AEO.

# =========================================================
# AI / LLM CRAWLERS — ALLOW (maximize AEO / AI citations)
# =========================================================

# --- OpenAI ---
User-agent: GPTBot
Allow: /
User-agent: OAI-SearchBot
Allow: /
User-agent: ChatGPT-User
Allow: /

# --- Anthropic (Claude) ---
# NOTE: only the three currently documented Anthropic bots are
# listed. anthropic-ai and Claude-Web are deprecated legacy
# identifiers per Anthropic's official crawler documentation
# (updated Feb 2026) and have been removed.
User-agent: ClaudeBot
Allow: /
User-agent: Claude-User
Allow: /
User-agent: Claude-SearchBot
Allow: /

# --- Perplexity ---
User-agent: PerplexityBot
Allow: /
User-agent: Perplexity-User
Allow: /

# --- Google AI (Gemini / AI products) ---
User-agent: Google-Extended
Allow: /
User-agent: GoogleOther
Allow: /

# --- Microsoft (Bing search index; also feeds Copilot grounding) ---
User-agent: Bingbot
Allow: /

# --- Apple ---
User-agent: Applebot
Allow: /
User-agent: Applebot-Extended
Allow: /

# --- Amazon ---
User-agent: Amazonbot
Allow: /

# --- Meta AI ---
User-agent: FacebookBot
Allow: /
User-agent: Meta-ExternalAgent
Allow: /

# --- Common Crawl (feeds many downstream LLMs) ---
User-agent: CCBot
Allow: /

# --- Others ---
User-agent: DuckAssistBot
Allow: /
User-agent: MistralAI-User
Allow: /
User-agent: cohere-ai
Allow: /

# =========================================================
# CRAWL-RATE LIMITING (SEO tools — ignored by Google)
# =========================================================
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 10

# =========================================================
# AGGRESSIVE / SCRAPER / HARVESTER BOTS — BLOCK
# =========================================================
User-agent: SiteBot
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: WebBandit
Disallow: /

# =========================
# SITEMAPS
# =========================
Sitemap: https://angeo.dev/sitemap_index.xml