IP = 45.82.31.247
robots.txt# ForkLog robots.txt
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
# -----------------------
# Search Engines
# -----------------------
User-agent: Googlebot
Allow: /
User-agent: Bingbot
Allow: /
User-agent: YandexBot
Allow: /
User-agent: Googlebot-News
Allow: /
# -----------------------
# AI Search / Citation (answer engines) - full access
# -----------------------
User-agent: OAI-SearchBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: PerplexityBot
Allow: /
# -----------------------
# AI Training Crawlers - news only, plus the AI-systems policy page; everything else blocked
# -----------------------
User-agent: GPTBot
Allow: /news/
Allow: /for-ai-systems
Allow: /.well-known/ai-access
Disallow: /
User-agent: ClaudeBot
Allow: /news/
Allow: /for-ai-systems
Allow: /.well-known/ai-access
Disallow: /
User-agent: anthropic-ai
Allow: /news/
Allow: /for-ai-systems
Allow: /.well-known/ai-access
Disallow: /
User-agent: CCBot
Allow: /news/
Allow: /for-ai-systems
Allow: /.well-known/ai-access
Disallow: /
User-agent: Applebot-Extended
Allow: /news/
Allow: /for-ai-systems
Allow: /.well-known/ai-access
Disallow: /
User-agent: Meta-ExternalAgent
Allow: /news/
Allow: /for-ai-systems
Allow: /.well-known/ai-access
Disallow: /
User-agent: Amazonbot
Allow: /news/
Allow: /for-ai-systems
Allow: /.well-known/ai-access
Disallow: /
# Bytespider (ByteDance) - aggressive scraper: content fully blocked, only the AI policy page is readable
User-agent: Bytespider
Allow: /for-ai-systems
Allow: /.well-known/ai-access
Disallow: /
# -----------------------
# Heavy Commercial Crawlers
# -----------------------
User-agent: AhrefsBot
Crawl-delay: 10
User-agent: SemrushBot
Crawl-delay: 10
# -----------------------
# Sitemaps
# -----------------------
Sitemap: https://forklog.com/sitemap.xml
Sitemap: https://forklog.com/news-sitemap.xml
Sitemap: https://forklog.com/hub-archive/sitemap-hub.xml
# AI systems access and licensing - https://forklog.com/for-ai-systems
AI-Access: https://forklog.com/.well-known/ai-access
AI-Access-JSON: https://forklog.com/.well-known/ai-access.json
Look up this url in the url tool
https://forklog.com/.well-known/acme-challenge: 404 text/html; charset=UTF-8
https://forklog.com/.well-known/csvm: 404 text/html; charset=UTF-8
https://forklog.com/.well-known/nostr.json: 404 text/html; charset=UTF-8
https://forklog.com/.well-known/security.txt: 404 text/html; charset=UTF-8
https://forklog.com/.well-known/traffic-advice: 404 text/html; charset=UTF-8