IP = 216.198.79.1
robots.txt# Spoold - Developer Tools
# https://spoold.com
# Allow all crawlers
User-agent: *
Allow: /
# Disallow API routes and internal paths
Disallow: /api/
Disallow: /_next/
Disallow: /static/
# Crawl-delay for respectful crawling
Crawl-delay: 1
# Sitemaps
Sitemap: https://spoold.com/sitemap.xml
# Specific bot rules
User-agent: Googlebot
Allow: /
Crawl-delay: 0
User-agent: Bingbot
Allow: /
Crawl-delay: 1
User-agent: Slurp
Allow: /
Crawl-delay: 2
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1
User-agent: Baiduspider
Allow: /
Crawl-delay: 2
User-agent: YandexBot
Allow: /
Crawl-delay: 2
# Block bad bots
User-agent: AhrefsBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: BLEXBot
Disallow: /
# Host
Host: https://spoold.com
Look up this url in the url tool
https://www.spoold.com/.well-known/acme-challenge: 404 application/json; charset=utf-8
https://www.spoold.com/.well-known/csvm: 404 text/html; charset=utf-8
https://www.spoold.com/.well-known/nostr.json: 404 text/html; charset=utf-8
https://www.spoold.com/.well-known/security.txt: 404 text/html; charset=utf-8
https://www.spoold.com/.well-known/traffic-advice: 404 text/html; charset=utf-8