Search

Matched domain: probe.dev

`robots.txt`

User-agent: *
Allow: /

# Important pages for crawlers
Allow: /api/analyze
Allow: /dashboard
Allow: /signup
Allow: /login

# Block sensitive areas
Disallow: /api/internal/
Disallow: /api/newsletter/
Disallow: /admin/

# Sitemap location
Sitemap: https://probe.dev/sitemap.xml

# Crawl-delay (optional, helps with server load)
Crawl-delay: 1

# Media file recommendations
User-agent: *
Allow: *.css
Allow: *.js
Allow: *.png
Allow: *.jpg
Allow: *.jpeg
Allow: *.gif
Allow: *.svg

Look up this url in the url tool https://probe.dev/.well-known/acme-challenge: 404 application/json; charset=utf-8
https://probe.dev/.well-known/csvm: 404 text/html; charset=utf-8
https://probe.dev/.well-known/nostr.json: 404 text/html; charset=utf-8
https://probe.dev/.well-known/security.txt: 404 text/html; charset=utf-8
https://probe.dev/.well-known/traffic-advice: 404 text/html; charset=utf-8