IP = 131.142.198.210
robots.txt
# let search engines know where things are
#Sitemap: http://adsabs.harvard.edu/sitemap_index.xml
Sitemap: https://ui.adsabs.harvard.edu/sitemap/sitemap_index.xml
# Google
# http://www.google.com/bot.html
User-agent: Googlebot
Disallow: /cgi-bin/
#Allow: /abs/
Allow: /full/
# MS Live
# http://search.msn.com/msnbot.htm
User-agent: msnbot
Disallow: /cgi-bin/
#Allow: /abs/
Allow: /full/
# Yahoo
# http://help.yahoo.com/help/us/ysearch/slurp
User-agent: Slurp
Disallow: /cgi-bin/
#Allow: /abs/
Allow: /full/
# Ask.com
# http://about.ask.com/en/docs/about/webmasters.shtml
User-agent: Teoma
Disallow: /cgi-bin/
#Allow: /abs/
Disallow: /full/
# disallow harvesting from all other robots
User-agent: *
Disallow: /cgi-bin/
Disallow: /abs/
Disallow: /full/
Look up this url in the url tool
https://adsabs.harvard.edu/.well-known/acme-challenge: 404 text/html; charset=iso-8859-1
https://adsabs.harvard.edu/.well-known/csvm: 404 text/html; charset=iso-8859-1
https://adsabs.harvard.edu/.well-known/nostr.json: 404 text/html; charset=iso-8859-1
https://adsabs.harvard.edu/.well-known/security.txt: 404 text/html; charset=iso-8859-1
https://adsabs.harvard.edu/.well-known/traffic-advice: 404 text/html; charset=iso-8859-1