Search

#Google Search Engine Robot
User-agent: Googlebot
Allow: /*
Allow: /p/*
Allow: /q/*

#Microsoft Search Engine Robot
User-Agent: msnbot
Allow: /*
Allow: /p/*
Allow: /q/*

#Twitter Card Engine Robot
User-Agent: Twitterbot
Allow: /*
Allow: /p/*
Allow: /q/*

#Yahoo! Search Engine Robot
User-Agent: Slurp
Allow: /*
Allow: /p/*
Allow: /q/*

#Yandex Search Engine Robot
User-agent: Yandex
Allow: /*
Allow: /p/*
Allow: /q/*

# Restrict Web Archive for the time being
# User-agent: ia_archiver
# Disallow: /*
# User-agent: archive.org_bot
# Disallow: /*
User-agent: ia_archiver
Allow: /

# Block SemRush bot from causing errors
User-agent: SemrushBot-SA
Disallow: /
User-agent: SemrushBot-BA
Disallow: /
User-agent: SemrushBot-SI
Disallow: /
User-agent: SemrushBot-SWA
Disallow: /
User-agent: SemrushBot-CT
Disallow: /
User-agent: SemrushBot-BM
Disallow: /
User-agent: SemrushBot-SEOAB
Disallow: /

# Remove pesky data crawler from Hunter.io
User-agent: VelenPublicWebCrawler
Bot Disallow: /

# Every bot that might possibly read and respect this file.
User-agent: *
Allow: /*
Allow: /p/*
Allow: /q/*

# Wait 1 second between successive requests. Because copy the best
# Crawl-delay: 1

# Independent of user agent. Links in the sitemap are full URLs using https:// and need to match
# the protocol of the sitemap.
Sitemap: https://smashnotes.com/sitemap.xml.gz