Search

# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:
# User-agent: *
# Disallow: /
Sitemap: https://coursereport-s3-production.global.ssl.fastly.net/sitemaps/sitemap.xml.gz

User-agent: *
Crawl-delay: 10
Disallow: /results
Disallow: /login?*
Disallow: /logout?*
Disallow: /admin
Disallow: /schools/*/admin
# Disallow: /schools/*?page=*
# Disallow: /schools/*&page=*
Disallow: /schools/*?reviews_page=*&news_page=*
Disallow: /schools/*?news_page=*&reviews_page=*
Disallow: /reviews/*/votes
Disallow: /*claim-page
Disallow: /*redirect_path
# Disallow: /*shared_review
Disallow: /*contact_redirect
Disallow: /do-not-sell
Disallow: /posts/index*
Disallow: /*cost=
Disallow: /*focus=
Disallow: /*location=
Disallow: /*review_offer_id=
Disallow: /*track=
Disallow: /admin/
Disallow: /users/auth/
Disallow: /terms-of-service/
# Disallow: /*.modal$