Search

#v1.2
User-agent: *

Crawl-delay: 10

 

# Specific rules for search engine bots

User-agent: Googlebot

Allow: /

 

User-agent: Bingbot

Allow: /

Crawl-delay: 90

 

User-agent: AdsBot-Google

Allow: /

 

User-agent: AdsBot-Google-Mobile

Allow: /

 

User-agent: AhrefsBot

Allow: /

 

User-agent: AhrefsSiteAudit

Allow: /

 

User-agent: SiteAuditBot

Allow: /

 

User-agent: Screaming Frog SEO Spider

Allow: /

 

User-agent: SemrushBot

Allow: /

 

# Special handling for some other bots

 

User-agent: YandexBot

Disallow: /

 

User-agent: Mail.RU

Disallow: /

 

# Sitemap Locations

 

# Directories
User-agent: *

Disallow: /app/

Disallow: /bin/

Disallow: /dev/

Disallow: /lib/

Disallow: /phpserver/

Disallow: /pkginfo/

Disallow: /pub/

Disallow: /api/

Disallow: /setup/

Disallow: /update/

Disallow: /var/

Disallow: /vendor/

 

# Paths (clean URLs)

Disallow: /cart

Disallow: /checkout

Disallow: /checkout/thank-you

Disallow: /my-account*

Disallow: /compare

Disallow: /track-order/

Disallow: /order-information/

Disallow: /search*

Disallow: /*?itemsPerPage=

Disallow: /search?q=*

Disallow: /index.php/

Disallow: /catalog/

Disallow: /catalog/product_compare/

Disallow: /catalog/category/view/

Disallow: /catalog/product/view/

Disallow: /catalogsearch/

Disallow: /control/

Disallow: /contacts/

Disallow: /customer/

Disallow: /customize/

Disallow: /newsletter/

Disallow: /sendfriend/

Disallow: /wishlist/

Disallow: /tag/

Disallow: /report/

Disallow: /review/

Disallow: *?bvstate=*

Disallow: /*SID=

 

# Files

Disallow: /composer.json

Disallow: /composer.lock

Disallow: /CONTRIBUTING.md

Disallow: /CONTRIBUTOR_LICENSE_AGREEMENT.html

Disallow: /COPYING.txt

Disallow: /Gruntfile.js

Disallow: /LICENSE.txt

Disallow: /LICENSE_AFL.txt

Disallow: /ISSUE_TEMPLATE.md

Disallow: /nginx.conf.sample

Disallow: /package.json

Disallow: /php.ini.sample

Disallow: /RELEASE_NOTES.txt

 

# Do not index pages that are sorted or filtered.

Disallow: /*?*product_list_mode=

Disallow: /*?*product_list_order=

Disallow: /*?*limit=

Disallow: /*?*product_list_dir=

Disallow: /*?*p=

Disallow: /*?*color=

Disallow: /*?*manufacturer=

Disallow: /*?*price=

Disallow: /*?*category=

Disallow: /*?*brand=

 

 

# Do not index session ID

Disallow: /*.php$

Disallow: /*?SID=

 

# CVS, SVN directory and dump files

Disallow: /*.cvs

Disallow: /*.zip$

Disallow: /*.svn$

Disallow: /*.idea$

Disallow: /*.sql$

Disallow: /*.sql.gz$

Disallow: /*.tgz$

Disallow: /*.tar.gz$

Disallow: /*.tar.bz2$

Sitemap: https://www.sparkfun.com/media/sitemap.xml