Search

User-Agent: *
# Block pages with no unique content
Disallow: /startTopic/
Disallow: /discover/unread/
Disallow: /markallread/
Disallow: /staff/
Disallow: /cookie/
Disallow: /online/
Disallow: /discover/
Disallow: /leaderboard/
Disallow: /search/
Disallow: /tags/
Disallow: /*?advancedSearchForm=
Disallow: /register/
Disallow: /lostpassword/
Disallow: /login/

# Block faceted pages and 301 redirect pages
Disallow: /*?do=add
Disallow: /*?do=submit
Disallow: /*?do=log
Disallow: /*?do=buy
Disallow: /*?do=downloadd
Disallow: /*?sortby=
Disallow: /*?filter=
Disallow: /*?tab=
Disallow: /*?do=
Disallow: /*ref=
Disallow: /*?forumId*
Disallow: /*?&controller=embed
Disallow: /*?controller=

# Block profile pages as these have little unique value, consume a lot of crawl time and contain hundreds of 301 links
# Disallow: /profile/

# Custom rules
Disallow: /checkout/
Disallow: /db/
Disallow: /modcp/
Disallow: /forums/providers/
Disallow: /earnings/
Disallow: /files/manage
Disallow: /applications/core/interface/file/attachment.php

# Sitemap URL
Sitemap: https://codefling.com/sitemap.php