Search

Matched domain: spyfu.com
`robots.txt`

# robots.txt - Updated: 06/05/25
#
# Sitemap Files
Sitemap: https://www.spyfu.com/sitemap.xml
Sitemap: https://www.spyfu.com/blog/sitemap.xml
Sitemap: https://www.spyfu.com/SpyGPT/SpyGPT_Software_Sitemap.xml
Sitemap: https://www.spyfu.com/SpyGPT/SpyGPT_Brands_Sitemap.xml
Sitemap: https://www.spyfu.com/SpyGPT/SpyGPT_Companies_Sitemap.xml
#


###############################################################################
# 1. Googlebot gets full access to ?query= / &query= / ?vwot= / &vwot= (For now)
# noindex for googlebot implemented in AppBaseController.cs and associated layout partials
###############################################################################

User-agent: Googlebot
# / / / / / / / / / / / / / Result Pages / / / / / / / / / / / / /
Allow: /*?query=*
Allow: /*&query=*
# / / / / / / / / / / / / / Lead Pages / / / / / / / / / / / / /
Disallow: *leads?*
Disallow: *?titleSlug=*
# / / / / / / / / / / / / / Top List Pages / / / / / / / / / / / / /
Allow: *?listId=3$
Disallow: *listId=*
# / / / / / / / / / / / / / Blog Pages / / / / / / / / / / / / /
Disallow: /blog/staging/
Disallow: /blog/ghost/
# / / / / / / / / / / / / / Directories / / / / / / / / / / / / /
Disallow: /Admin/
Disallow: /admin/
Disallow: /apis/core_api/facebookdata/
Disallow: /Chat/
Disallow: /chart/
Disallow: /Chart/
Disallow: /includes/
Disallow: /industry/
Disallow: /Industry/
Disallow: /m/
Disallow: /NsaApi/Core/GetFacebookData*
Disallow: /PressRoom/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /term.aspx/
Disallow: /Term.aspx/
Disallow: /term_api/
Disallow: /themes/
Disallow: /ToolTips/
# / / / / / / / / / / / / / Pages / / / / / / / / / / / / /
Disallow: /CHANGELOG.txt
Disallow: /categories.aspx*
Disallow: /Categories.aspx*
Disallow: /chart.aspx*
Disallow: /Chart.aspx*
Disallow: /error.aspx*
Disallow: /forumns
Disallow: /Forumns
Disallow: /forums.aspx*
Disallow: /Forums.aspx*
Disallow: /GanttChart.aspx*
Disallow: /Host.aspx*
Disallow: /industry.aspx*
Disallow: /Industry.aspx*
Disallow: /industries.aspx*
Disallow: /Industries.aspx*
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /Login.aspx
Disallow: /MAINTAINERS.txt
Disallow: /MoreList.aspx*
Disallow: /MyAccountInfo.aspx*
Disallow: /o/UrlRedirector*
Disallow: /search.aspx
Disallow: /SubscribeOrLogin.aspx*
Disallow: /Tracking/Add
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /UrlRedirector*
Disallow: /xmlrpc.php
# / / / / / / / / / / / / Terms / / / / / / / / / / / / / / /
Disallow: /*AllDomainResults*
Disallow: /*apiref*
Disallow: /*BrowseCompanies.aspx*
Disallow: /*ChartFeed*
Disallow: /*DmozTopic*
Disallow: /*FullDomainList*
Disallow: /*FullTermList*
Disallow: /*navigation.aspx*
Disallow: /*Navigation.aspx*
Disallow: /*SearchResults.aspx*
Disallow: /*SubscribeOrLogin.aspx*
Disallow: /*UrlRedirector*
Disallow: /*VennFeed*
#  / / / / / / / / / / / / / All of UK / / / / / / / / / / / / /
Disallow: /uk/
Disallow: /UK/
#  / / / / / / / / / / / / / add users page / / / / / / / / / / / / /
Disallow: /add-users*
#  / / / / / / / / / / / / / cached serp / / / / / / / / / / / / /
Disallow: /cachedserp
Disallow: /upgrade*
Disallow: /account*


###############################################################################
# 2. GSC inspection tool :)
# noindex for googlebot implemented in AppBaseController.cs and associated layout partials
###############################################################################

User-agent: Google-InspectionTool
# / / / / / / / / / / / / / Result Pages / / / / / / / / / / / / /
Allow: /*?query=*
Allow: /*&query=*
# / / / / / / / / / / / / / Lead Pages / / / / / / / / / / / / /
Disallow: *leads?*
Disallow: *?titleSlug=*
# / / / / / / / / / / / / / Top List Pages / / / / / / / / / / / / /
Allow: *?listId=3$
Disallow: *listId=*
# / / / / / / / / / / / / / Blog Pages / / / / / / / / / / / / /
Disallow: /blog/staging/
Disallow: /blog/ghost/
# / / / / / / / / / / / / / Directories / / / / / / / / / / / / /
Disallow: /Admin/
Disallow: /admin/
Disallow: /apis/core_api/facebookdata/
Disallow: /Chat/
Disallow: /chart/
Disallow: /Chart/
Disallow: /includes/
Disallow: /industry/
Disallow: /Industry/
Disallow: /m/
Disallow: /NsaApi/Core/GetFacebookData*
Disallow: /PressRoom/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /term.aspx/
Disallow: /Term.aspx/
Disallow: /term_api/
Disallow: /themes/
Disallow: /ToolTips/
# / / / / / / / / / / / / / Pages / / / / / / / / / / / / /
Disallow: /CHANGELOG.txt
Disallow: /categories.aspx*
Disallow: /Categories.aspx*
Disallow: /chart.aspx*
Disallow: /Chart.aspx*
Disallow: /error.aspx*
Disallow: /forumns
Disallow: /Forumns
Disallow: /forums.aspx*
Disallow: /Forums.aspx*
Disallow: /GanttChart.aspx*
Disallow: /Host.aspx*
Disallow: /industry.aspx*
Disallow: /Industry.aspx*
Disallow: /industries.aspx*
Disallow: /Industries.aspx*
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /Login.aspx
Disallow: /MAINTAINERS.txt
Disallow: /MoreList.aspx*
Disallow: /MyAccountInfo.aspx*
Disallow: /o/UrlRedirector*
Disallow: /search.aspx
Disallow: /SubscribeOrLogin.aspx*
Disallow: /Tracking/Add
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /UrlRedirector*
Disallow: /xmlrpc.php
# / / / / / / / / / / / / Terms / / / / / / / / / / / / / / /
Disallow: /*AllDomainResults*
Disallow: /*apiref*
Disallow: /*BrowseCompanies.aspx*
Disallow: /*ChartFeed*
Disallow: /*DmozTopic*
Disallow: /*FullDomainList*
Disallow: /*FullTermList*
Disallow: /*navigation.aspx*
Disallow: /*Navigation.aspx*
Disallow: /*SearchResults.aspx*
Disallow: /*SubscribeOrLogin.aspx*
Disallow: /*UrlRedirector*
Disallow: /*VennFeed*
#  / / / / / / / / / / / / / All of UK / / / / / / / / / / / / /
Disallow: /uk/
Disallow: /UK/
#  / / / / / / / / / / / / / add users page / / / / / / / / / / / / /
Disallow: /add-users*
#  / / / / / / / / / / / / / cached serp / / / / / / / / / / / / /
Disallow: /cachedserp
Disallow: /upgrade*
Disallow: /account*


User-agent: ChatGPT-User
# / / / / / / / / / / / / / Result Pages / / / / / / / / / / / / /
Allow: /*?query=*
Allow: /*&query=*
# / / / / / / / / / / / / / Lead Pages / / / / / / / / / / / / /
Disallow: *leads?*
Disallow: *?titleSlug=*
# / / / / / / / / / / / / / Top List Pages / / / / / / / / / / / / /
Allow: *?listId=3$
Disallow: *listId=*
# / / / / / / / / / / / / / Blog Pages / / / / / / / / / / / / /
Disallow: /blog/staging/
Disallow: /blog/ghost/
# / / / / / / / / / / / / / Directories / / / / / / / / / / / / /
Disallow: /Admin/
Disallow: /admin/
Disallow: /apis/core_api/facebookdata/
Disallow: /Chat/
Disallow: /chart/
Disallow: /Chart/
Disallow: /includes/
Disallow: /industry/
Disallow: /Industry/
Disallow: /m/
Disallow: /NsaApi/Core/GetFacebookData*
Disallow: /PressRoom/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /term.aspx/
Disallow: /Term.aspx/
Disallow: /term_api/
Disallow: /themes/
Disallow: /ToolTips/
# / / / / / / / / / / / / / Pages / / / / / / / / / / / / /
Disallow: /CHANGELOG.txt
Disallow: /categories.aspx*
Disallow: /Categories.aspx*
Disallow: /chart.aspx*
Disallow: /Chart.aspx*
Disallow: /error.aspx*
Disallow: /forumns
Disallow: /Forumns
Disallow: /forums.aspx*
Disallow: /Forums.aspx*
Disallow: /GanttChart.aspx*
Disallow: /Host.aspx*
Disallow: /industry.aspx*
Disallow: /Industry.aspx*
Disallow: /industries.aspx*
Disallow: /Industries.aspx*
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /Login.aspx
Disallow: /MAINTAINERS.txt
Disallow: /MoreList.aspx*
Disallow: /MyAccountInfo.aspx*
Disallow: /o/UrlRedirector*
Disallow: /search.aspx
Disallow: /SubscribeOrLogin.aspx*
Disallow: /Tracking/Add
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /UrlRedirector*
Disallow: /xmlrpc.php
# / / / / / / / / / / / / Terms / / / / / / / / / / / / / / /
Disallow: /*AllDomainResults*
Disallow: /*apiref*
Disallow: /*BrowseCompanies.aspx*
Disallow: /*ChartFeed*
Disallow: /*DmozTopic*
Disallow: /*FullDomainList*
Disallow: /*FullTermList*
Disallow: /*navigation.aspx*
Disallow: /*Navigation.aspx*
Disallow: /*SearchResults.aspx*
Disallow: /*SubscribeOrLogin.aspx*
Disallow: /*UrlRedirector*
Disallow: /*VennFeed*
#  / / / / / / / / / / / / / All of UK / / / / / / / / / / / / /
Disallow: /uk/
Disallow: /UK/
#  / / / / / / / / / / / / / add users page / / / / / / / / / / / / /
Disallow: /add-users*
#  / / / / / / / / / / / / / cached serp / / / / / / / / / / / / /
Disallow: /cachedserp
Disallow: /upgrade*
Disallow: /account*


User-agent: Claude-User
# / / / / / / / / / / / / / Result Pages / / / / / / / / / / / / /
Allow: /*?query=*
Allow: /*&query=*
# / / / / / / / / / / / / / Lead Pages / / / / / / / / / / / / /
Disallow: *leads?*
Disallow: *?titleSlug=*
# / / / / / / / / / / / / / Top List Pages / / / / / / / / / / / / /
Allow: *?listId=3$
Disallow: *listId=*
# / / / / / / / / / / / / / Blog Pages / / / / / / / / / / / / /
Disallow: /blog/staging/
Disallow: /blog/ghost/
# / / / / / / / / / / / / / Directories / / / / / / / / / / / / /
Disallow: /Admin/
Disallow: /admin/
Disallow: /apis/core_api/facebookdata/
Disallow: /Chat/
Disallow: /chart/
Disallow: /Chart/
Disallow: /includes/
Disallow: /industry/
Disallow: /Industry/
Disallow: /m/
Disallow: /NsaApi/Core/GetFacebookData*
Disallow: /PressRoom/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /term.aspx/
Disallow: /Term.aspx/
Disallow: /term_api/
Disallow: /themes/
Disallow: /ToolTips/
# / / / / / / / / / / / / / Pages / / / / / / / / / / / / /
Disallow: /CHANGELOG.txt
Disallow: /categories.aspx*
Disallow: /Categories.aspx*
Disallow: /chart.aspx*
Disallow: /Chart.aspx*
Disallow: /error.aspx*
Disallow: /forumns
Disallow: /Forumns
Disallow: /forums.aspx*
Disallow: /Forums.aspx*
Disallow: /GanttChart.aspx*
Disallow: /Host.aspx*
Disallow: /industry.aspx*
Disallow: /Industry.aspx*
Disallow: /industries.aspx*
Disallow: /Industries.aspx*
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /Login.aspx
Disallow: /MAINTAINERS.txt
Disallow: /MoreList.aspx*
Disallow: /MyAccountInfo.aspx*
Disallow: /o/UrlRedirector*
Disallow: /search.aspx
Disallow: /SubscribeOrLogin.aspx*
Disallow: /Tracking/Add
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /UrlRedirector*
Disallow: /xmlrpc.php
# / / / / / / / / / / / / Terms / / / / / / / / / / / / / / /
Disallow: /*AllDomainResults*
Disallow: /*apiref*
Disallow: /*BrowseCompanies.aspx*
Disallow: /*ChartFeed*
Disallow: /*DmozTopic*
Disallow: /*FullDomainList*
Disallow: /*FullTermList*
Disallow: /*navigation.aspx*
Disallow: /*Navigation.aspx*
Disallow: /*SearchResults.aspx*
Disallow: /*SubscribeOrLogin.aspx*
Disallow: /*UrlRedirector*
Disallow: /*VennFeed*
#  / / / / / / / / / / / / / All of UK / / / / / / / / / / / / /
Disallow: /uk/
Disallow: /UK/
#  / / / / / / / / / / / / / add users page / / / / / / / / / / / / /
Disallow: /add-users*
#  / / / / / / / / / / / / / cached serp / / / / / / / / / / / / /
Disallow: /cachedserp
Disallow: /upgrade*
Disallow: /account*


User-agent: Perplexity-User
# / / / / / / / / / / / / / Result Pages / / / / / / / / / / / / /
Allow: /*?query=*
Allow: /*&query=*
# / / / / / / / / / / / / / Lead Pages / / / / / / / / / / / / /
Disallow: *leads?*
Disallow: *?titleSlug=*
# / / / / / / / / / / / / / Top List Pages / / / / / / / / / / / / /
Allow: *?listId=3$
Disallow: *listId=*
# / / / / / / / / / / / / / Blog Pages / / / / / / / / / / / / /
Disallow: /blog/staging/
Disallow: /blog/ghost/
# / / / / / / / / / / / / / Directories / / / / / / / / / / / / /
Disallow: /Admin/
Disallow: /admin/
Disallow: /apis/core_api/facebookdata/
Disallow: /Chat/
Disallow: /chart/
Disallow: /Chart/
Disallow: /includes/
Disallow: /industry/
Disallow: /Industry/
Disallow: /m/
Disallow: /NsaApi/Core/GetFacebookData*
Disallow: /PressRoom/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /term.aspx/
Disallow: /Term.aspx/
Disallow: /term_api/
Disallow: /themes/
Disallow: /ToolTips/
# / / / / / / / / / / / / / Pages / / / / / / / / / / / / /
Disallow: /CHANGELOG.txt
Disallow: /categories.aspx*
Disallow: /Categories.aspx*
Disallow: /chart.aspx*
Disallow: /Chart.aspx*
Disallow: /error.aspx*
Disallow: /forumns
Disallow: /Forumns
Disallow: /forums.aspx*
Disallow: /Forums.aspx*
Disallow: /GanttChart.aspx*
Disallow: /Host.aspx*
Disallow: /industry.aspx*
Disallow: /Industry.aspx*
Disallow: /industries.aspx*
Disallow: /Industries.aspx*
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /Login.aspx
Disallow: /MAINTAINERS.txt
Disallow: /MoreList.aspx*
Disallow: /MyAccountInfo.aspx*
Disallow: /o/UrlRedirector*
Disallow: /search.aspx
Disallow: /SubscribeOrLogin.aspx*
Disallow: /Tracking/Add
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /UrlRedirector*
Disallow: /xmlrpc.php
# / / / / / / / / / / / / Terms / / / / / / / / / / / / / / /
Disallow: /*AllDomainResults*
Disallow: /*apiref*
Disallow: /*BrowseCompanies.aspx*
Disallow: /*ChartFeed*
Disallow: /*DmozTopic*
Disallow: /*FullDomainList*
Disallow: /*FullTermList*
Disallow: /*navigation.aspx*
Disallow: /*Navigation.aspx*
Disallow: /*SearchResults.aspx*
Disallow: /*SubscribeOrLogin.aspx*
Disallow: /*UrlRedirector*
Disallow: /*VennFeed*
#  / / / / / / / / / / / / / All of UK / / / / / / / / / / / / /
Disallow: /uk/
Disallow: /UK/
#  / / / / / / / / / / / / / add users page / / / / / / / / / / / / /
Disallow: /add-users*
#  / / / / / / / / / / / / / cached serp / / / / / / / / / / / / /
Disallow: /cachedserp
Disallow: /upgrade*
Disallow: /account*


User-agent: MistralAI-User
# / / / / / / / / / / / / / Result Pages / / / / / / / / / / / / /
Allow: /*?query=*
Allow: /*&query=*
# / / / / / / / / / / / / / Lead Pages / / / / / / / / / / / / /
Disallow: *leads?*
Disallow: *?titleSlug=*
# / / / / / / / / / / / / / Top List Pages / / / / / / / / / / / / /
Allow: *?listId=3$
Disallow: *listId=*
# / / / / / / / / / / / / / Blog Pages / / / / / / / / / / / / /
Disallow: /blog/staging/
Disallow: /blog/ghost/
# / / / / / / / / / / / / / Directories / / / / / / / / / / / / /
Disallow: /Admin/
Disallow: /admin/
Disallow: /apis/core_api/facebookdata/
Disallow: /Chat/
Disallow: /chart/
Disallow: /Chart/
Disallow: /includes/
Disallow: /industry/
Disallow: /Industry/
Disallow: /m/
Disallow: /NsaApi/Core/GetFacebookData*
Disallow: /PressRoom/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /term.aspx/
Disallow: /Term.aspx/
Disallow: /term_api/
Disallow: /themes/
Disallow: /ToolTips/
# / / / / / / / / / / / / / Pages / / / / / / / / / / / / /
Disallow: /CHANGELOG.txt
Disallow: /categories.aspx*
Disallow: /Categories.aspx*
Disallow: /chart.aspx*
Disallow: /Chart.aspx*
Disallow: /error.aspx*
Disallow: /forumns
Disallow: /Forumns
Disallow: /forums.aspx*
Disallow: /Forums.aspx*
Disallow: /GanttChart.aspx*
Disallow: /Host.aspx*
Disallow: /industry.aspx*
Disallow: /Industry.aspx*
Disallow: /industries.aspx*
Disallow: /Industries.aspx*
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /Login.aspx
Disallow: /MAINTAINERS.txt
Disallow: /MoreList.aspx*
Disallow: /MyAccountInfo.aspx*
Disallow: /o/UrlRedirector*
Disallow: /search.aspx
Disallow: /SubscribeOrLogin.aspx*
Disallow: /Tracking/Add
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /UrlRedirector*
Disallow: /xmlrpc.php
# / / / / / / / / / / / / Terms / / / / / / / / / / / / / / /
Disallow: /*AllDomainResults*
Disallow: /*apiref*
Disallow: /*BrowseCompanies.aspx*
Disallow: /*ChartFeed*
Disallow: /*DmozTopic*
Disallow: /*FullDomainList*
Disallow: /*FullTermList*
Disallow: /*navigation.aspx*
Disallow: /*Navigation.aspx*
Disallow: /*SearchResults.aspx*
Disallow: /*SubscribeOrLogin.aspx*
Disallow: /*UrlRedirector*
Disallow: /*VennFeed*
#  / / / / / / / / / / / / / All of UK / / / / / / / / / / / / /
Disallow: /uk/
Disallow: /UK/
#  / / / / / / / / / / / / / add users page / / / / / / / / / / / / /
Disallow: /add-users*
#  / / / / / / / / / / / / / cached serp / / / / / / / / / / / / /
Disallow: /cachedserp
Disallow: /upgrade*
Disallow: /account*


User-agent: *
# / / / / / / / / / / / / / Result Pages / / / / / / / / / / / / /
Disallow: /*?query=*
Disallow: /*&query=*
# / / / / / / / / / / / / / Lead Pages / / / / / / / / / / / / /
Disallow: *leads?*
Disallow: *?titleSlug=*
# / / / / / / / / / / / / / Top List Pages / / / / / / / / / / / / /
Allow: *?listId=3$
Disallow: *listId=*
# / / / / / / / / / / / / / Blog Pages / / / / / / / / / / / / /
Disallow: /blog/staging/
Disallow: /blog/ghost/
# / / / / / / / / / / / / / Directories / / / / / / / / / / / / /
Disallow: /Admin/
Disallow: /admin/
Disallow: /apis/core_api/facebookdata/
Disallow: /Chat/
Disallow: /chart/
Disallow: /Chart/
Disallow: /includes/
Disallow: /industry/
Disallow: /Industry/
Disallow: /m/
Disallow: /NsaApi/Core/GetFacebookData*
Disallow: /PressRoom/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /term.aspx/
Disallow: /Term.aspx/
Disallow: /term_api/
Disallow: /themes/
Disallow: /ToolTips/
# / / / / / / / / / / / / / Pages / / / / / / / / / / / / /
Disallow: /CHANGELOG.txt
Disallow: /categories.aspx*
Disallow: /Categories.aspx*
Disallow: /chart.aspx*
Disallow: /Chart.aspx*
Disallow: /error.aspx*
Disallow: /forumns
Disallow: /Forumns
Disallow: /forums.aspx*
Disallow: /Forums.aspx*
Disallow: /GanttChart.aspx*
Disallow: /Host.aspx*
Disallow: /industry.aspx*
Disallow: /Industry.aspx*
Disallow: /industries.aspx*
Disallow: /Industries.aspx*
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /Login.aspx
Disallow: /MAINTAINERS.txt
Disallow: /MoreList.aspx*
Disallow: /MyAccountInfo.aspx*
Disallow: /o/UrlRedirector*
Disallow: /search.aspx
Disallow: /SubscribeOrLogin.aspx*
Disallow: /Tracking/Add
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /UrlRedirector*
Disallow: /xmlrpc.php
# / / / / / / / / / / / / Terms / / / / / / / / / / / / / / /
Disallow: /*AllDomainResults*
Disallow: /*apiref*
Disallow: /*BrowseCompanies.aspx*
Disallow: /*ChartFeed*
Disallow: /*DmozTopic*
Disallow: /*FullDomainList*
Disallow: /*FullTermList*
Disallow: /*navigation.aspx*
Disallow: /*Navigation.aspx*
Disallow: /*SearchResults.aspx*
Disallow: /*SubscribeOrLogin.aspx*
Disallow: /*UrlRedirector*
Disallow: /*VennFeed*
#  / / / / / / / / / / / / / All of UK / / / / / / / / / / / / /
Disallow: /uk/
Disallow: /UK/
#  / / / / / / / / / / / / / add users page / / / / / / / / / / / / /
Disallow: /add-users*
#  / / / / / / / / / / / / / cached serp / / / / / / / / / / / / /
Disallow: /cachedserp
Disallow: /upgrade*
Disallow: /account*


User-agent: Yandex
Disallow: /outreach/leads/
# / / / / / / / / / / / / / Result Pages / / / / / / / / / / / / /
Disallow: /*?query=*
Disallow: /*&query=*
# / / / / / / / / / / / / / Lead Pages / / / / / / / / / / / / /
Disallow: *leads?*
Disallow: *?titleSlug=*
# / / / / / / / / / / / / / Top List Pages / / / / / / / / / / / / /
Allow: *?listId=3$
Disallow: *listId=*
# / / / / / / / / / / / / / Blog Pages / / / / / / / / / / / / /
Disallow: /blog/staging/
Disallow: /blog/ghost/
# / / / / / / / / / / / / / Directories / / / / / / / / / / / / /
Disallow: /Admin/
Disallow: /admin/
Disallow: /apis/core_api/facebookdata/
Disallow: /Chat/
Disallow: /chart/
Disallow: /Chart/
Disallow: /includes/
Disallow: /industry/
Disallow: /Industry/
Disallow: /m/
Disallow: /NsaApi/Core/GetFacebookData*
Disallow: /PressRoom/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /term.aspx/
Disallow: /Term.aspx/
Disallow: /term_api/
Disallow: /themes/
Disallow: /ToolTips/
# / / / / / / / / / / / / / Pages / / / / / / / / / / / / /
Disallow: /CHANGELOG.txt
Disallow: /categories.aspx*
Disallow: /Categories.aspx*
Disallow: /chart.aspx*
Disallow: /Chart.aspx*
Disallow: /error.aspx*
Disallow: /forumns
Disallow: /Forumns
Disallow: /forums.aspx*
Disallow: /Forums.aspx*
Disallow: /GanttChart.aspx*
Disallow: /Host.aspx*
Disallow: /industry.aspx*
Disallow: /Industry.aspx*
Disallow: /industries.aspx*
Disallow: /Industries.aspx*
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /Login.aspx
Disallow: /MAINTAINERS.txt
Disallow: /MoreList.aspx*
Disallow: /MyAccountInfo.aspx*
Disallow: /o/UrlRedirector*
Disallow: /search.aspx
Disallow: /SubscribeOrLogin.aspx*
Disallow: /Tracking/Add
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /UrlRedirector*
Disallow: /xmlrpc.php
# / / / / / / / / / / / / Terms / / / / / / / / / / / / / / /
Disallow: /*AllDomainResults*
Disallow: /*apiref*
Disallow: /*BrowseCompanies.aspx*
Disallow: /*ChartFeed*
Disallow: /*DmozTopic*
Disallow: /*FullDomainList*
Disallow: /*FullTermList*
Disallow: /*navigation.aspx*
Disallow: /*Navigation.aspx*
Disallow: /*SearchResults.aspx*
Disallow: /*SubscribeOrLogin.aspx*
Disallow: /*UrlRedirector*
Disallow: /*VennFeed*
#  / / / / / / / / / / / / / All of UK / / / / / / / / / / / / /
Disallow: /uk/
Disallow: /UK/
#  / / / / / / / / / / / / / add users page / / / / / / / / / / / / /
Disallow: /add-users*
#  / / / / / / / / / / / / / cached serp / / / / / / / / / / / / /
Disallow: /cachedserp
Disallow: /upgrade*
Disallow: /account*
Look up this url in the url tool https://spyfu.com/.well-known/acme-challenge: 404 text/html; charset=utf-8
https://spyfu.com/.well-known/csvm: 404 text/html; charset=utf-8
https://spyfu.com/.well-known/nostr.json: 404 text/html; charset=utf-8
https://spyfu.com/.well-known/security.txt: 404 text/html; charset=utf-8
https://spyfu.com/.well-known/traffic-advice: 404 text/html; charset=utf-8