Search

# robots.txt - block common restricted paths for all crawlers

User-agent: *
# Admin and system areas
Disallow: /admin/
Disallow: /administrator/
Disallow: /backend/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /cms/
Disallow: /core/
Disallow: /system/
Disallow: /dashboard/
Disallow: /manage/
Disallow: /panel/
Disallow: /controlpanel/
Disallow: /cpanel/

# Login and authentication
Disallow: /login/
Disallow: /logout/
Disallow: /register/
Disallow: /signup/
Disallow: /user/
Disallow: /account/
Disallow: /profile/
Disallow: /auth/
Disallow: /session/

# Sensitive resources
Disallow: /cgi-bin/
Disallow: /server-status/
Disallow: /error_log
Disallow: /logs/
Disallow: /tmp/
Disallow: /cache/
Disallow: /private/
Disallow: /secret/
Disallow: /config/
Disallow: /settings/
Disallow: /data/
Disallow: /uploads/
Disallow: /backup/
Disallow: /db/
Disallow: /_internal/
Disallow: /.git/
Disallow: /.svn/
Disallow: /.env

# API and development endpoints
Disallow: /api/
Disallow: /dev/
Disallow: /test/
Disallow: /staging/
Disallow: /beta/

# Allow everything else
Allow: /

# Sitemap (optional)
# Sitemap: https://example.com/sitemap.xml