Search

#05-Feb-26: all user agents
User-agent: *
Disallow: /0000-*/
Disallow: /accueil/
Disallow: /bas*/
Disallow: /*jahia/
Disallow: /ex*/
Disallow: /Jahia/
Disallow: /refonte-*/
Disallow: /test*/
Disallow: /modules/
Disallow: /webdav/
Disallow: /cms/
Disallow: /generated-resources/
Disallow: /*?*actunilMenuParam=*
Disallow: /*?*actunilParam=*
Disallow: /*?*c=*
Disallow: /*?*cl=*
Disallow: /*?*doLogin=*
Disallow: /*?*matrix=*
Disallow: /*?*pubsIdParam=*
Disallow: /*?*redirect=*
Disallow: /*?*rememberme=*
Disallow: /*?*set_language=*
Disallow: /*?*showActu=*
Disallow: /*?*showFrom=*
Disallow: /*?*site=*
Disallow: /*?*url_params=*
Disallow: /*?*url=*
Disallow: /*?*utm_campaign=*
Disallow: /*?*utm_medium=*
Disallow: /*?*utm_source_platform=*
Disallow: /*?*utm_source=*
Disallow: /*?*CSRFTOKEN=*
Disallow: /*?*channelIds=*
Disallow: /*?*sortedBy=*
Disallow: /*?*status=*
Disallow: /*?*publicationStatus=*
Disallow: /*?*summarize=*
Disallow: /*?*languages=*
Disallow: /*?*size=*
Disallow: /*?*windowDays=*
Disallow: /*?*resourceType=*
Disallow: /*?*resourceId=*
Disallow: /*?*eco=*
Disallow: /*?*beginEventDate=*
Disallow: /*?*endEventDate=*
Disallow: /*?*nodeIdK=*
Disallow: /*?*parentNodeIdK=*
Disallow: /*mobileMenu.do*
Disallow: /*resourcesProxy.do*
Disallow: /*generateEventIcs.do*
Disallow: /*newsMostViewedProxy.do*
Disallow: /*newsProxy.do*
Disallow: /*eventsProxy.do*
Disallow: /*?*cat_publication*
Disallow: /*?*cat_mastersthesis*
Disallow: /*?*cat_phdthesis=*
Disallow: /*?*current_cat*
Allow: /

#26-Feb-24: add sitemaps index
Sitemap: https://www.unil.ch/sitemap_www.xml

#11-Mar-25: exclude some ai bots
# Block all known AI crawlers and assistants
# from using content for training AI models.
# Source: https://robotstxt.com/ai
User-Agent: ClaudeBot
User-Agent: Claude-User
User-Agent: Claude-SearchBot
User-Agent: CCBot
User-Agent: Googlebot-Extended
User-Agent: Applebot-Extended
User-Agent: Facebookbot
User-Agent: Meta-ExternalAgent
User-Agent: Meta-ExternalFetcher
User-Agent: diffbot
User-Agent: PerplexityBot
User-Agent: Perplexity‑User
User-Agent: Omgili
User-Agent: Omgilibot
User-Agent: webzio-extended
User-Agent: ImagesiftBot
User-Agent: Bytespider
User-agent: TikTokSpider
User-Agent: Amazonbot
User-Agent: Youbot
User-Agent: SemrushBot-OCOB
User-Agent: Petalbot
User-Agent: VelenPublicWebCrawler
User-Agent: TurnitinBot
User-Agent: Timpibot
User-Agent: OAI-SearchBot
User-Agent: ICC-Crawler
User-Agent: AI2Bot
User-Agent: AI2Bot-Dolma
User-Agent: DataForSeoBot
User-Agent: AwarioBot
User-Agent: AwarioSmartBot
User-Agent: AwarioRssBot
User-Agent: Google-CloudVertexBot
User-Agent: PanguBot
User-Agent: Kangaroo Bot
User-Agent: Sentibot
User-Agent: img2dataset
User-Agent: Meltwater
User-Agent: Seekr
User-Agent: peer39_crawler
User-Agent: cohere-ai
User-Agent: cohere-training-data-crawler
User-Agent: DuckAssistBot
User-Agent: Scrapy
User-Agent: Cotoyogi
User-Agent: aiHitBot
User-Agent: Factset_spyderbot
User-Agent: FirecrawlAgent
Disallow: /
DisallowAITraining: /