# ============================================================
# robots.txt — Thalgo
# Dernière mise à jour : avril 2026
# ============================================================

# ------------------------------------------------------------
# Règles générales pour tous les crawlers
# ------------------------------------------------------------
User-agent: *
Crawl-delay: 5
Allow: /

# Pages exclues de l'indexation (espace client, technique, légal non prioritaire)
Disallow: /mentions-legales/
Disallow: /conditions-generales-de-ventes/
Disallow: /mon-compte/
Disallow: /mon-compte/se-deconnecter/
Disallow: /s-inscrire/
Disallow: /mon-panier/
Disallow: /logistique/
Disallow: /p3_web/
Disallow: /search/
Disallow: /newsletter-pop-in/
Disallow: /acces-pro/boutique-pro/
Disallow: /acces-pro/banque-images-equipement/
Disallow: /acces-pro/site-formation/

# ------------------------------------------------------------
# Crawlers LLM / IA — autorisés explicitement
# (accès au contenu et aux fichiers llms.txt)
# ------------------------------------------------------------
User-agent: GPTBot
Allow: /
Allow: /llms-fr.txt
Disallow: /mon-compte/
Disallow: /mon-panier/
Disallow: /search/

User-agent: ClaudeBot
Allow: /
Allow: /llms-fr.txt
Disallow: /mon-compte/
Disallow: /mon-panier/
Disallow: /search/

User-agent: Claude-Web
Allow: /
Allow: /llms-fr.txt
Disallow: /mon-compte/
Disallow: /mon-panier/
Disallow: /search/

User-agent: anthropic-ai
Allow: /
Allow: /llms-fr.txt
Disallow: /mon-compte/
Disallow: /mon-panier/
Disallow: /search/

User-agent: PerplexityBot
Allow: /
Allow: /llms-fr.txt
Disallow: /mon-compte/
Disallow: /mon-panier/
Disallow: /search/

User-agent: Applebot
Allow: /
Allow: /llms-fr.txt
Disallow: /mon-compte/
Disallow: /mon-panier/
Disallow: /search/

User-agent: cohere-ai
Allow: /
Allow: /llms-fr.txt
Disallow: /mon-compte/
Disallow: /mon-panier/
Disallow: /search/

User-agent: Meta-ExternalAgent
Allow: /
Allow: /llms-fr.txt
Disallow: /mon-compte/
Disallow: /mon-panier/
Disallow: /search/

User-agent: Bytespider
Allow: /
Allow: /llms-fr.txt
Disallow: /mon-compte/
Disallow: /mon-panier/
Disallow: /search/

# ------------------------------------------------------------
# Bots indésirables — bloqués
# ------------------------------------------------------------
User-agent: TurnitinBot
Disallow: /

User-agent: ConveraCrawler
Disallow: /

User-agent: serstatsbot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# ------------------------------------------------------------
# Fichiers GEO / LLM
# ------------------------------------------------------------
# llms-fr.txt      → https://www.thalgo.fr/llms-fr.txt
# llms.txt (UK)    → https://www.thalgo.co.uk/llms-co_uk.txt
# llms.txt (BE)    → https://www.thalgo-belgium.com/llms-be.txt
# llms.txt (CH)    → https://www.thalgo-suisse.ch/llms-ch.txt
# llms.txt (RE)    → https://www.thalgo.re/llms-re.txt
# llms.txt (IN)    → https://www.thalgo.in/llms-in.txt
# llms.txt (MT)    → https://www.thalgo.com.mt/llms-mt.txt
# llms.txt (MA)    → https://www.thalgo.ma/llms-ma.txt
# llms.txt (MY)    → https://www.thalgo.my/llms-my.txt
# llms.txt (NZ)    → https://www.thalgo.co.nz/llms-co_nz.txt
# llms.txt (ES)    → https://www.thalgo.es/llms-es.txt
# llms.txt (ZA)    → https://www.thalgo.co.za/llms-co_za.txt
# llms.txt (USA)   → https://www.thalgo-usa.com/llms-us.txt
# llms.txt (CA)   → https://www.thalgo-canada.com/llms-canada.txt

# ------------------------------------------------------------
# Sitemaps XML
# ------------------------------------------------------------
Sitemap: https://www.thalgo.fr/sitemap-fr.xml
Sitemap: https://www.thalgo.co.uk/sitemap-co_uk.xml
Sitemap: https://www.thalgo-belgium.com/sitemap-be.xml
Sitemap: https://www.thalgo-suisse.ch/sitemap-ch.xml
Sitemap: https://www.thalgo.re/sitemap-re.xml
Sitemap: https://www.thalgo.in/sitemap-in.xml
Sitemap: https://www.thalgo.com.mt/sitemap-mt.xml
Sitemap: https://www.thalgo.ma/sitemap-ma.xml
Sitemap: https://www.thalgo.my/sitemap-my.xml
Sitemap: https://www.thalgo.co.nz/sitemap-co_nz.xml
Sitemap: https://www.thalgo.es/sitemap-es.xml
Sitemap: https://www.thalgo.co.za/sitemap-co_za.xml
Sitemap: https://www.thalgo-usa.com/sitemap-us.xml
Sitemap: https://www.thalgo.de/sitemap-de.xml
Sitemap: https://www.thalgo.gr/sitemap-gr.xml
Sitemap: https://www.thalgocosmetics.nl/sitemap-nl.xml
Sitemap: https://www.thalgo-belgie.com/sitemap-lu.xml
Sitemap: https://www.thalgo-canada.com/sitemap-canada.xml