# =====================================================================
# robots.txt for Vet Price Comparison Site
#
# Policy:
#   * Major search engines & social previews: full access
#   * AI assistants / LLM crawlers: ALLOWED so we appear in citations
#     (ChatGPT, Claude, Perplexity, Gemini/Google AI, Apple Intelligence, etc.)
#   * Generic scraping/data-harvesting bots: DISALLOWED
#
# Note: robots.txt is advisory — it works against well-behaved bots only.
# Hard enforcement (Firecrawl, ScrapingBee, headless Chromium farms, etc.)
# happens at the Cloudflare edge via Bot Fight Mode, AI Labyrinth, custom
# WAF rules and rate limiting. See /docs/CLOUDFLARE_SECURITY.md.
# =====================================================================

# --- Search engines ---
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: Applebot
Allow: /

# --- Social preview crawlers ---
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

# --- AI assistants & LLM crawlers (ALLOWED for citations / answers) ---
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bingbot
Allow: /

User-agent: CCBot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: MistralAI-User
Allow: /

User-agent: YouBot
Allow: /

# --- Generic scraping / data-harvesting bots (DISALLOWED) ---
# These are commercial scraping APIs and headless-browser farms. They are
# also blocked at the edge in Cloudflare; this is the polite request.
User-agent: FirecrawlAgent
Disallow: /

User-agent: Firecrawl
Disallow: /

User-agent: ScrapingBee
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: ApifyBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Bytespider-Scraper
Disallow: /

User-agent: SerpstatBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SeekportBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: wget
Disallow: /

User-agent: curl
Disallow: /

# --- Default: allow everything else, but keep private/internal paths out of indexes ---
User-agent: *
Disallow: /admin
Disallow: /admin/
Disallow: /practice
Disallow: /practice/
Disallow: /unsubscribe
Disallow: /confirm-subscription
Disallow: /concepts/
Disallow: /test
Disallow: /test-2
Disallow: /listing-test2
Disallow: /listing-test3
Disallow: /listing-test4
Allow: /

Sitemap: https://vetpricecomparisonsite.co.uk/sitemap.xml

# AI assistants: a Markdown site map is available at /llms.txt
# Markdown versions of content pages: /md/vet/<slug>, /md/vets-in/<slug>