# warre & vavasour — crawl policy
# Humans and agents: see https://warreandvavasour.com/llms.txt for routing guidance.
# Machine-readable page index: https://warreandvavasour.com/content.json
#
# Policy: the whole site is public and every crawler and agent is welcome,
# including AI search, answer and training crawlers. The site is written to be
# read by agents. Each bot is listed by name below so the intent is explicit.
# There are no Disallow rules: the previous file had none, and every page is public.
# Bot names were checked against vendor documentation where it exists on
# 2026-10-27; sources are in docs/agent-readability.md.

User-agent: *
Allow: /

# --- Search engines ---------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# --- AI search, answer and user-request bots --------------------------------
# OpenAI: OAI-SearchBot (ChatGPT search); ChatGPT-User (a person asks ChatGPT about a page).
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Perplexity: PerplexityBot (search); Perplexity-User (a person asks about a page).
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Anthropic: Claude-SearchBot (search quality); Claude-User (a person asks about a page).
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Apple: Applebot powers Spotlight, Siri and Safari search.
User-agent: Applebot
Allow: /

# Meta: Meta-WebIndexer improves Meta AI search results and citations.
User-agent: Meta-WebIndexer
Allow: /

# --- AI training and model crawlers: also welcome ---------------------------
# GPTBot: OpenAI. ClaudeBot: Anthropic. anthropic-ai: older Anthropic token
#   (not in Anthropic's current list; kept as a harmless extra).
# CCBot: Common Crawl's open archive. Google-Extended: Gemini training and
#   grounding (no effect on Google Search). Applebot-Extended: Apple generative AI.
# Bytespider: ByteDance (no ByteDance documentation found).
# Meta-ExternalAgent: Meta AI crawler. cohere-ai: Cohere (token from community
#   lists; Cohere documents no active crawler today).
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: CCBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: cohere-ai
Allow: /

Sitemap: https://warreandvavasour.com/sitemap.xml