# geo-parity.com — open to search engines and AI agents alike.
# Reference: https://www.rfc-editor.org/rfc/rfc9309
#
# Everything public is free to crawl, index, and use for answers, grounding and
# training. The site ships machine-readable resources for agents — llms.txt, an
# MCP manifest, an OpenAPI description, and open JSON APIs that need no key.
#
# ── Two things worth knowing before editing this file ──────────────────────────
#
# 1. Every agent below is in ONE group with `*`. Under RFC 9309 a crawler obeys
#    only the most specific group that matches it, so a separate
#    `User-agent: GPTBot / Allow: /` block *overrides* the `*` group entirely.
#    That is harmless while `*` has no Disallow lines — but the day anyone adds
#    one, every named crawler would keep crawling it and this file would still
#    look correct. Consecutive User-agent lines share one rule set, so the list
#    cannot drift out of step with the rules.
#
# 2. Naming a crawler does not grant it more access than `*` already gives. This
#    list is a statement of intent, not a permission grant. It matters for the
#    "-Extended" tokens (Google-Extended, Applebot-Extended), which are the
#    AI-training opt-outs — being listed as allowed says so explicitly rather
#    than relying on a default that may change. For everything else it is
#    documentation. New crawlers not listed here are still allowed by `*`.

User-agent: *

# OpenAI — training, search index, and user-initiated fetches
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User

# Anthropic
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: Claude-Web
User-agent: anthropic-ai

# Google (Google-Extended is the Gemini/Vertex training token)
User-agent: Google-Extended
User-agent: GoogleOther

# Microsoft / Copilot
User-agent: bingbot

# Apple (Applebot-Extended is the AI-training token)
User-agent: Applebot
User-agent: Applebot-Extended

# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User

# Meta
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: FacebookBot

# Amazon
User-agent: Amazonbot

# Mistral
User-agent: MistralAI-User

# Cohere
User-agent: cohere-ai
User-agent: cohere-training-data-crawler

# Allen Institute for AI
User-agent: AI2Bot

# DuckDuckGo assist
User-agent: DuckAssistBot

# You.com
User-agent: YouBot

# ByteDance
User-agent: Bytespider

# Huawei Petal
User-agent: PetalBot

# Yandex
User-agent: YandexBot

# Common Crawl — the corpus many smaller models are trained on
User-agent: CCBot

# Other crawlers that feed AI products or datasets
User-agent: Diffbot
User-agent: Timpibot
User-agent: Omgilibot
User-agent: Webzio-Extended
User-agent: ImagesiftBot
Allow: /
# Content Signals (contentsignals.org): the machine-readable form of the stance
# above. Everything public may be indexed, used as AI input, and used for
# training. All three are "yes" on purpose: this site exists to be cited.
# Applies to the whole group, i.e. every crawler named above and everyone else.
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# Machine-readable indexes for search engines and AI/LLM crawlers.
Sitemap: https://geo-parity.com/sitemap-index.xml
# Agentic Resource Discovery manifest (agenticresourcediscovery.org): where an
# agent finds the MCP server card, the OpenAPI spec, llms.txt and the skill.
Agentmap: https://geo-parity.com/.well-known/ard.json

# AI discovery: capabilities and structured endpoints are described here.
# llms.txt:     https://geo-parity.com/llms.txt
# MCP manifest: https://geo-parity.com/.well-known/mcp.json
# OpenAPI:      https://geo-parity.com/openapi.json