{
  "site": "https://better-robots.com/",
  "updated": "2026-07-07",
  "describes": "/bot-taxonomy.json",
  "human_readable": "/governance/bot-taxonomy",
  "purpose": "Maps documented crawler user agents to one canonical role category defined in /bot-taxonomy.json. Each classification follows the provider's own public documentation. Where a provider does not document a classification, the entry is marked not_publicly_specified rather than asserted.",
  "source_precedence": "The provider's official documentation is authoritative. This registry follows that documentation; it does not override it. A user agent may serve more than one role over time, so treat category as the primary documented role, not an exclusive one.",
  "category_ids": [
    "search_crawlers",
    "training_crawlers_or_tokens",
    "answer_or_retrieval_systems",
    "user_triggered_fetchers",
    "signed_or_verified_agents",
    "archive_bots",
    "seo_tool_bots",
    "low_value_or_abusive_bots"
  ],
  "plugin_control_mapping": {
    "ai_training": {
      "label": "AI training",
      "categories": ["training_crawlers_or_tokens"],
      "control_surface": "robots.txt"
    },
    "ai_search_discovery": {
      "label": "AI search / discovery",
      "categories": ["search_crawlers", "answer_or_retrieval_systems"],
      "control_surface": "robots.txt"
    },
    "user_action": {
      "label": "User-triggered access",
      "categories": ["user_triggered_fetchers"],
      "control_surface": "robots.txt with vendor and infrastructure caveats"
    },
    "infrastructure_layer": {
      "label": "Governed outside the plugin's robots.txt controls",
      "categories": ["signed_or_verified_agents", "archive_bots", "seo_tool_bots", "low_value_or_abusive_bots"],
      "control_surface": "CDN allowlisting, WAF rules, rate limiting, request verification"
    }
  },
  "status_values": {
    "documented": "The classification is stated in the provider's own current documentation.",
    "legacy_undocumented": "A historical user agent no longer listed in the provider's current documentation; role is inferred from prior use.",
    "not_publicly_specified": "The provider publishes no authoritative documentation for this user agent; role reflects third-party observation only."
  },
  "agents": [
    {
      "user_agent": "GPTBot",
      "vendor": "OpenAI",
      "category": "training_crawlers_or_tokens",
      "plugin_control": "ai_training",
      "role": "training",
      "trigger_mode": "automatic",
      "status": "documented",
      "source": "OpenAI - Overview of OpenAI crawlers",
      "source_url": "https://platform.openai.com/docs/bots"
    },
    {
      "user_agent": "OAI-SearchBot",
      "vendor": "OpenAI",
      "category": "search_crawlers",
      "plugin_control": "ai_search_discovery",
      "role": "search",
      "trigger_mode": "automatic",
      "status": "documented",
      "source": "OpenAI - Overview of OpenAI crawlers",
      "source_url": "https://platform.openai.com/docs/bots"
    },
    {
      "user_agent": "ChatGPT-User",
      "vendor": "OpenAI",
      "category": "user_triggered_fetchers",
      "plugin_control": "user_action",
      "role": "user_triggered",
      "trigger_mode": "user_triggered",
      "status": "documented",
      "source": "OpenAI - Overview of OpenAI crawlers",
      "source_url": "https://platform.openai.com/docs/bots"
    },
    {
      "user_agent": "OAI-AdsBot",
      "vendor": "OpenAI",
      "category": "user_triggered_fetchers",
      "plugin_control": "user_action",
      "role": "ads_landing_page_fetch",
      "trigger_mode": "mixed",
      "status": "documented",
      "source": "OpenAI - Overview of OpenAI crawlers",
      "source_url": "https://platform.openai.com/docs/bots",
      "note": "Documented for advertising-related landing-page fetching. Ads is a distinct documented use that does not map cleanly onto the eight role categories; it is placed with user-triggered fetchers as the closest fit."
    },
    {
      "user_agent": "ClaudeBot",
      "vendor": "Anthropic",
      "category": "training_crawlers_or_tokens",
      "plugin_control": "ai_training",
      "role": "training",
      "trigger_mode": "automatic",
      "status": "documented",
      "source": "Anthropic - Does Anthropic crawl data from the web, and how can site owners block the crawler?",
      "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
    },
    {
      "user_agent": "Claude-SearchBot",
      "vendor": "Anthropic",
      "category": "search_crawlers",
      "plugin_control": "ai_search_discovery",
      "role": "search",
      "trigger_mode": "automatic",
      "status": "documented",
      "source": "Anthropic - Does Anthropic crawl data from the web, and how can site owners block the crawler?",
      "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
    },
    {
      "user_agent": "Claude-User",
      "vendor": "Anthropic",
      "category": "user_triggered_fetchers",
      "plugin_control": "user_action",
      "role": "user_triggered",
      "trigger_mode": "user_triggered",
      "status": "documented",
      "source": "Anthropic - Does Anthropic crawl data from the web, and how can site owners block the crawler?",
      "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
    },
    {
      "user_agent": "anthropic-ai",
      "vendor": "Anthropic",
      "category": "training_crawlers_or_tokens",
      "plugin_control": "ai_training",
      "role": "training",
      "trigger_mode": "automatic",
      "status": "legacy_undocumented",
      "source": "Not listed in Anthropic's current crawler documentation; historical training user agent",
      "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
    },
    {
      "user_agent": "Claude-Web",
      "vendor": "Anthropic",
      "category": "answer_or_retrieval_systems",
      "plugin_control": "ai_search_discovery",
      "role": "answer_retrieval",
      "trigger_mode": "mixed",
      "status": "legacy_undocumented",
      "source": "Not listed in Anthropic's current crawler documentation; historical browsing user agent superseded by Claude-User and Claude-SearchBot",
      "source_url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
    },
    {
      "user_agent": "PerplexityBot",
      "vendor": "Perplexity",
      "category": "search_crawlers",
      "plugin_control": "ai_search_discovery",
      "role": "search",
      "trigger_mode": "automatic",
      "status": "documented",
      "also_relevant": ["answer_or_retrieval_systems"],
      "source": "Perplexity - Perplexity Crawlers",
      "source_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers"
    },
    {
      "user_agent": "Perplexity-User",
      "vendor": "Perplexity",
      "category": "user_triggered_fetchers",
      "plugin_control": "user_action",
      "role": "user_triggered",
      "trigger_mode": "user_triggered",
      "status": "documented",
      "source": "Perplexity - Perplexity Crawlers",
      "source_url": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers"
    },
    {
      "user_agent": "Googlebot",
      "vendor": "Google",
      "category": "search_crawlers",
      "plugin_control": "ai_search_discovery",
      "role": "search",
      "trigger_mode": "automatic",
      "status": "documented",
      "source": "Google Search Central - Google's common crawlers",
      "source_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers"
    },
    {
      "user_agent": "Google-Extended",
      "vendor": "Google",
      "category": "training_crawlers_or_tokens",
      "plugin_control": "ai_training",
      "role": "training_token",
      "trigger_mode": "automatic",
      "status": "documented",
      "source": "Google Search Central - Google's common crawlers",
      "source_url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "note": "A robots.txt control token, not a separate fetcher. It has no distinct HTTP user-agent string and does not affect Google Search ranking or indexing."
    },
    {
      "user_agent": "Applebot",
      "vendor": "Apple",
      "category": "search_crawlers",
      "plugin_control": "ai_search_discovery",
      "role": "search",
      "trigger_mode": "automatic",
      "status": "documented",
      "source": "Apple - About Applebot",
      "source_url": "https://support.apple.com/en-us/119829"
    },
    {
      "user_agent": "Applebot-Extended",
      "vendor": "Apple",
      "category": "training_crawlers_or_tokens",
      "plugin_control": "ai_training",
      "role": "training_token",
      "trigger_mode": "automatic",
      "status": "documented",
      "source": "Apple - About Applebot",
      "source_url": "https://support.apple.com/en-us/119829",
      "note": "A control token that does not crawl. It determines whether Applebot-crawled data may train Apple's generative models. Disallowing it does not remove pages from search."
    },
    {
      "user_agent": "CCBot",
      "vendor": "Common Crawl",
      "category": "training_crawlers_or_tokens",
      "plugin_control": "ai_training",
      "role": "training",
      "trigger_mode": "automatic",
      "status": "documented",
      "source": "Common Crawl - CCBot",
      "source_url": "https://commoncrawl.org/ccbot"
    },
    {
      "user_agent": "AI2Bot",
      "vendor": "Allen Institute for AI (Ai2)",
      "category": "training_crawlers_or_tokens",
      "plugin_control": "ai_training",
      "role": "training",
      "trigger_mode": "automatic",
      "status": "documented",
      "source": "Ai2 - Crawler",
      "source_url": "https://allenai.org/crawler",
      "note": "Builds open datasets for training open language models. A variant token Ai2Bot-Dolma may also appear and can be addressed with its own robots.txt rule."
    },
    {
      "user_agent": "Bytespider",
      "vendor": "ByteDance",
      "category": "low_value_or_abusive_bots",
      "plugin_control": "infrastructure_layer",
      "role": "training",
      "trigger_mode": "mixed",
      "status": "not_publicly_specified",
      "source": "No authoritative ByteDance crawler documentation; the user agent embeds a contact address only",
      "source_url": "mailto:spider-feedback@bytedance.com",
      "note": "The functional role appears to be training collection, but the operational bucket is low-value or abusive because the crawler is widely observed to ignore robots.txt and to crawl aggressively. Govern it at the infrastructure layer rather than by robots.txt alone."
    }
  ]
}