{
  "name": "AI Crawler User Agents",
  "description": "Every known AI crawler and agent user-agent string, with operator, purpose, and robots.txt compliance.",
  "source": "https://geoprompttracker.com/bots",
  "documentation": "https://geoprompttracker.com/guides/list-of-ai-crawlers",
  "license": "CC-BY-4.0",
  "attribution": "GeoPromptTracker (https://geoprompttracker.com)",
  "lastVerified": "2026-07-18",
  "count": 28,
  "crawlers": [
    {
      "userAgent": "GPTBot",
      "slug": "gptbot",
      "operator": "OpenAI",
      "purpose": "training",
      "description": "Crawls content to train OpenAI's models.",
      "robotsCompliance": "yes",
      "robotsNote": "OpenAI documents GPTBot's IP ranges and states it honors robots.txt disallow rules.",
      "docsUrl": "https://platform.openai.com/docs/bots",
      "url": "https://geoprompttracker.com/bots/gptbot"
    },
    {
      "userAgent": "OAI-SearchBot",
      "slug": "oai-searchbot",
      "operator": "OpenAI",
      "purpose": "search",
      "description": "Powers search-style results surfaced in ChatGPT.",
      "robotsCompliance": "yes",
      "robotsNote": "OpenAI states OAI-SearchBot respects robots.txt rules.",
      "docsUrl": "https://platform.openai.com/docs/bots",
      "url": "https://geoprompttracker.com/bots/oai-searchbot"
    },
    {
      "userAgent": "ChatGPT-User",
      "slug": "chatgpt-user",
      "operator": "OpenAI",
      "purpose": "search",
      "description": "Fetches pages a user asks ChatGPT to browse in real time.",
      "robotsCompliance": "yes",
      "robotsNote": "OpenAI states ChatGPT-User honors robots.txt, though as a user-triggered fetcher it visits single pages rather than crawling.",
      "docsUrl": "https://platform.openai.com/docs/bots",
      "url": "https://geoprompttracker.com/bots/chatgpt-user"
    },
    {
      "userAgent": "ClaudeBot",
      "slug": "claudebot",
      "operator": "Anthropic",
      "purpose": "training",
      "description": "Crawls content to train Anthropic's Claude models.",
      "robotsCompliance": "yes",
      "robotsNote": "Anthropic documents ClaudeBot and states it respects robots.txt directives and anti-circumvention signals.",
      "docsUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "url": "https://geoprompttracker.com/bots/claudebot"
    },
    {
      "userAgent": "Claude-User",
      "slug": "claude-user",
      "operator": "Anthropic",
      "purpose": "search",
      "description": "Fetches pages on behalf of a user during a Claude conversation.",
      "robotsCompliance": "yes",
      "robotsNote": "Anthropic documents Claude-User alongside its other agents; it fetches single user-requested pages rather than crawling at scale.",
      "docsUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "url": "https://geoprompttracker.com/bots/claude-user"
    },
    {
      "userAgent": "Claude-SearchBot",
      "slug": "claude-searchbot",
      "operator": "Anthropic",
      "purpose": "search",
      "description": "Indexes content to inform Claude's search-style answers.",
      "robotsCompliance": "yes",
      "robotsNote": "Anthropic states Claude-SearchBot respects robots.txt directives.",
      "docsUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "url": "https://geoprompttracker.com/bots/claude-searchbot"
    },
    {
      "userAgent": "PerplexityBot",
      "slug": "perplexitybot",
      "operator": "Perplexity",
      "purpose": "search",
      "description": "Crawls and indexes pages for Perplexity's answer engine.",
      "robotsCompliance": "partial",
      "robotsNote": "Perplexity says PerplexityBot respects robots.txt, but independent investigations (notably Cloudflare's 2024–2025 reports) have observed undeclared fetching that bypassed blocks.",
      "docsUrl": "https://docs.perplexity.ai/guides/bots",
      "url": "https://geoprompttracker.com/bots/perplexitybot"
    },
    {
      "userAgent": "Perplexity-User",
      "slug": "perplexity-user",
      "operator": "Perplexity",
      "purpose": "search",
      "description": "Fetches a page a user asks Perplexity to browse live.",
      "robotsCompliance": "no",
      "robotsNote": "Perplexity's own documentation says Perplexity-User generally ignores robots.txt because fetches are user-requested.",
      "docsUrl": "https://docs.perplexity.ai/guides/bots",
      "url": "https://geoprompttracker.com/bots/perplexity-user"
    },
    {
      "userAgent": "Google-Extended",
      "slug": "google-extended",
      "operator": "Google",
      "purpose": "training",
      "description": "Controls use of content for Gemini and Vertex AI training, separate from Googlebot.",
      "robotsCompliance": "yes",
      "robotsNote": "As a robots.txt token rather than a crawler, honoring your directive is its entire function; Google documents it officially.",
      "docsUrl": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
      "url": "https://geoprompttracker.com/bots/google-extended"
    },
    {
      "userAgent": "CCBot",
      "slug": "ccbot",
      "operator": "Common Crawl",
      "purpose": "training",
      "description": "Common Crawl's crawler; its dataset is widely used to train LLMs.",
      "robotsCompliance": "yes",
      "robotsNote": "Common Crawl is a nonprofit with a long public record of honoring robots.txt and crawl-delay directives.",
      "docsUrl": "https://commoncrawl.org/ccbot",
      "url": "https://geoprompttracker.com/bots/ccbot"
    },
    {
      "userAgent": "Bytespider",
      "slug": "bytespider",
      "operator": "ByteDance",
      "purpose": "training",
      "description": "ByteDance's crawler, used for training its AI models.",
      "robotsCompliance": "no",
      "robotsNote": "ByteDance publishes no crawler documentation, and multiple independent reports have found Bytespider ignoring robots.txt; blocking it reliably requires firewall or CDN rules.",
      "docsUrl": null,
      "url": "https://geoprompttracker.com/bots/bytespider"
    },
    {
      "userAgent": "Applebot-Extended",
      "slug": "applebot-extended",
      "operator": "Apple",
      "purpose": "training",
      "description": "Controls use of content for training Apple's AI models (Apple Intelligence).",
      "robotsCompliance": "yes",
      "robotsNote": "Apple documents the token officially and honors it via Applebot's crawl infrastructure.",
      "docsUrl": "https://support.apple.com/en-us/119829",
      "url": "https://geoprompttracker.com/bots/applebot-extended"
    },
    {
      "userAgent": "Amazonbot",
      "slug": "amazonbot",
      "operator": "Amazon",
      "purpose": "both",
      "description": "Crawls content for Amazon's search and AI products, including Alexa.",
      "robotsCompliance": "yes",
      "robotsNote": "Amazon documents Amazonbot and states it respects robots.txt, including wildcard rules.",
      "docsUrl": "https://developer.amazon.com/amazonbot",
      "url": "https://geoprompttracker.com/bots/amazonbot"
    },
    {
      "userAgent": "Meta-ExternalAgent",
      "slug": "meta-externalagent",
      "operator": "Meta",
      "purpose": "training",
      "description": "Crawls content to train Meta's AI models.",
      "robotsCompliance": "yes",
      "robotsNote": "Meta documents the agent and states it respects robots.txt rules.",
      "docsUrl": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "url": "https://geoprompttracker.com/bots/meta-externalagent"
    },
    {
      "userAgent": "cohere-ai",
      "slug": "cohere-ai",
      "operator": "Cohere",
      "purpose": "training",
      "description": "Cohere's crawler used for training and grounding its models.",
      "robotsCompliance": "unknown",
      "robotsNote": "Cohere publishes no official crawler documentation, so robots.txt compliance can't be verified; use firewall rules if blocking it matters to you.",
      "docsUrl": null,
      "url": "https://geoprompttracker.com/bots/cohere-ai"
    },
    {
      "userAgent": "Applebot",
      "slug": "applebot",
      "operator": "Apple",
      "purpose": "search",
      "description": "Apple's crawler powering Siri and Spotlight suggestions.",
      "robotsCompliance": "yes",
      "robotsNote": "Apple documents Applebot thoroughly and honors robots.txt directives.",
      "docsUrl": "https://support.apple.com/en-us/119829",
      "url": "https://geoprompttracker.com/bots/applebot"
    },
    {
      "userAgent": "GoogleOther",
      "slug": "googleother",
      "operator": "Google",
      "purpose": "both",
      "description": "Google's generic crawler for research and internal product uses.",
      "robotsCompliance": "yes",
      "robotsNote": "Officially documented by Google and respects robots.txt like the rest of its crawler family.",
      "docsUrl": "https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers",
      "url": "https://geoprompttracker.com/bots/googleother"
    },
    {
      "userAgent": "MistralAI-User",
      "slug": "mistralai-user",
      "operator": "Mistral",
      "purpose": "search",
      "description": "Fetches pages when a Le Chat user's question requires them.",
      "robotsCompliance": "yes",
      "robotsNote": "Mistral documents the agent and states it respects robots.txt directives.",
      "docsUrl": "https://docs.mistral.ai/robots/",
      "url": "https://geoprompttracker.com/bots/mistralai-user"
    },
    {
      "userAgent": "Meta-ExternalFetcher",
      "slug": "meta-externalfetcher",
      "operator": "Meta",
      "purpose": "search",
      "description": "Fetches specific links for Meta AI when users share or request them.",
      "robotsCompliance": "no",
      "robotsNote": "Meta's own documentation states Meta-ExternalFetcher may bypass robots.txt because fetches are performed on behalf of a user.",
      "docsUrl": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers/",
      "url": "https://geoprompttracker.com/bots/meta-externalfetcher"
    },
    {
      "userAgent": "AI2Bot",
      "slug": "ai2bot",
      "operator": "Allen Institute for AI",
      "purpose": "training",
      "description": "Crawls the web for the Allen Institute's open AI research models.",
      "robotsCompliance": "yes",
      "robotsNote": "Ai2 documents the crawler and honors robots.txt; as a research nonprofit it has strong incentives to stay compliant.",
      "docsUrl": "https://allenai.org/crawler",
      "url": "https://geoprompttracker.com/bots/ai2bot"
    },
    {
      "userAgent": "DuckAssistBot",
      "slug": "duckassistbot",
      "operator": "DuckDuckGo",
      "purpose": "search",
      "description": "Fetches pages to generate DuckDuckGo's AI-assisted answers.",
      "robotsCompliance": "yes",
      "robotsNote": "DuckDuckGo documents the agent and states it respects robots.txt.",
      "docsUrl": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/",
      "url": "https://geoprompttracker.com/bots/duckassistbot"
    },
    {
      "userAgent": "YouBot",
      "slug": "youbot",
      "operator": "You.com",
      "purpose": "search",
      "description": "Crawls and fetches pages for You.com's AI search answers.",
      "robotsCompliance": "yes",
      "robotsNote": "You.com documents YouBot and states it honors robots.txt rules.",
      "docsUrl": "https://about.you.com/youbot/",
      "url": "https://geoprompttracker.com/bots/youbot"
    },
    {
      "userAgent": "PetalBot",
      "slug": "petalbot",
      "operator": "Huawei",
      "purpose": "both",
      "description": "Huawei's Petal Search crawler, also feeding its AI products.",
      "robotsCompliance": "yes",
      "robotsNote": "Documented by Aspiegel/Huawei with published IP ranges; states it respects robots.txt and crawl-delay.",
      "docsUrl": "https://aspiegel.com/petalbot",
      "url": "https://geoprompttracker.com/bots/petalbot"
    },
    {
      "userAgent": "Diffbot",
      "slug": "diffbot",
      "operator": "Diffbot",
      "purpose": "both",
      "description": "Extracts structured data from pages for customers, including AI companies.",
      "robotsCompliance": "partial",
      "robotsNote": "Diffbot's bulk crawling honors robots.txt, but customer-triggered, on-demand extraction of specific URLs has been reported to proceed regardless — treat blocks as advisory.",
      "docsUrl": "https://docs.diffbot.com/docs/how-to-identify-diffbot",
      "url": "https://geoprompttracker.com/bots/diffbot"
    },
    {
      "userAgent": "ImagesiftBot",
      "slug": "imagesiftbot",
      "operator": "The Hive",
      "purpose": "training",
      "description": "Crawls images across the web for Hive's visual-AI products.",
      "robotsCompliance": "yes",
      "robotsNote": "The bot is documented and states it respects robots.txt; blocking it mainly affects image collection.",
      "docsUrl": "https://imagesift.com/about",
      "url": "https://geoprompttracker.com/bots/imagesiftbot"
    },
    {
      "userAgent": "omgilibot",
      "slug": "omgilibot",
      "operator": "Webz.io",
      "purpose": "training",
      "description": "Webz.io's crawler; its web-data feeds are sold to AI builders.",
      "robotsCompliance": "yes",
      "robotsNote": "Webz.io documents the bot and states it honors robots.txt; as a data vendor its compliance is part of its sales pitch.",
      "docsUrl": "https://webz.io/bot.html",
      "url": "https://geoprompttracker.com/bots/omgilibot"
    },
    {
      "userAgent": "Timpibot",
      "slug": "timpibot",
      "operator": "Timpi",
      "purpose": "search",
      "description": "Crawler for Timpi's decentralized search index.",
      "robotsCompliance": "unknown",
      "robotsNote": "Timpi publishes limited crawler documentation; compliance reports are sparse. If blocking matters, verify behavior in your logs and back the block with a firewall rule.",
      "docsUrl": null,
      "url": "https://geoprompttracker.com/bots/timpibot"
    },
    {
      "userAgent": "PanguBot",
      "slug": "pangubot",
      "operator": "Huawei",
      "purpose": "training",
      "description": "Associated with training Huawei's PanGu large language models.",
      "robotsCompliance": "unknown",
      "robotsNote": "No official documentation exists and independent compliance reports are limited — pair any robots.txt block with a firewall or CDN rule if opting out matters to you.",
      "docsUrl": null,
      "url": "https://geoprompttracker.com/bots/pangubot"
    }
  ]
}