{"name":"NumberHill AI crawler directory","source":"https://numberhill.com/tools/ai-crawlers","documentation":"https://numberhill.com/tools/ai-crawlers#method","updated":"2026-09-09","license":"CC BY 4.0 — reuse freely with a link to the source","count":49,"purposes":{"training":{"label":"Training crawlers","description":"Build the corpus a model is trained on. Blocking one keeps future models from learning from your pages; it has no effect on whether an AI product can cite you today."},"search":{"label":"AI search indexers","description":"Build the index an AI answer engine retrieves from. Blocking one removes your site from that product's citations — the traffic source most sites want to keep."},"user":{"label":"User-triggered fetchers","description":"Fetch a page because a person asked the assistant about it or pasted the link. Several operators state these fetchers act on the user's behalf and do not check robots.txt."},"agent":{"label":"Agentic browsers","description":"Browse and act on one person's instruction — comparing prices, filling forms, completing a task. Closer to a visit than a crawl, but they do not run analytics scripts."},"control":{"label":"Robots.txt control tokens","description":"Not a crawler you will see in logs. A token the operator reads in robots.txt to decide whether pages its ordinary crawler already fetched may be used for AI features."}},"robotsValues":{"yes":"Operator documents honouring robots.txt","no":"Operator documents that it may ignore robots.txt","unclear":"Operator has published nothing either way"},"crawlers":[{"slug":"gptbot","name":"GPTBot","operator":"OpenAI","purpose":"training","feeds":"ChatGPT","token":"GPTBot","aliases":[],"sampleUserAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.2; +https://openai.com/gptbot","robots":"yes","verification":"OpenAI publishes its IP ranges as JSON; a hit from outside them is spoofed.","docsUrl":"https://platform.openai.com/docs/bots","url":"https://numberhill.com/tools/ai-crawlers/gptbot"},{"slug":"oai-searchbot","name":"OAI-SearchBot","operator":"OpenAI","purpose":"search","feeds":"ChatGPT Search","token":"OAI-SearchBot","aliases":[],"sampleUserAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot","robots":"yes","verification":"OpenAI publishes its IP ranges as JSON; a hit from outside them is spoofed.","docsUrl":"https://platform.openai.com/docs/bots","url":"https://numberhill.com/tools/ai-crawlers/oai-searchbot"},{"slug":"chatgpt-user","name":"ChatGPT-User","operator":"OpenAI","purpose":"user","feeds":"ChatGPT (live browse)","token":"ChatGPT-User","aliases":[],"sampleUserAgent":"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot","robots":"yes","verification":"OpenAI publishes its IP ranges as JSON; a hit from outside them is spoofed.","docsUrl":"https://platform.openai.com/docs/bots","url":"https://numberhill.com/tools/ai-crawlers/chatgpt-user"},{"slug":"chatgpt-agent","name":"ChatGPT-Agent","operator":"OpenAI","purpose":"agent","feeds":"ChatGPT (agent)","token":"ChatGPT-Agent","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; ChatGPT-Agent/1.0; +https://openai.com/bot)","robots":"yes","verification":"OpenAI publishes its IP ranges as JSON; a hit from outside them is spoofed.","docsUrl":"https://platform.openai.com/docs/bots","url":"https://numberhill.com/tools/ai-crawlers/chatgpt-agent"},{"slug":"claudebot","name":"ClaudeBot","operator":"Anthropic","purpose":"training","feeds":"Claude","token":"ClaudeBot","aliases":["anthropic-ai","Claude-Web"],"sampleUserAgent":"Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)","robots":"yes","verification":"Anthropic publishes no machine-readable IP list, so a ClaudeBot hit cannot be verified by address. Treat the user agent as a claim, not proof.","docsUrl":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","url":"https://numberhill.com/tools/ai-crawlers/claudebot"},{"slug":"claude-searchbot","name":"Claude-SearchBot","operator":"Anthropic","purpose":"search","feeds":"Claude Search","token":"Claude-SearchBot","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; Claude-SearchBot/1.0; +https://www.anthropic.com)","robots":"yes","verification":"No published IP ranges; verify by behaviour only.","docsUrl":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","url":"https://numberhill.com/tools/ai-crawlers/claude-searchbot"},{"slug":"claude-user","name":"Claude-User","operator":"Anthropic","purpose":"user","feeds":"Claude (live browse)","token":"Claude-User","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; Claude-User/1.0; +https://www.anthropic.com)","robots":"yes","verification":"No published IP ranges; verify by behaviour only.","docsUrl":"https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler","url":"https://numberhill.com/tools/ai-crawlers/claude-user"},{"slug":"perplexitybot","name":"PerplexityBot","operator":"Perplexity","purpose":"search","feeds":"Perplexity","token":"PerplexityBot","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)","robots":"yes","verification":"Perplexity publishes its IP ranges as JSON; a hit from outside them is spoofed.","docsUrl":"https://docs.perplexity.ai/guides/bots","url":"https://numberhill.com/tools/ai-crawlers/perplexitybot"},{"slug":"perplexity-user","name":"Perplexity-User","operator":"Perplexity","purpose":"user","feeds":"Perplexity (live browse)","token":"Perplexity-User","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)","robots":"no","verification":"Perplexity publishes its IP ranges as JSON; a hit from outside them is spoofed.","docsUrl":"https://docs.perplexity.ai/guides/bots","url":"https://numberhill.com/tools/ai-crawlers/perplexity-user"},{"slug":"google-extended","name":"Google-Extended","operator":"Google","purpose":"control","feeds":"Gemini / AI Overviews","token":"Google-Extended","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; Google-Extended)","robots":"yes","verification":"There is nothing to verify: no request ever carries this token. Pages are fetched by Googlebot; this token only controls how they may be used.","docsUrl":"https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers","url":"https://numberhill.com/tools/ai-crawlers/google-extended"},{"slug":"googleother","name":"GoogleOther","operator":"Google","purpose":"training","feeds":"Google (other products)","token":"GoogleOther","aliases":["GoogleOther-Image","GoogleOther-Video"],"sampleUserAgent":"Mozilla/5.0 (compatible; GoogleOther)","robots":"yes","verification":"Google publishes IP ranges for its common crawlers; a hit from outside them is spoofed.","docsUrl":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","url":"https://numberhill.com/tools/ai-crawlers/googleother"},{"slug":"google-cloudvertexbot","name":"Google-CloudVertexBot","operator":"Google","purpose":"search","feeds":"Vertex AI grounding","token":"Google-CloudVertexBot","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; Google-CloudVertexBot/1.0; +http://www.google.com/bot.html)","robots":"yes","verification":"Google publishes IP ranges for its common crawlers; a hit from outside them is spoofed.","docsUrl":"https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers","url":"https://numberhill.com/tools/ai-crawlers/google-cloudvertexbot"},{"slug":"googleagent-mariner","name":"GoogleAgent-Mariner","operator":"Google","purpose":"agent","feeds":"Project Mariner (agent)","token":"GoogleAgent-Mariner","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; GoogleAgent-Mariner/1.0)","robots":"unclear","verification":"Google publishes IP ranges for its user-triggered fetchers.","docsUrl":"https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers","url":"https://numberhill.com/tools/ai-crawlers/googleagent-mariner"},{"slug":"applebot-extended","name":"Applebot-Extended","operator":"Apple","purpose":"control","feeds":"Apple Intelligence","token":"Applebot-Extended","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; Applebot-Extended/1.0; +http://www.apple.com/go/applebot)","robots":"yes","verification":"No request carries this token; pages are fetched by Applebot, whose hits can be verified by reverse DNS to *.applebot.apple.com.","docsUrl":"https://support.apple.com/en-us/119829","url":"https://numberhill.com/tools/ai-crawlers/applebot-extended"},{"slug":"amazonbot","name":"Amazonbot","operator":"Amazon","purpose":"training","feeds":"Alexa","token":"Amazonbot","aliases":[],"sampleUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)","robots":"yes","verification":"Reverse DNS of a genuine hit resolves to *.crawl.amazonbot.amazon.","docsUrl":"https://developer.amazon.com/amazonbot","url":"https://numberhill.com/tools/ai-crawlers/amazonbot"},{"slug":"novaact","name":"NovaAct","operator":"Amazon","purpose":"agent","feeds":"Amazon Nova Act (agent)","token":"NovaAct","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; NovaAct/1.0)","robots":"unclear","verification":"No published ranges.","docsUrl":null,"url":"https://numberhill.com/tools/ai-crawlers/novaact"},{"slug":"meta-externalagent","name":"Meta-ExternalAgent","operator":"Meta","purpose":"training","feeds":"Meta AI","token":"Meta-ExternalAgent","aliases":["FacebookBot"],"sampleUserAgent":"meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)","robots":"yes","verification":"Meta publishes its crawler IP ranges via its AS32934 route announcements.","docsUrl":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers","url":"https://numberhill.com/tools/ai-crawlers/meta-externalagent"},{"slug":"meta-externalfetcher","name":"Meta-ExternalFetcher","operator":"Meta","purpose":"user","feeds":"Meta AI (link fetch)","token":"Meta-ExternalFetcher","aliases":[],"sampleUserAgent":"meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)","robots":"no","verification":"Meta publishes its crawler IP ranges via its AS32934 route announcements.","docsUrl":"https://developers.facebook.com/docs/sharing/webmasters/web-crawlers","url":"https://numberhill.com/tools/ai-crawlers/meta-externalfetcher"},{"slug":"duckassistbot","name":"DuckAssistBot","operator":"DuckDuckGo","purpose":"search","feeds":"DuckDuckGo AI","token":"DuckAssistBot","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; DuckAssistBot/1.0; +http://duckduckgo.com/duckassistbot.html)","robots":"yes","verification":"DuckDuckGo publishes the IP addresses its bots use.","docsUrl":"https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/","url":"https://numberhill.com/tools/ai-crawlers/duckassistbot"},{"slug":"mistralai-user","name":"MistralAI-User","operator":"Mistral AI","purpose":"user","feeds":"Le Chat","token":"MistralAI-User","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)","robots":"yes","verification":"Mistral publishes the IP ranges its fetcher uses.","docsUrl":"https://docs.mistral.ai/robots/","url":"https://numberhill.com/tools/ai-crawlers/mistralai-user"},{"slug":"ccbot","name":"CCBot","operator":"Common Crawl","purpose":"training","feeds":"Common Crawl corpus","token":"CCBot","aliases":[],"sampleUserAgent":"CCBot/2.0 (https://commoncrawl.org/faq/)","robots":"yes","verification":"Common Crawl documents that it crawls from AWS; reverse DNS points at amazonaws.com hosts.","docsUrl":"https://commoncrawl.org/ccbot","url":"https://numberhill.com/tools/ai-crawlers/ccbot"},{"slug":"bytespider","name":"Bytespider","operator":"ByteDance","purpose":"training","feeds":"Doubao / TikTok","token":"Bytespider","aliases":["TikTokSpider"],"sampleUserAgent":"Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)","robots":"unclear","verification":"ByteDance publishes no IP list.","docsUrl":null,"url":"https://numberhill.com/tools/ai-crawlers/bytespider"},{"slug":"cohere","name":"Cohere","operator":"Cohere","purpose":"training","feeds":"Cohere","token":"cohere-ai","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; cohere-ai/1.0)","robots":"unclear","verification":"No published ranges.","docsUrl":null,"url":"https://numberhill.com/tools/ai-crawlers/cohere"},{"slug":"diffbot","name":"Diffbot","operator":"Diffbot","purpose":"training","feeds":"Diffbot KG","token":"Diffbot","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; Diffbot/0.1; +http://www.diffbot.com)","robots":"unclear","verification":"No published ranges.","docsUrl":null,"url":"https://numberhill.com/tools/ai-crawlers/diffbot"},{"slug":"petalbot","name":"PetalBot","operator":"Huawei","purpose":"search","feeds":"Petal Search","token":"PetalBot","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible;PetalBot;+https://webmaster.petalsearch.com/site/petalbot)","robots":"yes","verification":"Huawei documents reverse-DNS verification against petalsearch.com hosts.","docsUrl":"https://webmaster.petalsearch.com/site/petalbot","url":"https://numberhill.com/tools/ai-crawlers/petalbot"},{"slug":"pangubot","name":"PanguBot","operator":"Huawei","purpose":"training","feeds":"Huawei PanGu","token":"PanguBot","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; PanguBot/1.0)","robots":"unclear","verification":"No published ranges.","docsUrl":null,"url":"https://numberhill.com/tools/ai-crawlers/pangubot"},{"slug":"youbot","name":"YouBot","operator":"You.com","purpose":"search","feeds":"You.com","token":"YouBot","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; YouBot/1.0; +https://about.you.com/youbot/)","robots":"yes","verification":"No published ranges.","docsUrl":"https://about.you.com/youbot/","url":"https://numberhill.com/tools/ai-crawlers/youbot"},{"slug":"imagesiftbot","name":"ImagesiftBot","operator":"The Hive","purpose":"training","feeds":"ImageSift","token":"ImagesiftBot","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; ImagesiftBot/1.0; +https://imagesift.com/about)","robots":"yes","verification":"No published ranges.","docsUrl":"https://imagesift.com/about","url":"https://numberhill.com/tools/ai-crawlers/imagesiftbot"},{"slug":"ai2bot","name":"AI2Bot","operator":"Allen Institute for AI","purpose":"training","feeds":"Allen Institute","token":"AI2Bot","aliases":["AI2Bot-Dolma"],"sampleUserAgent":"Mozilla/5.0 (compatible; AI2Bot-Dolma/1.0)","robots":"yes","verification":"No published ranges.","docsUrl":"https://allenai.org/crawler","url":"https://numberhill.com/tools/ai-crawlers/ai2bot"},{"slug":"timpibot","name":"Timpibot","operator":"Timpi","purpose":"search","feeds":"Timpi","token":"Timpibot","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; Timpibot/1.0)","robots":"unclear","verification":"No published ranges.","docsUrl":null,"url":"https://numberhill.com/tools/ai-crawlers/timpibot"},{"slug":"omgili","name":"Omgili","operator":"Webz.io","purpose":"training","feeds":"Webz.io corpus","token":"omgili","aliases":["omgilibot"],"sampleUserAgent":"omgili/0.5 +http://omgili.com","robots":"yes","verification":"No published ranges.","docsUrl":"https://webz.io/bot.html","url":"https://numberhill.com/tools/ai-crawlers/omgili"},{"slug":"webzio-extended","name":"Webzio-Extended","operator":"Webz.io","purpose":"control","feeds":"Webz.io corpus","token":"Webzio-Extended","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; Webzio-Extended/1.0)","robots":"yes","verification":"No request carries this token; it controls how omgili's crawl is used.","docsUrl":"https://webz.io/bot.html","url":"https://numberhill.com/tools/ai-crawlers/webzio-extended"},{"slug":"devin","name":"Devin","operator":"Cognition","purpose":"agent","feeds":"Devin (agent)","token":"Devin","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; Devin/1.0)","robots":"unclear","verification":"No published ranges.","docsUrl":null,"url":"https://numberhill.com/tools/ai-crawlers/devin"},{"slug":"firecrawlagent","name":"FirecrawlAgent","operator":"Firecrawl","purpose":"agent","feeds":"Firecrawl (agent scraping)","token":"FirecrawlAgent","aliases":["Firecrawl"],"sampleUserAgent":"Mozilla/5.0 (compatible; FirecrawlAgent/1.0)","robots":"unclear","verification":"No published ranges.","docsUrl":null,"url":"https://numberhill.com/tools/ai-crawlers/firecrawlagent"},{"slug":"xai-bot","name":"xAI-Bot","operator":"xAI","purpose":"search","feeds":"Grok","token":"xAI-Bot","aliases":["GrokBot"],"sampleUserAgent":"Mozilla/5.0 (compatible; xAI-Bot/1.0; +https://x.ai/bot)","robots":"unclear","verification":"No published ranges.","docsUrl":null,"url":"https://numberhill.com/tools/ai-crawlers/xai-bot"},{"slug":"yandexadditional","name":"YandexAdditional","operator":"Yandex","purpose":"training","feeds":"Yandex AI","token":"YandexAdditional","aliases":["YandexAdditionalBot"],"sampleUserAgent":"Mozilla/5.0 (compatible; YandexAdditional/1.0; +http://yandex.com/bots)","robots":"yes","verification":"Yandex documents reverse-DNS verification against yandex.ru, yandex.net and yandex.com hosts.","docsUrl":"https://yandex.com/support/webmaster/robot-workings/check-yandex-robots.html","url":"https://numberhill.com/tools/ai-crawlers/yandexadditional"},{"slug":"semrushbot-ocob","name":"SemrushBot-OCOB","operator":"Semrush","purpose":"user","feeds":"Semrush AI corpus","token":"SemrushBot-OCOB","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; SemrushBot-OCOB/1.0; +http://www.semrush.com/bot.html)","robots":"yes","verification":"No published ranges.","docsUrl":"https://www.semrush.com/bot/","url":"https://numberhill.com/tools/ai-crawlers/semrushbot-ocob"},{"slug":"phindbot","name":"PhindBot","operator":"Phind","purpose":"search","feeds":"Phind","token":"PhindBot","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; PhindBot/1.0)","robots":"unclear","verification":"No published ranges.","docsUrl":null,"url":"https://numberhill.com/tools/ai-crawlers/phindbot"},{"slug":"kagibot","name":"Kagibot","operator":"Kagi","purpose":"search","feeds":"Kagi Assistant","token":"Kagibot","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)","robots":"yes","verification":"Kagi publishes the IP addresses its crawler uses.","docsUrl":"https://kagi.com/bot","url":"https://numberhill.com/tools/ai-crawlers/kagibot"},{"slug":"bravebot","name":"Bravebot","operator":"Brave","purpose":"search","feeds":"Brave Leo","token":"Bravebot","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; Bravebot/1.0)","robots":"yes","verification":"No published ranges.","docsUrl":"https://search.brave.com/help/brave-search-crawler","url":"https://numberhill.com/tools/ai-crawlers/bravebot"},{"slug":"sbintuitionsbot","name":"SBIntuitionsBot","operator":"SB Intuitions (SoftBank)","purpose":"training","feeds":"SB Intuitions","token":"SBIntuitionsBot","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; SBIntuitionsBot/1.0; +https://www.sbintuitions.co.jp/en/bot/)","robots":"yes","verification":"No published ranges.","docsUrl":"https://www.sbintuitions.co.jp/en/bot/","url":"https://numberhill.com/tools/ai-crawlers/sbintuitionsbot"},{"slug":"icc-crawler","name":"ICC-Crawler","operator":"NICT (Japan)","purpose":"training","feeds":"NICT corpus","token":"ICC-Crawler","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; ICC-Crawler/1.0; +https://ucri.nict.go.jp/en/icccrawler.html)","robots":"yes","verification":"No published ranges.","docsUrl":"https://ucri.nict.go.jp/en/icccrawler.html","url":"https://numberhill.com/tools/ai-crawlers/icc-crawler"},{"slug":"iaskspider","name":"iAskSpider","operator":"iAsk.ai","purpose":"search","feeds":"iAsk.ai","token":"iAskSpider","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; iAskSpider/1.0)","robots":"unclear","verification":"No published ranges.","docsUrl":null,"url":"https://numberhill.com/tools/ai-crawlers/iaskspider"},{"slug":"velenpublicwebcrawler","name":"VelenPublicWebCrawler","operator":"Velen","purpose":"training","feeds":"Velen corpus","token":"VelenPublicWebCrawler","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; VelenPublicWebCrawler/1.0; +https://velen.io)","robots":"unclear","verification":"No published ranges.","docsUrl":null,"url":"https://numberhill.com/tools/ai-crawlers/velenpublicwebcrawler"},{"slug":"img2dataset","name":"img2dataset","operator":"Open-source tool","purpose":"training","feeds":"Dataset builder","token":"img2dataset","aliases":[],"sampleUserAgent":"img2dataset","robots":"no","verification":"Not an operator; anyone can run it, from any address.","docsUrl":"https://github.com/rom1504/img2dataset","url":"https://numberhill.com/tools/ai-crawlers/img2dataset"},{"slug":"aihitbot","name":"aiHitBot","operator":"aiHit","purpose":"training","feeds":"aiHit corpus","token":"aiHitBot","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; aiHitBot/1.0; +https://www.aihitdata.com/about)","robots":"unclear","verification":"No published ranges.","docsUrl":null,"url":"https://numberhill.com/tools/ai-crawlers/aihitbot"},{"slug":"andibot","name":"Andibot","operator":"Andi","purpose":"search","feeds":"Andi","token":"Andibot","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; Andibot/1.0)","robots":"unclear","verification":"No published ranges.","docsUrl":null,"url":"https://numberhill.com/tools/ai-crawlers/andibot"},{"slug":"crawlspace","name":"Crawlspace","operator":"Crawlspace","purpose":"training","feeds":"Crawlspace corpus","token":"Crawlspace","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; Crawlspace/1.0)","robots":"unclear","verification":"No published ranges.","docsUrl":null,"url":"https://numberhill.com/tools/ai-crawlers/crawlspace"},{"slug":"poseidon-research","name":"Poseidon Research","operator":"Poseidon Research","purpose":"training","feeds":"Poseidon corpus","token":"Poseidon Research Crawler","aliases":[],"sampleUserAgent":"Mozilla/5.0 (compatible; Poseidon Research Crawler/1.0)","robots":"unclear","verification":"No published ranges.","docsUrl":null,"url":"https://numberhill.com/tools/ai-crawlers/poseidon-research"}]}