{"name":"AI crawler robots.txt study","source":"https://geoprompttracker.com/research/ai-crawler-robots-txt-study","license":"CC-BY-4.0","attribution":"GeoPromptTracker (https://geoprompttracker.com)","methodology":{"source":"Tranco top sites list","trancoListId":"Y895G","trancoUrl":"https://tranco-list.eu/list/Y895G","domainsChecked":10000,"sitesWithRobotsTxt":5250,"fetchedAt":"2026-11-11","rules":"RFC 9309 group selection: a bot's own named group wins over User-agent: *. Blocked = the root path / is disallowed for that bot (longest match, Allow wins ties)."},"bots":{"gptbot":{"userAgent":"GPTBot","mentioned":1158,"blockedByName":805,"blockedViaWildcard":203,"blockedTotal":1008,"partialByName":199,"top1k":{"sites":503,"blocked":113},"examples":[{"rank":5,"domain":"facebook.com"},{"rank":6,"domain":"fbcdn.net"},{"rank":14,"domain":"instagram.com"},{"rank":17,"domain":"linkedin.com"},{"rank":29,"domain":"amazon.com"},{"rank":46,"domain":"whatsapp.net"},{"rank":63,"domain":"tiktok.com"},{"rank":66,"domain":"whatsapp.com"},{"rank":69,"domain":"yahoo.com"},{"rank":96,"domain":"msn.com"},{"rank":101,"domain":"t-online.de"},{"rank":159,"domain":"snapchat.com"},{"rank":160,"domain":"nytimes.com"},{"rank":167,"domain":"gravatar.com"},{"rank":192,"domain":"cnn.com"},{"rank":196,"domain":"ebay.com"},{"rank":208,"domain":"medium.com"},{"rank":218,"domain":"bbc.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":242,"domain":"bbc.co.uk"},{"rank":291,"domain":"launchpad.net"},{"rank":295,"domain":"nature.com"},{"rank":307,"domain":"alibaba.com"},{"rank":312,"domain":"canva.com"},{"rank":318,"domain":"wiley.com"}]},"oai-searchbot":{"userAgent":"OAI-SearchBot","mentioned":595,"blockedByName":244,"blockedViaWildcard":204,"blockedTotal":448,"partialByName":192,"top1k":{"sites":503,"blocked":65},"examples":[{"rank":29,"domain":"amazon.com"},{"rank":63,"domain":"tiktok.com"},{"rank":96,"domain":"msn.com"},{"rank":160,"domain":"nytimes.com"},{"rank":192,"domain":"cnn.com"},{"rank":218,"domain":"bbc.com"},{"rank":242,"domain":"bbc.co.uk"},{"rank":291,"domain":"launchpad.net"},{"rank":326,"domain":"weibo.com"},{"rank":332,"domain":"amazon.co.uk"},{"rank":384,"domain":"amazon.de"},{"rank":412,"domain":"claude.ai"},{"rank":491,"domain":"cnbc.com"},{"rank":500,"domain":"amazon.co.jp"},{"rank":505,"domain":"quora.com"},{"rank":582,"domain":"dailymail.com"},{"rank":628,"domain":"usatoday.com"},{"rank":632,"domain":"telegraph.co.uk"},{"rank":637,"domain":"amazon.fr"},{"rank":639,"domain":"amazon.ca"},{"rank":677,"domain":"amazon.in"},{"rank":692,"domain":"yelp.com"},{"rank":701,"domain":"spiegel.de"},{"rank":717,"domain":"amazon.es"},{"rank":721,"domain":"sharethrough.com"}]},"chatgpt-user":{"userAgent":"ChatGPT-User","mentioned":681,"blockedByName":341,"blockedViaWildcard":205,"blockedTotal":546,"partialByName":189,"top1k":{"sites":503,"blocked":74},"examples":[{"rank":17,"domain":"linkedin.com"},{"rank":29,"domain":"amazon.com"},{"rank":63,"domain":"tiktok.com"},{"rank":69,"domain":"yahoo.com"},{"rank":82,"domain":"amazonvideo.com"},{"rank":96,"domain":"msn.com"},{"rank":160,"domain":"nytimes.com"},{"rank":192,"domain":"cnn.com"},{"rank":218,"domain":"bbc.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":242,"domain":"bbc.co.uk"},{"rank":295,"domain":"nature.com"},{"rank":326,"domain":"weibo.com"},{"rank":332,"domain":"amazon.co.uk"},{"rank":339,"domain":"espn.com"},{"rank":384,"domain":"amazon.de"},{"rank":412,"domain":"claude.ai"},{"rank":491,"domain":"cnbc.com"},{"rank":500,"domain":"amazon.co.jp"},{"rank":505,"domain":"quora.com"},{"rank":582,"domain":"dailymail.com"},{"rank":626,"domain":"techcrunch.com"},{"rank":628,"domain":"usatoday.com"},{"rank":632,"domain":"telegraph.co.uk"},{"rank":637,"domain":"amazon.fr"}]},"claudebot":{"userAgent":"ClaudeBot","mentioned":1028,"blockedByName":749,"blockedViaWildcard":217,"blockedTotal":966,"partialByName":145,"top1k":{"sites":503,"blocked":114},"examples":[{"rank":5,"domain":"facebook.com"},{"rank":6,"domain":"fbcdn.net"},{"rank":14,"domain":"instagram.com"},{"rank":17,"domain":"linkedin.com"},{"rank":29,"domain":"amazon.com"},{"rank":46,"domain":"whatsapp.net"},{"rank":63,"domain":"tiktok.com"},{"rank":66,"domain":"whatsapp.com"},{"rank":69,"domain":"yahoo.com"},{"rank":96,"domain":"msn.com"},{"rank":101,"domain":"t-online.de"},{"rank":159,"domain":"snapchat.com"},{"rank":160,"domain":"nytimes.com"},{"rank":192,"domain":"cnn.com"},{"rank":196,"domain":"ebay.com"},{"rank":201,"domain":"theguardian.com"},{"rank":208,"domain":"medium.com"},{"rank":218,"domain":"bbc.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":242,"domain":"bbc.co.uk"},{"rank":291,"domain":"launchpad.net"},{"rank":295,"domain":"nature.com"},{"rank":312,"domain":"canva.com"},{"rank":318,"domain":"wiley.com"},{"rank":332,"domain":"amazon.co.uk"}]},"claude-user":{"userAgent":"Claude-User","mentioned":407,"blockedByName":240,"blockedViaWildcard":220,"blockedTotal":460,"partialByName":94,"top1k":{"sites":503,"blocked":76},"examples":[{"rank":17,"domain":"linkedin.com"},{"rank":29,"domain":"amazon.com"},{"rank":63,"domain":"tiktok.com"},{"rank":82,"domain":"amazonvideo.com"},{"rank":96,"domain":"msn.com"},{"rank":160,"domain":"nytimes.com"},{"rank":192,"domain":"cnn.com"},{"rank":201,"domain":"theguardian.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":291,"domain":"launchpad.net"},{"rank":332,"domain":"amazon.co.uk"},{"rank":384,"domain":"amazon.de"},{"rank":491,"domain":"cnbc.com"},{"rank":500,"domain":"amazon.co.jp"},{"rank":505,"domain":"quora.com"},{"rank":566,"domain":"wired.com"},{"rank":580,"domain":"uol.com.br"},{"rank":582,"domain":"dailymail.com"},{"rank":628,"domain":"usatoday.com"},{"rank":632,"domain":"telegraph.co.uk"},{"rank":637,"domain":"amazon.fr"},{"rank":639,"domain":"amazon.ca"},{"rank":677,"domain":"amazon.in"},{"rank":679,"domain":"elpais.com"},{"rank":683,"domain":"primevideo.com"}]},"claude-searchbot":{"userAgent":"Claude-SearchBot","mentioned":402,"blockedByName":220,"blockedViaWildcard":221,"blockedTotal":441,"partialByName":112,"top1k":{"sites":503,"blocked":75},"examples":[{"rank":29,"domain":"amazon.com"},{"rank":63,"domain":"tiktok.com"},{"rank":96,"domain":"msn.com"},{"rank":160,"domain":"nytimes.com"},{"rank":192,"domain":"cnn.com"},{"rank":201,"domain":"theguardian.com"},{"rank":218,"domain":"bbc.com"},{"rank":242,"domain":"bbc.co.uk"},{"rank":291,"domain":"launchpad.net"},{"rank":332,"domain":"amazon.co.uk"},{"rank":384,"domain":"amazon.de"},{"rank":433,"domain":"unsplash.com"},{"rank":491,"domain":"cnbc.com"},{"rank":500,"domain":"amazon.co.jp"},{"rank":505,"domain":"quora.com"},{"rank":566,"domain":"wired.com"},{"rank":580,"domain":"uol.com.br"},{"rank":582,"domain":"dailymail.com"},{"rank":628,"domain":"usatoday.com"},{"rank":632,"domain":"telegraph.co.uk"},{"rank":637,"domain":"amazon.fr"},{"rank":639,"domain":"amazon.ca"},{"rank":677,"domain":"amazon.in"},{"rank":679,"domain":"elpais.com"},{"rank":692,"domain":"yelp.com"}]},"perplexitybot":{"userAgent":"PerplexityBot","mentioned":741,"blockedByName":409,"blockedViaWildcard":206,"blockedTotal":615,"partialByName":172,"top1k":{"sites":503,"blocked":92},"examples":[{"rank":5,"domain":"facebook.com"},{"rank":6,"domain":"fbcdn.net"},{"rank":14,"domain":"instagram.com"},{"rank":17,"domain":"linkedin.com"},{"rank":29,"domain":"amazon.com"},{"rank":46,"domain":"whatsapp.net"},{"rank":63,"domain":"tiktok.com"},{"rank":66,"domain":"whatsapp.com"},{"rank":69,"domain":"yahoo.com"},{"rank":82,"domain":"amazonvideo.com"},{"rank":96,"domain":"msn.com"},{"rank":101,"domain":"t-online.de"},{"rank":115,"domain":"chatgpt.com"},{"rank":160,"domain":"nytimes.com"},{"rank":192,"domain":"cnn.com"},{"rank":196,"domain":"ebay.com"},{"rank":201,"domain":"theguardian.com"},{"rank":218,"domain":"bbc.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":242,"domain":"bbc.co.uk"},{"rank":291,"domain":"launchpad.net"},{"rank":295,"domain":"nature.com"},{"rank":332,"domain":"amazon.co.uk"},{"rank":384,"domain":"amazon.de"},{"rank":401,"domain":"bloomberg.com"}]},"perplexity-user":{"userAgent":"Perplexity-User","mentioned":401,"blockedByName":251,"blockedViaWildcard":228,"blockedTotal":479,"partialByName":90,"top1k":{"sites":503,"blocked":81},"examples":[{"rank":17,"domain":"linkedin.com"},{"rank":29,"domain":"amazon.com"},{"rank":63,"domain":"tiktok.com"},{"rank":82,"domain":"amazonvideo.com"},{"rank":96,"domain":"msn.com"},{"rank":160,"domain":"nytimes.com"},{"rank":192,"domain":"cnn.com"},{"rank":218,"domain":"bbc.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":242,"domain":"bbc.co.uk"},{"rank":326,"domain":"weibo.com"},{"rank":332,"domain":"amazon.co.uk"},{"rank":384,"domain":"amazon.de"},{"rank":491,"domain":"cnbc.com"},{"rank":500,"domain":"amazon.co.jp"},{"rank":544,"domain":"ft.com"},{"rank":566,"domain":"wired.com"},{"rank":580,"domain":"uol.com.br"},{"rank":582,"domain":"dailymail.com"},{"rank":628,"domain":"usatoday.com"},{"rank":632,"domain":"telegraph.co.uk"},{"rank":637,"domain":"amazon.fr"},{"rank":639,"domain":"amazon.ca"},{"rank":677,"domain":"amazon.in"},{"rank":679,"domain":"elpais.com"}]},"google-extended":{"userAgent":"Google-Extended","mentioned":976,"blockedByName":683,"blockedViaWildcard":209,"blockedTotal":892,"partialByName":140,"top1k":{"sites":503,"blocked":100},"examples":[{"rank":5,"domain":"facebook.com"},{"rank":6,"domain":"fbcdn.net"},{"rank":14,"domain":"instagram.com"},{"rank":16,"domain":"twitter.com"},{"rank":17,"domain":"linkedin.com"},{"rank":29,"domain":"amazon.com"},{"rank":46,"domain":"whatsapp.net"},{"rank":49,"domain":"x.com"},{"rank":63,"domain":"tiktok.com"},{"rank":66,"domain":"whatsapp.com"},{"rank":69,"domain":"yahoo.com"},{"rank":96,"domain":"msn.com"},{"rank":115,"domain":"chatgpt.com"},{"rank":159,"domain":"snapchat.com"},{"rank":160,"domain":"nytimes.com"},{"rank":167,"domain":"gravatar.com"},{"rank":218,"domain":"bbc.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":242,"domain":"bbc.co.uk"},{"rank":291,"domain":"launchpad.net"},{"rank":295,"domain":"nature.com"},{"rank":307,"domain":"alibaba.com"},{"rank":318,"domain":"wiley.com"},{"rank":332,"domain":"amazon.co.uk"},{"rank":339,"domain":"espn.com"}]},"ccbot":{"userAgent":"CCBot","mentioned":1022,"blockedByName":856,"blockedViaWildcard":226,"blockedTotal":1082,"partialByName":95,"top1k":{"sites":503,"blocked":127},"examples":[{"rank":17,"domain":"linkedin.com"},{"rank":29,"domain":"amazon.com"},{"rank":63,"domain":"tiktok.com"},{"rank":69,"domain":"yahoo.com"},{"rank":82,"domain":"amazonvideo.com"},{"rank":86,"domain":"vimeo.com"},{"rank":96,"domain":"msn.com"},{"rank":101,"domain":"t-online.de"},{"rank":115,"domain":"chatgpt.com"},{"rank":159,"domain":"snapchat.com"},{"rank":160,"domain":"nytimes.com"},{"rank":167,"domain":"gravatar.com"},{"rank":189,"domain":"forter.com"},{"rank":192,"domain":"cnn.com"},{"rank":196,"domain":"ebay.com"},{"rank":201,"domain":"theguardian.com"},{"rank":218,"domain":"bbc.com"},{"rank":226,"domain":"weather.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":242,"domain":"bbc.co.uk"},{"rank":270,"domain":"who.int"},{"rank":291,"domain":"launchpad.net"},{"rank":295,"domain":"nature.com"},{"rank":312,"domain":"canva.com"},{"rank":318,"domain":"wiley.com"}]},"bytespider":{"userAgent":"Bytespider","mentioned":945,"blockedByName":852,"blockedViaWildcard":231,"blockedTotal":1083,"partialByName":60,"top1k":{"sites":503,"blocked":132},"examples":[{"rank":17,"domain":"linkedin.com"},{"rank":29,"domain":"amazon.com"},{"rank":34,"domain":"github.com"},{"rank":63,"domain":"tiktok.com"},{"rank":69,"domain":"yahoo.com"},{"rank":82,"domain":"amazonvideo.com"},{"rank":86,"domain":"vimeo.com"},{"rank":96,"domain":"msn.com"},{"rank":101,"domain":"t-online.de"},{"rank":115,"domain":"chatgpt.com"},{"rank":159,"domain":"snapchat.com"},{"rank":160,"domain":"nytimes.com"},{"rank":167,"domain":"gravatar.com"},{"rank":189,"domain":"forter.com"},{"rank":192,"domain":"cnn.com"},{"rank":196,"domain":"ebay.com"},{"rank":201,"domain":"theguardian.com"},{"rank":206,"domain":"unity3d.com"},{"rank":208,"domain":"medium.com"},{"rank":218,"domain":"bbc.com"},{"rank":226,"domain":"weather.com"},{"rank":233,"domain":"amplitude.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":242,"domain":"bbc.co.uk"},{"rank":291,"domain":"launchpad.net"}]},"applebot-extended":{"userAgent":"Applebot-Extended","mentioned":777,"blockedByName":644,"blockedViaWildcard":225,"blockedTotal":869,"partialByName":73,"top1k":{"sites":503,"blocked":99},"examples":[{"rank":5,"domain":"facebook.com"},{"rank":6,"domain":"fbcdn.net"},{"rank":14,"domain":"instagram.com"},{"rank":46,"domain":"whatsapp.net"},{"rank":63,"domain":"tiktok.com"},{"rank":66,"domain":"whatsapp.com"},{"rank":96,"domain":"msn.com"},{"rank":101,"domain":"t-online.de"},{"rank":159,"domain":"snapchat.com"},{"rank":160,"domain":"nytimes.com"},{"rank":192,"domain":"cnn.com"},{"rank":196,"domain":"ebay.com"},{"rank":201,"domain":"theguardian.com"},{"rank":208,"domain":"medium.com"},{"rank":218,"domain":"bbc.com"},{"rank":230,"domain":"researchgate.net"},{"rank":234,"domain":"sciencedirect.com"},{"rank":242,"domain":"bbc.co.uk"},{"rank":284,"domain":"reuters.com"},{"rank":291,"domain":"launchpad.net"},{"rank":295,"domain":"nature.com"},{"rank":312,"domain":"canva.com"},{"rank":318,"domain":"wiley.com"},{"rank":347,"domain":"trustpilot.com"},{"rank":401,"domain":"bloomberg.com"}]},"amazonbot":{"userAgent":"Amazonbot","mentioned":773,"blockedByName":636,"blockedViaWildcard":223,"blockedTotal":859,"partialByName":83,"top1k":{"sites":503,"blocked":91},"examples":[{"rank":5,"domain":"facebook.com"},{"rank":6,"domain":"fbcdn.net"},{"rank":14,"domain":"instagram.com"},{"rank":46,"domain":"whatsapp.net"},{"rank":66,"domain":"whatsapp.com"},{"rank":96,"domain":"msn.com"},{"rank":159,"domain":"snapchat.com"},{"rank":167,"domain":"gravatar.com"},{"rank":192,"domain":"cnn.com"},{"rank":196,"domain":"ebay.com"},{"rank":201,"domain":"theguardian.com"},{"rank":208,"domain":"medium.com"},{"rank":218,"domain":"bbc.com"},{"rank":226,"domain":"weather.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":242,"domain":"bbc.co.uk"},{"rank":291,"domain":"launchpad.net"},{"rank":295,"domain":"nature.com"},{"rank":318,"domain":"wiley.com"},{"rank":401,"domain":"bloomberg.com"},{"rank":410,"domain":"washingtonpost.com"},{"rank":449,"domain":"aliexpress.com"},{"rank":491,"domain":"cnbc.com"},{"rank":566,"domain":"wired.com"},{"rank":570,"domain":"threads.com"}]},"meta-externalagent":{"userAgent":"Meta-ExternalAgent","mentioned":803,"blockedByName":680,"blockedViaWildcard":225,"blockedTotal":905,"partialByName":76,"top1k":{"sites":503,"blocked":107},"examples":[{"rank":16,"domain":"twitter.com"},{"rank":17,"domain":"linkedin.com"},{"rank":29,"domain":"amazon.com"},{"rank":49,"domain":"x.com"},{"rank":57,"domain":"myfritz.net"},{"rank":63,"domain":"tiktok.com"},{"rank":82,"domain":"amazonvideo.com"},{"rank":96,"domain":"msn.com"},{"rank":101,"domain":"t-online.de"},{"rank":159,"domain":"snapchat.com"},{"rank":160,"domain":"nytimes.com"},{"rank":189,"domain":"forter.com"},{"rank":196,"domain":"ebay.com"},{"rank":201,"domain":"theguardian.com"},{"rank":208,"domain":"medium.com"},{"rank":218,"domain":"bbc.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":242,"domain":"bbc.co.uk"},{"rank":291,"domain":"launchpad.net"},{"rank":295,"domain":"nature.com"},{"rank":312,"domain":"canva.com"},{"rank":318,"domain":"wiley.com"},{"rank":326,"domain":"weibo.com"},{"rank":332,"domain":"amazon.co.uk"},{"rank":352,"domain":"ea.com"}]},"cohere-ai":{"userAgent":"cohere-ai","mentioned":714,"blockedByName":631,"blockedViaWildcard":233,"blockedTotal":864,"partialByName":52,"top1k":{"sites":503,"blocked":100},"examples":[{"rank":17,"domain":"linkedin.com"},{"rank":29,"domain":"amazon.com"},{"rank":69,"domain":"yahoo.com"},{"rank":82,"domain":"amazonvideo.com"},{"rank":96,"domain":"msn.com"},{"rank":159,"domain":"snapchat.com"},{"rank":160,"domain":"nytimes.com"},{"rank":192,"domain":"cnn.com"},{"rank":218,"domain":"bbc.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":242,"domain":"bbc.co.uk"},{"rank":291,"domain":"launchpad.net"},{"rank":295,"domain":"nature.com"},{"rank":318,"domain":"wiley.com"},{"rank":332,"domain":"amazon.co.uk"},{"rank":347,"domain":"trustpilot.com"},{"rank":352,"domain":"ea.com"},{"rank":384,"domain":"amazon.de"},{"rank":401,"domain":"bloomberg.com"},{"rank":427,"domain":"aol.com"},{"rank":491,"domain":"cnbc.com"},{"rank":500,"domain":"amazon.co.jp"},{"rank":544,"domain":"ft.com"},{"rank":566,"domain":"wired.com"},{"rank":580,"domain":"uol.com.br"}]},"applebot":{"userAgent":"Applebot","mentioned":274,"blockedByName":91,"blockedViaWildcard":196,"blockedTotal":287,"partialByName":111,"top1k":{"sites":503,"blocked":33},"examples":[{"rank":6,"domain":"fbcdn.net"},{"rank":96,"domain":"msn.com"},{"rank":580,"domain":"uol.com.br"},{"rank":758,"domain":"cnet.com"},{"rank":961,"domain":"corriere.it"},{"rank":1197,"domain":"nu.nl"},{"rank":1201,"domain":"theconversation.com"},{"rank":1205,"domain":"ad.nl"},{"rank":1245,"domain":"furaffinity.net"},{"rank":1273,"domain":"hln.be"},{"rank":1397,"domain":"dw.com"},{"rank":1662,"domain":"humoruniv.com"},{"rank":1791,"domain":"streamrizz.com"},{"rank":1975,"domain":"zdnet.com"},{"rank":2133,"domain":"ign.com"},{"rank":2211,"domain":"mashable.com"},{"rank":2473,"domain":"techradar.com"},{"rank":2600,"domain":"heise.de"},{"rank":2771,"domain":"livescience.com"},{"rank":2929,"domain":"chicagotribune.com"},{"rank":3164,"domain":"tomshardware.com"},{"rank":3290,"domain":"franceinfo.fr"},{"rank":3397,"domain":"lifehacker.com"},{"rank":3566,"domain":"politico.eu"},{"rank":3569,"domain":"smi2.ru"}]},"googleother":{"userAgent":"GoogleOther","mentioned":382,"blockedByName":322,"blockedViaWildcard":222,"blockedTotal":544,"partialByName":38,"top1k":{"sites":503,"blocked":73},"examples":[{"rank":29,"domain":"amazon.com"},{"rank":82,"domain":"amazonvideo.com"},{"rank":96,"domain":"msn.com"},{"rank":159,"domain":"snapchat.com"},{"rank":192,"domain":"cnn.com"},{"rank":208,"domain":"medium.com"},{"rank":226,"domain":"weather.com"},{"rank":230,"domain":"researchgate.net"},{"rank":295,"domain":"nature.com"},{"rank":318,"domain":"wiley.com"},{"rank":332,"domain":"amazon.co.uk"},{"rank":384,"domain":"amazon.de"},{"rank":435,"domain":"indeed.com"},{"rank":500,"domain":"amazon.co.jp"},{"rank":544,"domain":"ft.com"},{"rank":566,"domain":"wired.com"},{"rank":628,"domain":"usatoday.com"},{"rank":632,"domain":"telegraph.co.uk"},{"rank":637,"domain":"amazon.fr"},{"rank":639,"domain":"amazon.ca"},{"rank":677,"domain":"amazon.in"},{"rank":683,"domain":"primevideo.com"},{"rank":691,"domain":"kick.com"},{"rank":695,"domain":"lemonde.fr"},{"rank":717,"domain":"amazon.es"}]},"mistralai-user":{"userAgent":"MistralAI-User","mentioned":256,"blockedByName":206,"blockedViaWildcard":239,"blockedTotal":445,"partialByName":32,"top1k":{"sites":503,"blocked":74},"examples":[{"rank":29,"domain":"amazon.com"},{"rank":63,"domain":"tiktok.com"},{"rank":82,"domain":"amazonvideo.com"},{"rank":96,"domain":"msn.com"},{"rank":192,"domain":"cnn.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":332,"domain":"amazon.co.uk"},{"rank":384,"domain":"amazon.de"},{"rank":500,"domain":"amazon.co.jp"},{"rank":566,"domain":"wired.com"},{"rank":580,"domain":"uol.com.br"},{"rank":582,"domain":"dailymail.com"},{"rank":628,"domain":"usatoday.com"},{"rank":632,"domain":"telegraph.co.uk"},{"rank":637,"domain":"amazon.fr"},{"rank":639,"domain":"amazon.ca"},{"rank":677,"domain":"amazon.in"},{"rank":679,"domain":"elpais.com"},{"rank":683,"domain":"primevideo.com"},{"rank":695,"domain":"lemonde.fr"},{"rank":701,"domain":"spiegel.de"},{"rank":717,"domain":"amazon.es"},{"rank":729,"domain":"dailymail.co.uk"},{"rank":737,"domain":"amazon.it"},{"rank":758,"domain":"cnet.com"}]},"meta-externalfetcher":{"userAgent":"Meta-ExternalFetcher","mentioned":305,"blockedByName":262,"blockedViaWildcard":233,"blockedTotal":495,"partialByName":27,"top1k":{"sites":503,"blocked":74},"examples":[{"rank":16,"domain":"twitter.com"},{"rank":17,"domain":"linkedin.com"},{"rank":29,"domain":"amazon.com"},{"rank":49,"domain":"x.com"},{"rank":82,"domain":"amazonvideo.com"},{"rank":96,"domain":"msn.com"},{"rank":159,"domain":"snapchat.com"},{"rank":160,"domain":"nytimes.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":291,"domain":"launchpad.net"},{"rank":332,"domain":"amazon.co.uk"},{"rank":384,"domain":"amazon.de"},{"rank":401,"domain":"bloomberg.com"},{"rank":410,"domain":"washingtonpost.com"},{"rank":500,"domain":"amazon.co.jp"},{"rank":544,"domain":"ft.com"},{"rank":566,"domain":"wired.com"},{"rank":580,"domain":"uol.com.br"},{"rank":582,"domain":"dailymail.com"},{"rank":628,"domain":"usatoday.com"},{"rank":632,"domain":"telegraph.co.uk"},{"rank":637,"domain":"amazon.fr"},{"rank":639,"domain":"amazon.ca"},{"rank":677,"domain":"amazon.in"},{"rank":683,"domain":"primevideo.com"}]},"ai2bot":{"userAgent":"AI2Bot","mentioned":284,"blockedByName":257,"blockedViaWildcard":241,"blockedTotal":498,"partialByName":21,"top1k":{"sites":503,"blocked":84},"examples":[{"rank":29,"domain":"amazon.com"},{"rank":63,"domain":"tiktok.com"},{"rank":82,"domain":"amazonvideo.com"},{"rank":96,"domain":"msn.com"},{"rank":159,"domain":"snapchat.com"},{"rank":192,"domain":"cnn.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":291,"domain":"launchpad.net"},{"rank":312,"domain":"canva.com"},{"rank":332,"domain":"amazon.co.uk"},{"rank":352,"domain":"ea.com"},{"rank":384,"domain":"amazon.de"},{"rank":498,"domain":"scribd.com"},{"rank":500,"domain":"amazon.co.jp"},{"rank":530,"domain":"behance.net"},{"rank":564,"domain":"slideshare.net"},{"rank":580,"domain":"uol.com.br"},{"rank":582,"domain":"dailymail.com"},{"rank":622,"domain":"nintendo.com"},{"rank":628,"domain":"usatoday.com"},{"rank":632,"domain":"telegraph.co.uk"},{"rank":637,"domain":"amazon.fr"},{"rank":639,"domain":"amazon.ca"},{"rank":677,"domain":"amazon.in"},{"rank":683,"domain":"primevideo.com"}]},"duckassistbot":{"userAgent":"DuckAssistBot","mentioned":324,"blockedByName":246,"blockedViaWildcard":231,"blockedTotal":477,"partialByName":56,"top1k":{"sites":503,"blocked":76},"examples":[{"rank":17,"domain":"linkedin.com"},{"rank":29,"domain":"amazon.com"},{"rank":63,"domain":"tiktok.com"},{"rank":82,"domain":"amazonvideo.com"},{"rank":96,"domain":"msn.com"},{"rank":160,"domain":"nytimes.com"},{"rank":192,"domain":"cnn.com"},{"rank":201,"domain":"theguardian.com"},{"rank":218,"domain":"bbc.com"},{"rank":242,"domain":"bbc.co.uk"},{"rank":291,"domain":"launchpad.net"},{"rank":332,"domain":"amazon.co.uk"},{"rank":384,"domain":"amazon.de"},{"rank":401,"domain":"bloomberg.com"},{"rank":500,"domain":"amazon.co.jp"},{"rank":566,"domain":"wired.com"},{"rank":580,"domain":"uol.com.br"},{"rank":582,"domain":"dailymail.com"},{"rank":628,"domain":"usatoday.com"},{"rank":632,"domain":"telegraph.co.uk"},{"rank":637,"domain":"amazon.fr"},{"rank":639,"domain":"amazon.ca"},{"rank":677,"domain":"amazon.in"},{"rank":679,"domain":"elpais.com"},{"rank":683,"domain":"primevideo.com"}]},"youbot":{"userAgent":"YouBot","mentioned":428,"blockedByName":356,"blockedViaWildcard":233,"blockedTotal":589,"partialByName":44,"top1k":{"sites":503,"blocked":87},"examples":[{"rank":29,"domain":"amazon.com"},{"rank":69,"domain":"yahoo.com"},{"rank":82,"domain":"amazonvideo.com"},{"rank":96,"domain":"msn.com"},{"rank":159,"domain":"snapchat.com"},{"rank":160,"domain":"nytimes.com"},{"rank":192,"domain":"cnn.com"},{"rank":201,"domain":"theguardian.com"},{"rank":218,"domain":"bbc.com"},{"rank":226,"domain":"weather.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":242,"domain":"bbc.co.uk"},{"rank":291,"domain":"launchpad.net"},{"rank":295,"domain":"nature.com"},{"rank":332,"domain":"amazon.co.uk"},{"rank":347,"domain":"trustpilot.com"},{"rank":384,"domain":"amazon.de"},{"rank":401,"domain":"bloomberg.com"},{"rank":427,"domain":"aol.com"},{"rank":500,"domain":"amazon.co.jp"},{"rank":566,"domain":"wired.com"},{"rank":580,"domain":"uol.com.br"},{"rank":582,"domain":"dailymail.com"},{"rank":587,"domain":"tripadvisor.com"},{"rank":628,"domain":"usatoday.com"}]},"petalbot":{"userAgent":"PetalBot","mentioned":593,"blockedByName":556,"blockedViaWildcard":230,"blockedTotal":786,"partialByName":23,"top1k":{"sites":503,"blocked":98},"examples":[{"rank":5,"domain":"facebook.com"},{"rank":6,"domain":"fbcdn.net"},{"rank":14,"domain":"instagram.com"},{"rank":29,"domain":"amazon.com"},{"rank":46,"domain":"whatsapp.net"},{"rank":63,"domain":"tiktok.com"},{"rank":66,"domain":"whatsapp.com"},{"rank":69,"domain":"yahoo.com"},{"rank":82,"domain":"amazonvideo.com"},{"rank":159,"domain":"snapchat.com"},{"rank":192,"domain":"cnn.com"},{"rank":201,"domain":"theguardian.com"},{"rank":218,"domain":"bbc.com"},{"rank":229,"domain":"criteo.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":242,"domain":"bbc.co.uk"},{"rank":274,"domain":"wixsite.com"},{"rank":291,"domain":"launchpad.net"},{"rank":318,"domain":"wiley.com"},{"rank":332,"domain":"amazon.co.uk"},{"rank":384,"domain":"amazon.de"},{"rank":397,"domain":"issuu.com"},{"rank":401,"domain":"bloomberg.com"},{"rank":427,"domain":"aol.com"},{"rank":500,"domain":"amazon.co.jp"}]},"diffbot":{"userAgent":"Diffbot","mentioned":694,"blockedByName":657,"blockedViaWildcard":234,"blockedTotal":891,"partialByName":22,"top1k":{"sites":503,"blocked":108},"examples":[{"rank":17,"domain":"linkedin.com"},{"rank":29,"domain":"amazon.com"},{"rank":69,"domain":"yahoo.com"},{"rank":82,"domain":"amazonvideo.com"},{"rank":86,"domain":"vimeo.com"},{"rank":96,"domain":"msn.com"},{"rank":159,"domain":"snapchat.com"},{"rank":160,"domain":"nytimes.com"},{"rank":189,"domain":"forter.com"},{"rank":192,"domain":"cnn.com"},{"rank":218,"domain":"bbc.com"},{"rank":226,"domain":"weather.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":242,"domain":"bbc.co.uk"},{"rank":291,"domain":"launchpad.net"},{"rank":295,"domain":"nature.com"},{"rank":312,"domain":"canva.com"},{"rank":318,"domain":"wiley.com"},{"rank":332,"domain":"amazon.co.uk"},{"rank":347,"domain":"trustpilot.com"},{"rank":384,"domain":"amazon.de"},{"rank":401,"domain":"bloomberg.com"},{"rank":410,"domain":"washingtonpost.com"},{"rank":427,"domain":"aol.com"},{"rank":491,"domain":"cnbc.com"}]},"imagesiftbot":{"userAgent":"ImagesiftBot","mentioned":325,"blockedByName":308,"blockedViaWildcard":240,"blockedTotal":548,"partialByName":10,"top1k":{"sites":503,"blocked":92},"examples":[{"rank":29,"domain":"amazon.com"},{"rank":69,"domain":"yahoo.com"},{"rank":82,"domain":"amazonvideo.com"},{"rank":86,"domain":"vimeo.com"},{"rank":96,"domain":"msn.com"},{"rank":159,"domain":"snapchat.com"},{"rank":160,"domain":"nytimes.com"},{"rank":189,"domain":"forter.com"},{"rank":192,"domain":"cnn.com"},{"rank":226,"domain":"weather.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":291,"domain":"launchpad.net"},{"rank":295,"domain":"nature.com"},{"rank":312,"domain":"canva.com"},{"rank":332,"domain":"amazon.co.uk"},{"rank":347,"domain":"trustpilot.com"},{"rank":384,"domain":"amazon.de"},{"rank":410,"domain":"washingtonpost.com"},{"rank":427,"domain":"aol.com"},{"rank":433,"domain":"unsplash.com"},{"rank":498,"domain":"scribd.com"},{"rank":500,"domain":"amazon.co.jp"},{"rank":505,"domain":"quora.com"},{"rank":530,"domain":"behance.net"},{"rank":564,"domain":"slideshare.net"}]},"omgilibot":{"userAgent":"omgilibot","mentioned":391,"blockedByName":369,"blockedViaWildcard":240,"blockedTotal":609,"partialByName":16,"top1k":{"sites":503,"blocked":89},"examples":[{"rank":17,"domain":"linkedin.com"},{"rank":69,"domain":"yahoo.com"},{"rank":86,"domain":"vimeo.com"},{"rank":96,"domain":"msn.com"},{"rank":115,"domain":"chatgpt.com"},{"rank":159,"domain":"snapchat.com"},{"rank":160,"domain":"nytimes.com"},{"rank":167,"domain":"gravatar.com"},{"rank":189,"domain":"forter.com"},{"rank":192,"domain":"cnn.com"},{"rank":218,"domain":"bbc.com"},{"rank":226,"domain":"weather.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":242,"domain":"bbc.co.uk"},{"rank":291,"domain":"launchpad.net"},{"rank":295,"domain":"nature.com"},{"rank":339,"domain":"espn.com"},{"rank":347,"domain":"trustpilot.com"},{"rank":410,"domain":"washingtonpost.com"},{"rank":427,"domain":"aol.com"},{"rank":435,"domain":"indeed.com"},{"rank":491,"domain":"cnbc.com"},{"rank":530,"domain":"behance.net"},{"rank":544,"domain":"ft.com"},{"rank":580,"domain":"uol.com.br"}]},"timpibot":{"userAgent":"Timpibot","mentioned":353,"blockedByName":339,"blockedViaWildcard":238,"blockedTotal":577,"partialByName":8,"top1k":{"sites":503,"blocked":87},"examples":[{"rank":17,"domain":"linkedin.com"},{"rank":29,"domain":"amazon.com"},{"rank":82,"domain":"amazonvideo.com"},{"rank":96,"domain":"msn.com"},{"rank":159,"domain":"snapchat.com"},{"rank":160,"domain":"nytimes.com"},{"rank":192,"domain":"cnn.com"},{"rank":234,"domain":"sciencedirect.com"},{"rank":291,"domain":"launchpad.net"},{"rank":312,"domain":"canva.com"},{"rank":332,"domain":"amazon.co.uk"},{"rank":347,"domain":"trustpilot.com"},{"rank":384,"domain":"amazon.de"},{"rank":496,"domain":"businessinsider.com"},{"rank":500,"domain":"amazon.co.jp"},{"rank":530,"domain":"behance.net"},{"rank":566,"domain":"wired.com"},{"rank":580,"domain":"uol.com.br"},{"rank":582,"domain":"dailymail.com"},{"rank":622,"domain":"nintendo.com"},{"rank":628,"domain":"usatoday.com"},{"rank":632,"domain":"telegraph.co.uk"},{"rank":637,"domain":"amazon.fr"},{"rank":639,"domain":"amazon.ca"},{"rank":677,"domain":"amazon.in"}]},"pangubot":{"userAgent":"PanguBot","mentioned":213,"blockedByName":203,"blockedViaWildcard":247,"blockedTotal":450,"partialByName":8,"top1k":{"sites":503,"blocked":78},"examples":[{"rank":29,"domain":"amazon.com"},{"rank":82,"domain":"amazonvideo.com"},{"rank":96,"domain":"msn.com"},{"rank":159,"domain":"snapchat.com"},{"rank":192,"domain":"cnn.com"},{"rank":291,"domain":"launchpad.net"},{"rank":332,"domain":"amazon.co.uk"},{"rank":384,"domain":"amazon.de"},{"rank":500,"domain":"amazon.co.jp"},{"rank":530,"domain":"behance.net"},{"rank":566,"domain":"wired.com"},{"rank":580,"domain":"uol.com.br"},{"rank":582,"domain":"dailymail.com"},{"rank":587,"domain":"tripadvisor.com"},{"rank":622,"domain":"nintendo.com"},{"rank":628,"domain":"usatoday.com"},{"rank":632,"domain":"telegraph.co.uk"},{"rank":637,"domain":"amazon.fr"},{"rank":639,"domain":"amazon.ca"},{"rank":677,"domain":"amazon.in"},{"rank":683,"domain":"primevideo.com"},{"rank":695,"domain":"lemonde.fr"},{"rank":701,"domain":"spiegel.de"},{"rank":717,"domain":"amazon.es"},{"rank":729,"domain":"dailymail.co.uk"}]}}}