{
 "blogId": "D0191",
 "slug": "how-many-top-websites-block-ai-crawlers-2026",
 "title": "How Many Top Websites Block AI Crawlers in 2026?",
 "template": "CW1_ACCESS_INDEX",
 "editionId": "2026-06",
 "snapshotSha256": "741353c4304216ee37c1b1f874a70b9b370bcc5fcd242312052cab0a6b9369fb",
 "sealedAt": "2026-06-13",
 "source": "US Tech Automations Research",
 "methodology": "https://ustechautomations.com/resources/research",
 "canonicalUrl": "https://ustechautomations.com/resources/blog/how-many-top-websites-block-ai-crawlers-2026",
 "displays": [
  {
   "key": "edition.snapshotDate",
   "display": "June 13, 2026",
   "value": null,
   "kind": "window"
  },
  {
   "key": "edition.sitesChecked",
   "display": "122",
   "value": 122,
   "kind": "count"
  },
  {
   "key": "edition.sitesWithRobots",
   "display": "107",
   "value": 107,
   "kind": "count"
  },
  {
   "key": "edition.categories",
   "display": "10",
   "value": 10,
   "kind": "count"
  },
  {
   "key": "edition.botsTracked",
   "display": "21",
   "value": 21,
   "kind": "count"
  },
  {
   "key": "edition.operatorsTracked",
   "display": "12",
   "value": 12,
   "kind": "count"
  },
  {
   "key": "agg.anyAIBlock.count",
   "display": "48",
   "value": 48,
   "kind": "count"
  },
  {
   "key": "agg.anyAIBlock.pct",
   "display": "44.9%",
   "value": 44.9,
   "kind": "rate"
  },
  {
   "key": "agg.starBlock.count",
   "display": "9",
   "value": 9,
   "kind": "count"
  },
  {
   "key": "agg.starBlock.pct",
   "display": "8.4%",
   "value": 8.4,
   "kind": "rate"
  },
  {
   "key": "agg.llmsTxt.count",
   "display": "20",
   "value": 20,
   "kind": "count"
  },
  {
   "key": "agg.llmsTxt.pct",
   "display": "18.7%",
   "value": 18.7,
   "kind": "rate"
  },
  {
   "key": "bot.GPTBot.blockCount",
   "display": "33",
   "value": 33,
   "kind": "count"
  },
  {
   "key": "bot.GPTBot.blockPct",
   "display": "30.8%",
   "value": 30.8,
   "kind": "rate"
  },
  {
   "key": "bot.ClaudeBot.blockCount",
   "display": "38",
   "value": 38,
   "kind": "count"
  },
  {
   "key": "bot.ClaudeBot.blockPct",
   "display": "35.5%",
   "value": 35.5,
   "kind": "rate"
  },
  {
   "key": "bot.Google-Extended.blockCount",
   "display": "25",
   "value": 25,
   "kind": "count"
  },
  {
   "key": "bot.Google-Extended.blockPct",
   "display": "23.4%",
   "value": 23.4,
   "kind": "rate"
  },
  {
   "key": "bot.CCBot.blockCount",
   "display": "40",
   "value": 40,
   "kind": "count"
  },
  {
   "key": "bot.CCBot.blockPct",
   "display": "37.4%",
   "value": 37.4,
   "kind": "rate"
  },
  {
   "key": "bot.PerplexityBot.blockCount",
   "display": "29",
   "value": 29,
   "kind": "count"
  },
  {
   "key": "bot.PerplexityBot.blockPct",
   "display": "27.1%",
   "value": 27.1,
   "kind": "rate"
  },
  {
   "key": "bot.Bytespider.blockCount",
   "display": "37",
   "value": 37,
   "kind": "count"
  },
  {
   "key": "bot.Bytespider.blockPct",
   "display": "34.6%",
   "value": 34.6,
   "kind": "rate"
  },
  {
   "key": "bot.Meta-ExternalAgent.blockCount",
   "display": "30",
   "value": 30,
   "kind": "count"
  },
  {
   "key": "bot.Meta-ExternalAgent.blockPct",
   "display": "28%",
   "value": 28,
   "kind": "rate"
  },
  {
   "key": "bot.Amazonbot.blockCount",
   "display": "22",
   "value": 22,
   "kind": "count"
  },
  {
   "key": "bot.Amazonbot.blockPct",
   "display": "20.6%",
   "value": 20.6,
   "kind": "rate"
  },
  {
   "key": "bot.Applebot-Extended.blockCount",
   "display": "31",
   "value": 31,
   "kind": "count"
  },
  {
   "key": "bot.Applebot-Extended.blockPct",
   "display": "29%",
   "value": 29,
   "kind": "rate"
  },
  {
   "key": "operator.OpenAI.blockCount",
   "display": "35",
   "value": 35,
   "kind": "count"
  },
  {
   "key": "operator.OpenAI.blockPct",
   "display": "32.7%",
   "value": 32.7,
   "kind": "rate"
  },
  {
   "key": "operator.Anthropic.blockCount",
   "display": "39",
   "value": 39,
   "kind": "count"
  },
  {
   "key": "operator.Anthropic.blockPct",
   "display": "36.4%",
   "value": 36.4,
   "kind": "rate"
  },
  {
   "key": "operator.Google.blockCount",
   "display": "25",
   "value": 25,
   "kind": "count"
  },
  {
   "key": "operator.Google.blockPct",
   "display": "23.4%",
   "value": 23.4,
   "kind": "rate"
  },
  {
   "key": "operator.Common Crawl.blockCount",
   "display": "40",
   "value": 40,
   "kind": "count"
  },
  {
   "key": "operator.Common Crawl.blockPct",
   "display": "37.4%",
   "value": 37.4,
   "kind": "rate"
  },
  {
   "key": "operator.Perplexity.blockCount",
   "display": "29",
   "value": 29,
   "kind": "count"
  },
  {
   "key": "operator.Perplexity.blockPct",
   "display": "27.1%",
   "value": 27.1,
   "kind": "rate"
  },
  {
   "key": "operator.ByteDance.blockCount",
   "display": "37",
   "value": 37,
   "kind": "count"
  },
  {
   "key": "operator.ByteDance.blockPct",
   "display": "34.6%",
   "value": 34.6,
   "kind": "rate"
  },
  {
   "key": "operator.Meta.blockCount",
   "display": "35",
   "value": 35,
   "kind": "count"
  },
  {
   "key": "operator.Meta.blockPct",
   "display": "32.7%",
   "value": 32.7,
   "kind": "rate"
  },
  {
   "key": "operator.Amazon.blockCount",
   "display": "22",
   "value": 22,
   "kind": "count"
  },
  {
   "key": "operator.Amazon.blockPct",
   "display": "20.6%",
   "value": 20.6,
   "kind": "rate"
  },
  {
   "key": "operator.Apple.blockCount",
   "display": "31",
   "value": 31,
   "kind": "count"
  },
  {
   "key": "operator.Apple.blockPct",
   "display": "29%",
   "value": 29,
   "kind": "rate"
  },
  {
   "key": "operator.Cohere.blockCount",
   "display": "27",
   "value": 27,
   "kind": "count"
  },
  {
   "key": "operator.Cohere.blockPct",
   "display": "25.2%",
   "value": 25.2,
   "kind": "rate"
  },
  {
   "key": "operator.Mistral.blockCount",
   "display": "12",
   "value": 12,
   "kind": "count"
  },
  {
   "key": "operator.Mistral.blockPct",
   "display": "11.2%",
   "value": 11.2,
   "kind": "rate"
  },
  {
   "key": "operator.Diffbot.blockCount",
   "display": "30",
   "value": 30,
   "kind": "count"
  },
  {
   "key": "operator.Diffbot.blockPct",
   "display": "28%",
   "value": 28,
   "kind": "rate"
  },
  {
   "key": "category.News.sites",
   "display": "20",
   "value": 20,
   "kind": "count"
  },
  {
   "key": "category.News.withRobots",
   "display": "17",
   "value": 17,
   "kind": "count"
  },
  {
   "key": "category.News.anyBlock",
   "display": "14",
   "value": 14,
   "kind": "count"
  },
  {
   "key": "category.News.anyBlockPct",
   "display": "82.4%",
   "value": 82.4,
   "kind": "rate"
  },
  {
   "key": "category.Tech.sites",
   "display": "15",
   "value": 15,
   "kind": "count"
  },
  {
   "key": "category.Tech.withRobots",
   "display": "13",
   "value": 13,
   "kind": "count"
  },
  {
   "key": "category.Tech.anyBlock",
   "display": "9",
   "value": 9,
   "kind": "count"
  },
  {
   "key": "category.Tech.anyBlockPct",
   "display": "69.2%",
   "value": 69.2,
   "kind": "rate"
  },
  {
   "key": "category.Reference.sites",
   "display": "14",
   "value": 14,
   "kind": "count"
  },
  {
   "key": "category.Reference.withRobots",
   "display": "11",
   "value": 11,
   "kind": "count"
  },
  {
   "key": "category.Reference.anyBlock",
   "display": "6",
   "value": 6,
   "kind": "count"
  },
  {
   "key": "category.Reference.anyBlockPct",
   "display": "54.5%",
   "value": 54.5,
   "kind": "rate"
  },
  {
   "key": "category.Retail.sites",
   "display": "15",
   "value": 15,
   "kind": "count"
  },
  {
   "key": "category.Retail.withRobots",
   "display": "12",
   "value": 12,
   "kind": "count"
  },
  {
   "key": "category.Retail.anyBlock",
   "display": "2",
   "value": 2,
   "kind": "count"
  },
  {
   "key": "category.Retail.anyBlockPct",
   "display": "16.7%",
   "value": 16.7,
   "kind": "rate"
  },
  {
   "key": "category.Social.sites",
   "display": "10",
   "value": 10,
   "kind": "count"
  },
  {
   "key": "category.Social.withRobots",
   "display": "10",
   "value": 10,
   "kind": "count"
  },
  {
   "key": "category.Social.anyBlock",
   "display": "4",
   "value": 4,
   "kind": "count"
  },
  {
   "key": "category.Social.anyBlockPct",
   "display": "40%",
   "value": 40,
   "kind": "rate"
  },
  {
   "key": "category.Finance.sites",
   "display": "12",
   "value": 12,
   "kind": "count"
  },
  {
   "key": "category.Finance.withRobots",
   "display": "11",
   "value": 11,
   "kind": "count"
  },
  {
   "key": "category.Finance.anyBlock",
   "display": "2",
   "value": 2,
   "kind": "count"
  },
  {
   "key": "category.Finance.anyBlockPct",
   "display": "18.2%",
   "value": 18.2,
   "kind": "rate"
  },
  {
   "key": "category.Travel.sites",
   "display": "9",
   "value": 9,
   "kind": "count"
  },
  {
   "key": "category.Travel.withRobots",
   "display": "9",
   "value": 9,
   "kind": "count"
  },
  {
   "key": "category.Travel.anyBlock",
   "display": "3",
   "value": 3,
   "kind": "count"
  },
  {
   "key": "category.Travel.anyBlockPct",
   "display": "33.3%",
   "value": 33.3,
   "kind": "rate"
  },
  {
   "key": "category.Education.sites",
   "display": "9",
   "value": 9,
   "kind": "count"
  },
  {
   "key": "category.Education.withRobots",
   "display": "7",
   "value": 7,
   "kind": "count"
  },
  {
   "key": "category.Education.anyBlock",
   "display": "1",
   "value": 1,
   "kind": "count"
  },
  {
   "key": "category.Education.anyBlockPct",
   "display": "14.3%",
   "value": 14.3,
   "kind": "rate"
  },
  {
   "key": "category.Government.sites",
   "display": "9",
   "value": 9,
   "kind": "count"
  },
  {
   "key": "category.Government.withRobots",
   "display": "8",
   "value": 8,
   "kind": "count"
  },
  {
   "key": "category.Government.anyBlock",
   "display": "1",
   "value": 1,
   "kind": "count"
  },
  {
   "key": "category.Government.anyBlockPct",
   "display": "12.5%",
   "value": 12.5,
   "kind": "rate"
  },
  {
   "key": "category.Entertainment.sites",
   "display": "9",
   "value": 9,
   "kind": "count"
  },
  {
   "key": "category.Entertainment.withRobots",
   "display": "9",
   "value": 9,
   "kind": "count"
  },
  {
   "key": "category.Entertainment.anyBlock",
   "display": "6",
   "value": 6,
   "kind": "count"
  },
  {
   "key": "category.Entertainment.anyBlockPct",
   "display": "66.7%",
   "value": 66.7,
   "kind": "rate"
  }
 ]
}