{ "schema": "usta.closing-web-weekly.v1", "blogId": "D0417", "slug": "ai-crawler-blocking-trend-august-2026", "title": "AI Crawler Blocking by Bot: Weekly Closing Web Index", "editionId": "2026-08", "sealedAt": "2026-08-11", "comparisonDate": "2026-08-04", "canonicalUrl": "https://ustechautomations.com/resources/blog/ai-crawler-blocking-trend-august-2026", "methodologyUrl": "https://www.rfc-editor.org/rfc/rfc9309.html", "scope": { "panel": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "collection_method": "closing-web-v1", "request_path": "/", "verdicts": [ "DISALLOWED", "NOT_DISALLOWED_BY_TESTED_RULE", "UNKNOWN" ], "statement": "The result tests only whether the applicable stored robots.txt rules disallow the named product token from requesting /. It does not call an UNKNOWN allowed, imply permission for another path, prove crawler compliance, or grant access authorization.", "status_handling": "A sealed 4xx means no tested robots rule was available and is classified NOT_DISALLOWED_BY_TESTED_RULE. Transport failures, 5xx, other statuses, missing/truncated bodies, HTML-shaped soft 200s, and invalid UTF-8 are UNKNOWN.", "rule_precedence": "Exact case-insensitive product-token groups override wildcard groups; duplicate exact groups merge; the longest matching path rule wins; Allow wins an equal-length tie." }, "coverage": { "first_complete_date": "2026-06-09", "last_complete_date": "2026-08-11", "complete_date_count": 58, "gap_date_count": 6, "gap_dates": [ "2026-06-11", "2026-06-12", "2026-06-19", "2026-07-04", "2026-07-15", "2026-08-08" ], "incomplete_dates": [], "definition": "A complete date has one sealed row for each of four frozen resources for every frozen-panel domain; counts come from policy_snapshots, not run-log claims." }, "source_binding": { "universe_sha256": "8778484edc1487aa038aecb827b456950a6e583f8465a11b6810ba1223c11fd0", "methodology_sha256": "3f43a495546fa1c0027fad32ab69646191418c680dacb9ac2004e805ffbfe180", "read_model_method_sha256": "abb9e72b6829678773a8509caa8a873d91fd1fc160af98973ef88328a955ca44", "method_files": [ { "name": "closing_web/METHODOLOGY.md", "sha256": "3f43a495546fa1c0027fad32ab69646191418c680dacb9ac2004e805ffbfe180" }, { "name": "seo/scripts/closing-web-weekly.js", "sha256": "26bc6e95c3ed3d0abe266f41f032821941122943b1b3844d319461359266ab31" }, { "name": "seo/scripts/lib/closing-web-root-policy.js", "sha256": "092cc808b458d58d28feab6912bc83dd8559a3086103e48c9016eede832d4a63" } ] }, "negative_control": { "fixture_sha256": "0e05c8220b587675db452b151016220191bccb77d7972ce8215e7812e840a718", "tracked_token_verdict": "DISALLOWED", "fabricated_token_verdict": "NOT_DISALLOWED_BY_TESTED_RULE", "passed": true }, "stored_cell_accounting": { "publication_cutoff": "2026-08-11", "source_tables": [ "policy_snapshots", "blobs" ], "counts": { "stored_grid_cells": { "value": 23200000, "row_predicate": "policy_snapshots.snapshot_date <= :publication_cutoff" }, "transport_errors": { "value": 5642987, "row_predicate": "policy_snapshots.snapshot_date <= :publication_cutoff AND status_code IS NULL AND fetch_error IS NOT NULL" }, "answered_observations": { "value": 17557013, "row_predicate": "policy_snapshots.snapshot_date <= :publication_cutoff AND status_code IS NOT NULL AND fetch_error IS NULL" }, "retained_200_bodies": { "value": 5934853, "row_predicate": "policy_snapshots.snapshot_date <= :publication_cutoff AND status_code = 200 AND content_sha256 IS NOT NULL AND EXISTS (SELECT 1 FROM blobs WHERE blobs.content_sha256 = policy_snapshots.content_sha256)" }, "distinct_retained_blobs": { "value": 1020697, "row_predicate": "blobs.first_seen_date <= :publication_cutoff" } }, "reconciliation": { "partition_equation": "stored_grid_cells = transport_errors + answered_observations", "partition_delta": 0, "dangling_200_content_hashes": 0, "rows_without_answer_or_error": 0, "rows_with_answer_and_error": 0, "verdict": "PASS", "findings": [], "rule": "Any mismatch is published as a finding; the row predicates are never adjusted to make counts reconcile." }, "definitions": { "http_answer": "Any non-null HTTP status is an observation, including 404.", "transport_error": "A null HTTP status with a non-null fetch_error observed no HTTP answer and is not an observation.", "retained_body": "A retained body count requires an HTTP 200 row whose content_sha256 exists in blobs." } }, "transport_error_bias": { "publication_cutoff": "2026-08-11", "definition": "A domain-date fails under the published transport-error row predicate; persistent means at least 53 of 58 sealed dates (90% threshold).", "row_predicates": { "domain_first": "policy_snapshots.snapshot_date <= :publication_cutoff; group by domain; count distinct snapshot_date where status_code IS NULL AND fetch_error IS NOT NULL", "date_first": "policy_snapshots.snapshot_date <= :publication_cutoff AND status_code IS NULL AND fetch_error IS NOT NULL; distinct snapshot_date, domain; then group by domain" }, "snapshot_dates": 58, "persistence_threshold_pct": 90, "biased_domain_share_threshold_pct": 5, "minimum_error_dates": 53, "persistent_domains_any_resource": 23102, "panel_domains": 100000, "persistent_domain_share_pct": 23.1, "bias_verdict": "BIASED", "threshold_rule": "BIASED only when more than 5% of frozen-panel domains fail on at least 90% of sealed dates.", "tranco_rank_profile": [ { "tranco_rank_range": "1-1,000", "domains": 1000, "persistent_domains": 309, "persistent_share_pct": 30.9, "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "tranco_rank_range": "1,001-10,000", "domains": 9000, "persistent_domains": 2132, "persistent_share_pct": 23.69, "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "tranco_rank_range": "10,001-25,000", "domains": 15000, "persistent_domains": 3846, "persistent_share_pct": 25.64, "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "tranco_rank_range": "25,001-50,000", "domains": 25000, "persistent_domains": 6551, "persistent_share_pct": 26.2, "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "tranco_rank_range": "50,001-100,000", "domains": 50000, "persistent_domains": 10264, "persistent_share_pct": 20.53, "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." } ], "independent_recomputation": { "verdict": "PASS", "domain_first_count": 23102, "date_first_count": 23102, "disagreements": 0, "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, "absent_domain_control": { "domain": "synthetic-domain-not-in-panel.invalid", "in_universe": false, "persistent_error_dates": null, "persistent_share_pct": null, "verdict": "UNKNOWN", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, "disclosure": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, "comparison": { "snapshot_date": "2026-08-04", "collection_batch_sha256": "96ee475f43bc310458de9cd1db0cd67629c88b2cb47c73f037dff3f44754df23", "robots_rows_sha256": "d60a43a97e65b98da64ad7cb22374fa551a4b603052fdf195835e5ab8e128b6b", "panel_denominator": 100000, "status_classes": { "status_2xx": 62352, "status_4xx": 12899, "status_5xx": 478, "transport_error": 24128, "other_status": 143 }, "unknown_reasons": { "transport_error": 24128, "server_error": 478, "redirect_or_other_status": 143, "missing_body": 724, "truncated_body": 392, "html_soft_200": 4314, "invalid_utf8": 238 }, "crawlers": { "GPTBot": { "token": "GPTBot", "operator": "OpenAI", "DISALLOWED": 10258, "NOT_DISALLOWED_BY_TESTED_RULE": 59325, "UNKNOWN": 30417, "known_denominator": 69583, "panel_denominator": 100000, "disallowed_pct_of_known": 14.7, "disallowed_pct_of_panel": 10.3, "unknown_pct_of_panel": 30.4, "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, "ClaudeBot": { "token": "ClaudeBot", "operator": "Anthropic", "DISALLOWED": 9374, "NOT_DISALLOWED_BY_TESTED_RULE": 60209, "UNKNOWN": 30417, "known_denominator": 69583, "panel_denominator": 100000, "disallowed_pct_of_known": 13.5, "disallowed_pct_of_panel": 9.4, "unknown_pct_of_panel": 30.4, "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, "CCBot": { "token": "CCBot", "operator": "Common Crawl", "DISALLOWED": 10160, "NOT_DISALLOWED_BY_TESTED_RULE": 59423, "UNKNOWN": 30417, "known_denominator": 69583, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.4, "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, "Google-Extended": { "token": "Google-Extended", "operator": "Google", "DISALLOWED": 8746, "NOT_DISALLOWED_BY_TESTED_RULE": 60837, "UNKNOWN": 30417, "known_denominator": 69583, "panel_denominator": 100000, "disallowed_pct_of_known": 12.6, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.4, "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, "PerplexityBot": { "token": "PerplexityBot", "operator": "Perplexity", "DISALLOWED": 4194, "NOT_DISALLOWED_BY_TESTED_RULE": 65389, "UNKNOWN": 30417, "known_denominator": 69583, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.4, "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." } } }, "latest": { "snapshot_date": "2026-08-11", "collection_batch_sha256": "81b6ee750b6b1d913a77d3a57fed364d5215eeae857fac04f8e0ee00fa475887", "robots_rows_sha256": "718cca7fa413510cfff33147c93f56982d677d0164cafda77527bf5ba3fa68e7", "panel_denominator": 100000, "status_classes": { "status_2xx": 62158, "status_4xx": 12863, "status_5xx": 464, "transport_error": 24376, "other_status": 139 }, "unknown_reasons": { "transport_error": 24376, "server_error": 464, "redirect_or_other_status": 139, "missing_body": 701, "truncated_body": 400, "html_soft_200": 4443, "invalid_utf8": 235 }, "crawlers": { "GPTBot": { "token": "GPTBot", "operator": "OpenAI", "DISALLOWED": 10214, "NOT_DISALLOWED_BY_TESTED_RULE": 59028, "UNKNOWN": 30758, "known_denominator": 69242, "panel_denominator": 100000, "disallowed_pct_of_known": 14.8, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.8, "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, "ClaudeBot": { "token": "ClaudeBot", "operator": "Anthropic", "DISALLOWED": 9342, "NOT_DISALLOWED_BY_TESTED_RULE": 59900, "UNKNOWN": 30758, "known_denominator": 69242, "panel_denominator": 100000, "disallowed_pct_of_known": 13.5, "disallowed_pct_of_panel": 9.3, "unknown_pct_of_panel": 30.8, "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, "CCBot": { "token": "CCBot", "operator": "Common Crawl", "DISALLOWED": 10127, "NOT_DISALLOWED_BY_TESTED_RULE": 59115, "UNKNOWN": 30758, "known_denominator": 69242, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.8, "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, "Google-Extended": { "token": "Google-Extended", "operator": "Google", "DISALLOWED": 8729, "NOT_DISALLOWED_BY_TESTED_RULE": 60513, "UNKNOWN": 30758, "known_denominator": 69242, "panel_denominator": 100000, "disallowed_pct_of_known": 12.6, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.8, "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, "PerplexityBot": { "token": "PerplexityBot", "operator": "Perplexity", "DISALLOWED": 4140, "NOT_DISALLOWED_BY_TESTED_RULE": 65102, "UNKNOWN": 30758, "known_denominator": 69242, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.1, "unknown_pct_of_panel": 30.8, "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." } } }, "facts": { "honestyStatement": "Every published count is re-derived from the append-only closing_web.db seal. Nothing is estimated, modeled, smoothed, or extrapolated. UNKNOWN is retained as a first-class result.", "privacyStatement": "The public artifacts contain aggregate counts and hashes only; robots.txt bodies, headers, domain identities, and per-domain verdicts are not serialized.", "denominatorStatement": "Known-policy rates divide DISALLOWED by DISALLOWED plus NOT_DISALLOWED_BY_TESTED_RULE. Panel shares divide by the full frozen panel. UNKNOWN counts and shares travel beside both.", "observationStatement": "Stored grid cells are not observations. HTTP answers, including 404, are observations; transport errors are not.", "denominatorBiasStatement": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel.", "standardStatement": "Group and rule selection follows the frozen root-path interpretation of RFC 9309 described in scope." }, "displays": [ { "key": "coverage.firstDate", "display": "June 9, 2026", "value": null, "kind": "window" }, { "key": "coverage.latestDate", "display": "August 11, 2026", "value": null, "kind": "window" }, { "key": "coverage.compareDate", "display": "August 4, 2026", "value": null, "kind": "window" }, { "key": "coverage.observationCutoffIso", "display": "2026-08-11", "value": null, "kind": "window" }, { "key": "coverage.completeDates", "display": "58", "value": 58, "kind": "count" }, { "key": "coverage.gapDates", "display": "6", "value": 6, "kind": "count" }, { "key": "coverage.gapDate.1", "display": "June 11, 2026", "value": null, "kind": "window" }, { "key": "coverage.gapDate.2", "display": "June 12, 2026", "value": null, "kind": "window" }, { "key": "coverage.gapDate.3", "display": "June 19, 2026", "value": null, "kind": "window" }, { "key": "coverage.gapDate.4", "display": "July 4, 2026", "value": null, "kind": "window" }, { "key": "coverage.gapDate.5", "display": "July 15, 2026", "value": null, "kind": "window" }, { "key": "coverage.gapDate.6", "display": "August 8, 2026", "value": null, "kind": "window" }, { "key": "coverage.panel", "display": "100,000", "value": 100000, "kind": "count" }, { "key": "storedCellAccounting.stored_grid_cells", "display": "23,200,000", "value": 23200000, "kind": "count" }, { "key": "storedCellAccounting.transport_errors", "display": "5,642,987", "value": 5642987, "kind": "count" }, { "key": "storedCellAccounting.answered_observations", "display": "17,557,013", "value": 17557013, "kind": "count" }, { "key": "storedCellAccounting.retained_200_bodies", "display": "5,934,853", "value": 5934853, "kind": "count" }, { "key": "storedCellAccounting.distinct_retained_blobs", "display": "1,020,697", "value": 1020697, "kind": "count" }, { "key": "storedCellAccounting.predicateLiteral.httpStatus", "display": "200", "value": 200, "kind": "predicate_literal" }, { "key": "storedCellAccounting.predicateLiteral.exists", "display": "1", "value": 1, "kind": "predicate_literal" }, { "key": "transportBias.snapshotDates", "display": "58", "value": 58, "kind": "count" }, { "key": "transportBias.persistenceThresholdPct", "display": "90%", "value": 90, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "transportBias.biasedDomainShareThresholdPct", "display": "5%", "value": 5, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "transportBias.minimumErrorDates", "display": "53", "value": 53, "kind": "count" }, { "key": "transportBias.persistentDomains", "display": "23,102", "value": 23102, "kind": "count" }, { "key": "transportBias.panelDomains", "display": "100,000", "value": 100000, "kind": "count" }, { "key": "transportBias.persistentDomainSharePct", "display": "23.1%", "value": 23.1, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "transportBias.rank.1.range", "display": "1-1,000", "value": null, "kind": "window" }, { "key": "transportBias.rank.1.domains", "display": "1,000", "value": 1000, "kind": "count" }, { "key": "transportBias.rank.1.persistentDomains", "display": "309", "value": 309, "kind": "count" }, { "key": "transportBias.rank.1.persistentSharePct", "display": "30.9%", "value": 30.9, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "transportBias.rank.2.range", "display": "1,001-10,000", "value": null, "kind": "window" }, { "key": "transportBias.rank.2.domains", "display": "9,000", "value": 9000, "kind": "count" }, { "key": "transportBias.rank.2.persistentDomains", "display": "2,132", "value": 2132, "kind": "count" }, { "key": "transportBias.rank.2.persistentSharePct", "display": "23.69%", "value": 23.69, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "transportBias.rank.3.range", "display": "10,001-25,000", "value": null, "kind": "window" }, { "key": "transportBias.rank.3.domains", "display": "15,000", "value": 15000, "kind": "count" }, { "key": "transportBias.rank.3.persistentDomains", "display": "3,846", "value": 3846, "kind": "count" }, { "key": "transportBias.rank.3.persistentSharePct", "display": "25.64%", "value": 25.64, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "transportBias.rank.4.range", "display": "25,001-50,000", "value": null, "kind": "window" }, { "key": "transportBias.rank.4.domains", "display": "25,000", "value": 25000, "kind": "count" }, { "key": "transportBias.rank.4.persistentDomains", "display": "6,551", "value": 6551, "kind": "count" }, { "key": "transportBias.rank.4.persistentSharePct", "display": "26.2%", "value": 26.2, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "transportBias.rank.5.range", "display": "50,001-100,000", "value": null, "kind": "window" }, { "key": "transportBias.rank.5.domains", "display": "50,000", "value": 50000, "kind": "count" }, { "key": "transportBias.rank.5.persistentDomains", "display": "10,264", "value": 10264, "kind": "count" }, { "key": "transportBias.rank.5.persistentSharePct", "display": "20.53%", "value": 20.53, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "comparison.GPTBot.DISALLOWED", "display": "10,258", "value": 10258, "kind": "count" }, { "key": "comparison.GPTBot.NOT_DISALLOWED_BY_TESTED_RULE", "display": "59,325", "value": 59325, "kind": "count" }, { "key": "comparison.GPTBot.UNKNOWN", "display": "30,417", "value": 30417, "kind": "count" }, { "key": "comparison.GPTBot.known_denominator", "display": "69,583", "value": 69583, "kind": "count" }, { "key": "comparison.GPTBot.panel_denominator", "display": "100,000", "value": 100000, "kind": "count" }, { "key": "comparison.GPTBot.disallowed_pct_of_known", "display": "14.7%", "value": 14.7, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "comparison.GPTBot.disallowed_pct_of_panel", "display": "10.3%", "value": 10.3, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "comparison.GPTBot.unknown_pct_of_panel", "display": "30.4%", "value": 30.4, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "comparison.ClaudeBot.DISALLOWED", "display": "9,374", "value": 9374, "kind": "count" }, { "key": "comparison.ClaudeBot.NOT_DISALLOWED_BY_TESTED_RULE", "display": "60,209", "value": 60209, "kind": "count" }, { "key": "comparison.ClaudeBot.UNKNOWN", "display": "30,417", "value": 30417, "kind": "count" }, { "key": "comparison.ClaudeBot.known_denominator", "display": "69,583", "value": 69583, "kind": "count" }, { "key": "comparison.ClaudeBot.panel_denominator", "display": "100,000", "value": 100000, "kind": "count" }, { "key": "comparison.ClaudeBot.disallowed_pct_of_known", "display": "13.5%", "value": 13.5, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "comparison.ClaudeBot.disallowed_pct_of_panel", "display": "9.4%", "value": 9.4, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "comparison.ClaudeBot.unknown_pct_of_panel", "display": "30.4%", "value": 30.4, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "comparison.CCBot.DISALLOWED", "display": "10,160", "value": 10160, "kind": "count" }, { "key": "comparison.CCBot.NOT_DISALLOWED_BY_TESTED_RULE", "display": "59,423", "value": 59423, "kind": "count" }, { "key": "comparison.CCBot.UNKNOWN", "display": "30,417", "value": 30417, "kind": "count" }, { "key": "comparison.CCBot.known_denominator", "display": "69,583", "value": 69583, "kind": "count" }, { "key": "comparison.CCBot.panel_denominator", "display": "100,000", "value": 100000, "kind": "count" }, { "key": "comparison.CCBot.disallowed_pct_of_known", "display": "14.6%", "value": 14.6, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "comparison.CCBot.disallowed_pct_of_panel", "display": "10.2%", "value": 10.2, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "comparison.CCBot.unknown_pct_of_panel", "display": "30.4%", "value": 30.4, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "comparison.Google-Extended.DISALLOWED", "display": "8,746", "value": 8746, "kind": "count" }, { "key": "comparison.Google-Extended.NOT_DISALLOWED_BY_TESTED_RULE", "display": "60,837", "value": 60837, "kind": "count" }, { "key": "comparison.Google-Extended.UNKNOWN", "display": "30,417", "value": 30417, "kind": "count" }, { "key": "comparison.Google-Extended.known_denominator", "display": "69,583", "value": 69583, "kind": "count" }, { "key": "comparison.Google-Extended.panel_denominator", "display": "100,000", "value": 100000, "kind": "count" }, { "key": "comparison.Google-Extended.disallowed_pct_of_known", "display": "12.6%", "value": 12.6, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "comparison.Google-Extended.disallowed_pct_of_panel", "display": "8.7%", "value": 8.7, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "comparison.Google-Extended.unknown_pct_of_panel", "display": "30.4%", "value": 30.4, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "comparison.PerplexityBot.DISALLOWED", "display": "4,194", "value": 4194, "kind": "count" }, { "key": "comparison.PerplexityBot.NOT_DISALLOWED_BY_TESTED_RULE", "display": "65,389", "value": 65389, "kind": "count" }, { "key": "comparison.PerplexityBot.UNKNOWN", "display": "30,417", "value": 30417, "kind": "count" }, { "key": "comparison.PerplexityBot.known_denominator", "display": "69,583", "value": 69583, "kind": "count" }, { "key": "comparison.PerplexityBot.panel_denominator", "display": "100,000", "value": 100000, "kind": "count" }, { "key": "comparison.PerplexityBot.disallowed_pct_of_known", "display": "6%", "value": 6, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "comparison.PerplexityBot.disallowed_pct_of_panel", "display": "4.2%", "value": 4.2, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "comparison.PerplexityBot.unknown_pct_of_panel", "display": "30.4%", "value": 30.4, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "latest.GPTBot.DISALLOWED", "display": "10,214", "value": 10214, "kind": "count" }, { "key": "latest.GPTBot.NOT_DISALLOWED_BY_TESTED_RULE", "display": "59,028", "value": 59028, "kind": "count" }, { "key": "latest.GPTBot.UNKNOWN", "display": "30,758", "value": 30758, "kind": "count" }, { "key": "latest.GPTBot.known_denominator", "display": "69,242", "value": 69242, "kind": "count" }, { "key": "latest.GPTBot.panel_denominator", "display": "100,000", "value": 100000, "kind": "count" }, { "key": "latest.GPTBot.disallowed_pct_of_known", "display": "14.8%", "value": 14.8, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "latest.GPTBot.disallowed_pct_of_panel", "display": "10.2%", "value": 10.2, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "latest.GPTBot.unknown_pct_of_panel", "display": "30.8%", "value": 30.8, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "latest.ClaudeBot.DISALLOWED", "display": "9,342", "value": 9342, "kind": "count" }, { "key": "latest.ClaudeBot.NOT_DISALLOWED_BY_TESTED_RULE", "display": "59,900", "value": 59900, "kind": "count" }, { "key": "latest.ClaudeBot.UNKNOWN", "display": "30,758", "value": 30758, "kind": "count" }, { "key": "latest.ClaudeBot.known_denominator", "display": "69,242", "value": 69242, "kind": "count" }, { "key": "latest.ClaudeBot.panel_denominator", "display": "100,000", "value": 100000, "kind": "count" }, { "key": "latest.ClaudeBot.disallowed_pct_of_known", "display": "13.5%", "value": 13.5, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "latest.ClaudeBot.disallowed_pct_of_panel", "display": "9.3%", "value": 9.3, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "latest.ClaudeBot.unknown_pct_of_panel", "display": "30.8%", "value": 30.8, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "latest.CCBot.DISALLOWED", "display": "10,127", "value": 10127, "kind": "count" }, { "key": "latest.CCBot.NOT_DISALLOWED_BY_TESTED_RULE", "display": "59,115", "value": 59115, "kind": "count" }, { "key": "latest.CCBot.UNKNOWN", "display": "30,758", "value": 30758, "kind": "count" }, { "key": "latest.CCBot.known_denominator", "display": "69,242", "value": 69242, "kind": "count" }, { "key": "latest.CCBot.panel_denominator", "display": "100,000", "value": 100000, "kind": "count" }, { "key": "latest.CCBot.disallowed_pct_of_known", "display": "14.6%", "value": 14.6, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "latest.CCBot.disallowed_pct_of_panel", "display": "10.1%", "value": 10.1, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "latest.CCBot.unknown_pct_of_panel", "display": "30.8%", "value": 30.8, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "latest.Google-Extended.DISALLOWED", "display": "8,729", "value": 8729, "kind": "count" }, { "key": "latest.Google-Extended.NOT_DISALLOWED_BY_TESTED_RULE", "display": "60,513", "value": 60513, "kind": "count" }, { "key": "latest.Google-Extended.UNKNOWN", "display": "30,758", "value": 30758, "kind": "count" }, { "key": "latest.Google-Extended.known_denominator", "display": "69,242", "value": 69242, "kind": "count" }, { "key": "latest.Google-Extended.panel_denominator", "display": "100,000", "value": 100000, "kind": "count" }, { "key": "latest.Google-Extended.disallowed_pct_of_known", "display": "12.6%", "value": 12.6, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "latest.Google-Extended.disallowed_pct_of_panel", "display": "8.7%", "value": 8.7, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "latest.Google-Extended.unknown_pct_of_panel", "display": "30.8%", "value": 30.8, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "latest.PerplexityBot.DISALLOWED", "display": "4,140", "value": 4140, "kind": "count" }, { "key": "latest.PerplexityBot.NOT_DISALLOWED_BY_TESTED_RULE", "display": "65,102", "value": 65102, "kind": "count" }, { "key": "latest.PerplexityBot.UNKNOWN", "display": "30,758", "value": 30758, "kind": "count" }, { "key": "latest.PerplexityBot.known_denominator", "display": "69,242", "value": 69242, "kind": "count" }, { "key": "latest.PerplexityBot.panel_denominator", "display": "100,000", "value": 100000, "kind": "count" }, { "key": "latest.PerplexityBot.disallowed_pct_of_known", "display": "6%", "value": 6, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "latest.PerplexityBot.disallowed_pct_of_panel", "display": "4.1%", "value": 4.1, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "latest.PerplexityBot.unknown_pct_of_panel", "display": "30.8%", "value": 30.8, "kind": "rate", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "weekly.GPTBot.known_rate_delta_pp", "display": "+0.1", "value": 0.1, "kind": "delta", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "weekly.ClaudeBot.known_rate_delta_pp", "display": "0", "value": 0, "kind": "delta", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "weekly.CCBot.known_rate_delta_pp", "display": "0", "value": 0, "kind": "delta", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "weekly.Google-Extended.known_rate_delta_pp", "display": "0", "value": 0, "kind": "delta", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "weekly.PerplexityBot.known_rate_delta_pp", "display": "0", "value": 0, "kind": "delta", "denominator_bias": "Transport-error missingness is systematic: 23,102 of 100,000 domains had a transport error on at least 53 of 58 sealed dates. Classified-policy rates describe the answered subpopulation; panel and UNKNOWN shares use the full frozen panel." }, { "key": "latest.unknown_reason.transport_error", "display": "24,376", "value": 24376, "kind": "count" }, { "key": "latest.unknown_reason.server_error", "display": "464", "value": 464, "kind": "count" }, { "key": "latest.unknown_reason.redirect_or_other_status", "display": "139", "value": 139, "kind": "count" }, { "key": "latest.unknown_reason.missing_body", "display": "701", "value": 701, "kind": "count" }, { "key": "latest.unknown_reason.truncated_body", "display": "400", "value": 400, "kind": "count" }, { "key": "latest.unknown_reason.html_soft_200", "display": "4,443", "value": 4443, "kind": "count" }, { "key": "latest.unknown_reason.invalid_utf8", "display": "235", "value": 235, "kind": "count" } ], "sha256": "5ba00c28b989c6da9883d58a11770fbad7a5a435edb5f8c0b4ebd8d2729835e2" }