{ "schema": "usta.closing-web-crawler-scoreboard.v1", "blogId": "D0395", "slug": "business-sites-ai-crawler-access-benchmark-2026", "title": "58-Date AI Crawler Robots Policy Scoreboard", "editionId": "2026-08", "sealedAt": "2026-08-11", "canonicalUrl": "https://ustechautomations.com/resources/blog/business-sites-ai-crawler-access-benchmark-2026", "methodologyUrl": "https://www.rfc-editor.org/rfc/rfc9309.html", "sponsoredBenchmarkRequestUrl": "https://ustechautomations.com/permits/sponsored-benchmark-brief", "scope": { "panel": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "collection_method": "closing-web-v1", "request_path": "/", "tracked_tokens": [ "GPTBot", "ClaudeBot", "CCBot", "Google-Extended", "PerplexityBot" ], "verdicts": [ "DISALLOWED", "NOT_DISALLOWED_BY_TESTED_RULE", "UNKNOWN" ], "statement": "Each observation tests only whether the applicable stored robots.txt rules disallow one exact product token from requesting /. It does not establish crawler behavior, citations, model use, access authorization, or legal effect.", "status_handling": "A sealed 4xx means no tested rule was available and is NOT_DISALLOWED_BY_TESTED_RULE. Transport failures, 5xx, other statuses, missing or truncated bodies, HTML-shaped responses, and invalid UTF-8 are UNKNOWN.", "material_presence_rule": "A token needs exact case-insensitive User-agent presence in at least 100 stored policies on every included date. Below that floor its rates are UNKNOWN, even if wildcard groups would produce a mechanical verdict.", "sponsor_independence": "The method, panel, dates, token set, ordering rule, and results are frozen independently of any future sponsor. Sponsorship cannot change a score, competitor set, method, or conclusion." }, "coverage": { "first_complete_date": "2026-06-09", "last_complete_date": "2026-08-11", "complete_date_count": 58, "calendar_span_days": 64, "gap_date_count": 6, "gap_dates": [ "2026-06-11", "2026-06-12", "2026-06-19", "2026-07-04", "2026-07-15", "2026-08-08" ], "panel_denominator": 100000, "definition": "A complete date has one sealed row for each of four frozen resources for every frozen-panel domain; counts come from policy_snapshots, not run-log claims." }, "preflight": { "verdict": "PASS", "material_floor_per_complete_date": 100, "required_token_count": 5, "material_token_count": 5, "tokens": { "GPTBot": { "minimum_exact_token_policy_count": 10223, "latest_exact_token_policy_count": 10926, "verdict": "OBSERVED" }, "ClaudeBot": { "minimum_exact_token_policy_count": 8797, "latest_exact_token_policy_count": 9675, "verdict": "OBSERVED" }, "CCBot": { "minimum_exact_token_policy_count": 8757, "latest_exact_token_policy_count": 9400, "verdict": "OBSERVED" }, "Google-Extended": { "minimum_exact_token_policy_count": 8056, "latest_exact_token_policy_count": 8833, "verdict": "OBSERVED" }, "PerplexityBot": { "minimum_exact_token_policy_count": 4215, "latest_exact_token_policy_count": 4615, "verdict": "OBSERVED" } }, "absent_token_control": { "token": "USTAAbsentControlBotZeta", "exact_token_policy_count_across_window": 0, "published_verdict": "UNKNOWN", "numeric_result": null, "statement": "The fabricated product token is absent from the sealed corpus. Its wildcard-derived raw classifications are withheld; absence produces UNKNOWN, never 0%." } }, "source_binding": { "universe_sha256": "8778484edc1487aa038aecb827b456950a6e583f8465a11b6810ba1223c11fd0", "methodology_sha256": "3f43a495546fa1c0027fad32ab69646191418c680dacb9ac2004e805ffbfe180", "method_files": [ { "name": "closing_web/METHODOLOGY.md", "sha256": "3f43a495546fa1c0027fad32ab69646191418c680dacb9ac2004e805ffbfe180" }, { "name": "seo/scripts/closing-web-weekly.js", "sha256": "48589d68c25d0f41f5a2fd0cf62222149e1451dba4f1fe02117aea98c3707d3a" }, { "name": "seo/scripts/lib/closing-web-root-policy.js", "sha256": "092cc808b458d58d28feab6912bc83dd8559a3086103e48c9016eede832d4a63" }, { "name": "seo/scripts/closing-web-scoreboard.js", "sha256": "d38f1d01311d70e45c37ea653f2e46a8f871cf49a92af96f894674917de35bf1" } ], "read_model_method_sha256": "c9f09fbfef7d1989178b85e1c05fa543bb413bc95fb0ca831f692b066b550ea7", "weekly_endpoint_artifact_sha256": "4786bb94f8852139c5e3aea02895aa9d581694a54b7b3b269f7bbdc7d8b37934", "complete_date_batches_sha256": "70b0fd0a027cb10e1b0b1ea1bf4673eb3f16712e3a28ee6e33b484da684a3cd3" }, "integrity": { "complete_batches_verified": 58, "batch_member_hashes_reaggregated": 23200000, "robots_rows_verified": 5800000, "unique_blobs_verified": 196919, "collector_root_complete_dates": 57, "partitioned_dates_verified": 2, "partial_manifest_dates": 1, "unmanifested_rows": 390000, "independent_endpoint_matches": [ "2026-06-10", "2026-08-11" ], "statement": "The scoreboard independently re-derived both endpoint aggregates and matched the weekly read model. Every robots row and referenced blob was checked. Collector roots fully bind 57 dates; June 13 has one verified 10,000-row run and 390,000 append-only rows left by a crashed process without a stored run root, reported as PARTIAL_COLLECTOR_ROOT rather than silently passed." }, "first": { "snapshot_date": "2026-06-09", "collection_batch_sha256": "3e641f37c079416da5e6f67f8a2119076f21df4bcdff36ed4e9cb9d63073850b", "collection_batch_parts": [ { "run_id": "2026-06-09:2026-06-09T19:42:42.000123+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa#limit50", "row_count": 200, "batch_sha256": "b787ef837a424145abfc8dc75d154afcc8e9cefe329c147b44d1904a29ff61fd", "collector_root_status": "VERIFIED" }, { "run_id": "2026-06-09:2026-06-09T19:43:27.803543+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 399800, "batch_sha256": "e37935cfd1e7408e1592f62f729a7ed3c0a36bb60d93c9e34110b1000c16f8c3", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "8ff51c7a80554ca3dc068a493b1da04b4929defe7cbe0a2237a6e18dd6cb8334", "panel_denominator": 100000, "status_classes": { "status_2xx": 62387, "status_4xx": 12773, "status_5xx": 479, "transport_error": 24220, "other_status": 141 }, "unknown_reasons": { "transport_error": 24220, "server_error": 479, "redirect_or_other_status": 141, "missing_body": 728, "truncated_body": 378, "html_soft_200": 4367, "invalid_utf8": 233 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10223, "material_floor": 100, "DISALLOWED": 9877, "NOT_DISALLOWED_BY_TESTED_RULE": 59577, "UNKNOWN": 30546, "known_denominator": 69454, "panel_denominator": 100000, "disallowed_pct_of_known": 14.2, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.5, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8797, "material_floor": 100, "DISALLOWED": 8901, "NOT_DISALLOWED_BY_TESTED_RULE": 60553, "UNKNOWN": 30546, "known_denominator": 69454, "panel_denominator": 100000, "disallowed_pct_of_known": 12.8, "disallowed_pct_of_panel": 8.9, "unknown_pct_of_panel": 30.5, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8757, "material_floor": 100, "DISALLOWED": 9707, "NOT_DISALLOWED_BY_TESTED_RULE": 59747, "UNKNOWN": 30546, "known_denominator": 69454, "panel_denominator": 100000, "disallowed_pct_of_known": 14, "disallowed_pct_of_panel": 9.7, "unknown_pct_of_panel": 30.5, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8056, "material_floor": 100, "DISALLOWED": 8352, "NOT_DISALLOWED_BY_TESTED_RULE": 61102, "UNKNOWN": 30546, "known_denominator": 69454, "panel_denominator": 100000, "disallowed_pct_of_known": 12, "disallowed_pct_of_panel": 8.4, "unknown_pct_of_panel": 30.5, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4215, "material_floor": 100, "DISALLOWED": 4262, "NOT_DISALLOWED_BY_TESTED_RULE": 65192, "UNKNOWN": 30546, "known_denominator": 69454, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.3, "unknown_pct_of_panel": 30.5, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, "latest": { "snapshot_date": "2026-08-11", "collection_batch_sha256": "81b6ee750b6b1d913a77d3a57fed364d5215eeae857fac04f8e0ee00fa475887", "collection_batch_parts": [ { "run_id": "2026-08-11:2026-08-11T10:01:42.131256+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "81b6ee750b6b1d913a77d3a57fed364d5215eeae857fac04f8e0ee00fa475887", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "718cca7fa413510cfff33147c93f56982d677d0164cafda77527bf5ba3fa68e7", "panel_denominator": 100000, "status_classes": { "status_2xx": 62158, "status_4xx": 12863, "status_5xx": 464, "transport_error": 24376, "other_status": 139 }, "unknown_reasons": { "transport_error": 24376, "server_error": 464, "redirect_or_other_status": 139, "missing_body": 701, "truncated_body": 400, "html_soft_200": 4443, "invalid_utf8": 235 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10926, "material_floor": 100, "DISALLOWED": 10214, "NOT_DISALLOWED_BY_TESTED_RULE": 59028, "UNKNOWN": 30758, "known_denominator": 69242, "panel_denominator": 100000, "disallowed_pct_of_known": 14.8, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.8, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9675, "material_floor": 100, "DISALLOWED": 9342, "NOT_DISALLOWED_BY_TESTED_RULE": 59900, "UNKNOWN": 30758, "known_denominator": 69242, "panel_denominator": 100000, "disallowed_pct_of_known": 13.5, "disallowed_pct_of_panel": 9.3, "unknown_pct_of_panel": 30.8, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9400, "material_floor": 100, "DISALLOWED": 10127, "NOT_DISALLOWED_BY_TESTED_RULE": 59115, "UNKNOWN": 30758, "known_denominator": 69242, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.8, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8833, "material_floor": 100, "DISALLOWED": 8729, "NOT_DISALLOWED_BY_TESTED_RULE": 60513, "UNKNOWN": 30758, "known_denominator": 69242, "panel_denominator": 100000, "disallowed_pct_of_known": 12.6, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.8, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4615, "material_floor": 100, "DISALLOWED": 4140, "NOT_DISALLOWED_BY_TESTED_RULE": 65102, "UNKNOWN": 30758, "known_denominator": 69242, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.1, "unknown_pct_of_panel": 30.8, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, "trends": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "first_date": "2026-06-09", "latest_date": "2026-08-11", "complete_date_count": 58, "first_known_rate": 14.2, "latest_known_rate": 14.8, "known_rate_delta_pp": 0.6, "first_panel_share": 9.9, "latest_panel_share": 10.2, "panel_share_delta_pp": 0.3, "minimum_known_rate": 14.2, "minimum_known_rate_date": "2026-06-09", "maximum_known_rate": 14.8, "maximum_known_rate_date": "2026-07-31", "minimum_exact_token_policy_count": 10223, "latest_exact_token_policy_count": 10926 }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "first_date": "2026-06-09", "latest_date": "2026-08-11", "complete_date_count": 58, "first_known_rate": 12.8, "latest_known_rate": 13.5, "known_rate_delta_pp": 0.7, "first_panel_share": 8.9, "latest_panel_share": 9.3, "panel_share_delta_pp": 0.4, "minimum_known_rate": 12.8, "minimum_known_rate_date": "2026-06-09", "maximum_known_rate": 13.5, "maximum_known_rate_date": "2026-07-31", "minimum_exact_token_policy_count": 8797, "latest_exact_token_policy_count": 9675 }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "first_date": "2026-06-09", "latest_date": "2026-08-11", "complete_date_count": 58, "first_known_rate": 14, "latest_known_rate": 14.6, "known_rate_delta_pp": 0.6, "first_panel_share": 9.7, "latest_panel_share": 10.1, "panel_share_delta_pp": 0.4, "minimum_known_rate": 14, "minimum_known_rate_date": "2026-06-09", "maximum_known_rate": 14.6, "maximum_known_rate_date": "2026-07-29", "minimum_exact_token_policy_count": 8757, "latest_exact_token_policy_count": 9400 }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "first_date": "2026-06-09", "latest_date": "2026-08-11", "complete_date_count": 58, "first_known_rate": 12, "latest_known_rate": 12.6, "known_rate_delta_pp": 0.6, "first_panel_share": 8.4, "latest_panel_share": 8.7, "panel_share_delta_pp": 0.3, "minimum_known_rate": 12, "minimum_known_rate_date": "2026-06-09", "maximum_known_rate": 12.6, "maximum_known_rate_date": "2026-07-31", "minimum_exact_token_policy_count": 8056, "latest_exact_token_policy_count": 8833 }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "first_date": "2026-06-09", "latest_date": "2026-08-11", "complete_date_count": 58, "first_known_rate": 6.1, "latest_known_rate": 6, "known_rate_delta_pp": -0.1, "first_panel_share": 4.3, "latest_panel_share": 4.1, "panel_share_delta_pp": -0.2, "minimum_known_rate": 6, "minimum_known_rate_date": "2026-06-20", "maximum_known_rate": 6.2, "maximum_known_rate_date": "2026-06-14", "minimum_exact_token_policy_count": 4215, "latest_exact_token_policy_count": 4615 } }, "series": [ { "snapshot_date": "2026-06-09", "collection_batch_sha256": "3e641f37c079416da5e6f67f8a2119076f21df4bcdff36ed4e9cb9d63073850b", "collection_batch_parts": [ { "run_id": "2026-06-09:2026-06-09T19:42:42.000123+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa#limit50", "row_count": 200, "batch_sha256": "b787ef837a424145abfc8dc75d154afcc8e9cefe329c147b44d1904a29ff61fd", "collector_root_status": "VERIFIED" }, { "run_id": "2026-06-09:2026-06-09T19:43:27.803543+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 399800, "batch_sha256": "e37935cfd1e7408e1592f62f729a7ed3c0a36bb60d93c9e34110b1000c16f8c3", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "8ff51c7a80554ca3dc068a493b1da04b4929defe7cbe0a2237a6e18dd6cb8334", "panel_denominator": 100000, "status_classes": { "status_2xx": 62387, "status_4xx": 12773, "status_5xx": 479, "transport_error": 24220, "other_status": 141 }, "unknown_reasons": { "transport_error": 24220, "server_error": 479, "redirect_or_other_status": 141, "missing_body": 728, "truncated_body": 378, "html_soft_200": 4367, "invalid_utf8": 233 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10223, "material_floor": 100, "DISALLOWED": 9877, "NOT_DISALLOWED_BY_TESTED_RULE": 59577, "UNKNOWN": 30546, "known_denominator": 69454, "panel_denominator": 100000, "disallowed_pct_of_known": 14.2, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.5, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8797, "material_floor": 100, "DISALLOWED": 8901, "NOT_DISALLOWED_BY_TESTED_RULE": 60553, "UNKNOWN": 30546, "known_denominator": 69454, "panel_denominator": 100000, "disallowed_pct_of_known": 12.8, "disallowed_pct_of_panel": 8.9, "unknown_pct_of_panel": 30.5, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8757, "material_floor": 100, "DISALLOWED": 9707, "NOT_DISALLOWED_BY_TESTED_RULE": 59747, "UNKNOWN": 30546, "known_denominator": 69454, "panel_denominator": 100000, "disallowed_pct_of_known": 14, "disallowed_pct_of_panel": 9.7, "unknown_pct_of_panel": 30.5, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8056, "material_floor": 100, "DISALLOWED": 8352, "NOT_DISALLOWED_BY_TESTED_RULE": 61102, "UNKNOWN": 30546, "known_denominator": 69454, "panel_denominator": 100000, "disallowed_pct_of_known": 12, "disallowed_pct_of_panel": 8.4, "unknown_pct_of_panel": 30.5, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4215, "material_floor": 100, "DISALLOWED": 4262, "NOT_DISALLOWED_BY_TESTED_RULE": 65192, "UNKNOWN": 30546, "known_denominator": 69454, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.3, "unknown_pct_of_panel": 30.5, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-06-10", "collection_batch_sha256": "aebdc52f9adaf495d5e26063ce9c240f2568d9444dfa0f8e70a08dff9c6d389b", "collection_batch_parts": [ { "run_id": "2026-06-10:2026-06-10T10:08:06.940900+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "aebdc52f9adaf495d5e26063ce9c240f2568d9444dfa0f8e70a08dff9c6d389b", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "c6fadada045243823f98790a2bd91e89bb66ae802b491de712b73a9ebf32a09a", "panel_denominator": 100000, "status_classes": { "status_2xx": 62366, "status_4xx": 12783, "status_5xx": 478, "transport_error": 24232, "other_status": 141 }, "unknown_reasons": { "transport_error": 24232, "server_error": 478, "redirect_or_other_status": 141, "missing_body": 727, "truncated_body": 378, "html_soft_200": 4455, "invalid_utf8": 224 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10240, "material_floor": 100, "DISALLOWED": 9884, "NOT_DISALLOWED_BY_TESTED_RULE": 59481, "UNKNOWN": 30635, "known_denominator": 69365, "panel_denominator": 100000, "disallowed_pct_of_known": 14.2, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.6, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8815, "material_floor": 100, "DISALLOWED": 8906, "NOT_DISALLOWED_BY_TESTED_RULE": 60459, "UNKNOWN": 30635, "known_denominator": 69365, "panel_denominator": 100000, "disallowed_pct_of_known": 12.8, "disallowed_pct_of_panel": 8.9, "unknown_pct_of_panel": 30.6, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8762, "material_floor": 100, "DISALLOWED": 9709, "NOT_DISALLOWED_BY_TESTED_RULE": 59656, "UNKNOWN": 30635, "known_denominator": 69365, "panel_denominator": 100000, "disallowed_pct_of_known": 14, "disallowed_pct_of_panel": 9.7, "unknown_pct_of_panel": 30.6, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8070, "material_floor": 100, "DISALLOWED": 8354, "NOT_DISALLOWED_BY_TESTED_RULE": 61011, "UNKNOWN": 30635, "known_denominator": 69365, "panel_denominator": 100000, "disallowed_pct_of_known": 12, "disallowed_pct_of_panel": 8.4, "unknown_pct_of_panel": 30.6, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4218, "material_floor": 100, "DISALLOWED": 4259, "NOT_DISALLOWED_BY_TESTED_RULE": 65106, "UNKNOWN": 30635, "known_denominator": 69365, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.3, "unknown_pct_of_panel": 30.6, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-06-13", "collection_batch_sha256": "558e57a34de6e5a5be959964f40c26d7f6fd9169b27ecfb246d27c4a56a25970", "collection_batch_parts": [ { "run_id": null, "universe_version": null, "row_count": 390000, "batch_sha256": "5f684581733244727a23a02f06765e39d31accbb630f8047ffaf03caa85a973b", "collector_root_status": "MISSING" }, { "run_id": "2026-06-13:2026-06-13T13:03:06.417890+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 10000, "batch_sha256": "4f9cf77cb4cd8a7ce07c8d1d7babf2febf6d1d2a768c864b15d1bbe5dd266916", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "PARTIAL_COLLECTOR_ROOT", "unmanifested_row_count": 390000, "robots_rows_sha256": "954611ea71efaaaca7ae993be5069064d3aa7a50fad01e6b83d790c32c0bf890", "panel_denominator": 100000, "status_classes": { "status_2xx": 62381, "status_4xx": 12789, "status_5xx": 477, "transport_error": 24213, "other_status": 140 }, "unknown_reasons": { "transport_error": 24213, "server_error": 477, "redirect_or_other_status": 140, "missing_body": 722, "truncated_body": 385, "html_soft_200": 4321, "invalid_utf8": 229 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10328, "material_floor": 100, "DISALLOWED": 9948, "NOT_DISALLOWED_BY_TESTED_RULE": 59565, "UNKNOWN": 30487, "known_denominator": 69513, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.5, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8900, "material_floor": 100, "DISALLOWED": 8971, "NOT_DISALLOWED_BY_TESTED_RULE": 60542, "UNKNOWN": 30487, "known_denominator": 69513, "panel_denominator": 100000, "disallowed_pct_of_known": 12.9, "disallowed_pct_of_panel": 9, "unknown_pct_of_panel": 30.5, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8846, "material_floor": 100, "DISALLOWED": 9776, "NOT_DISALLOWED_BY_TESTED_RULE": 59737, "UNKNOWN": 30487, "known_denominator": 69513, "panel_denominator": 100000, "disallowed_pct_of_known": 14.1, "disallowed_pct_of_panel": 9.8, "unknown_pct_of_panel": 30.5, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8146, "material_floor": 100, "DISALLOWED": 8407, "NOT_DISALLOWED_BY_TESTED_RULE": 61106, "UNKNOWN": 30487, "known_denominator": 69513, "panel_denominator": 100000, "disallowed_pct_of_known": 12.1, "disallowed_pct_of_panel": 8.4, "unknown_pct_of_panel": 30.5, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4251, "material_floor": 100, "DISALLOWED": 4268, "NOT_DISALLOWED_BY_TESTED_RULE": 65245, "UNKNOWN": 30487, "known_denominator": 69513, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.3, "unknown_pct_of_panel": 30.5, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-06-14", "collection_batch_sha256": "268c18fb066678b7695916dee71424c2c57ceee31d0668abb696a8ac7fa49ddd", "collection_batch_parts": [ { "run_id": "2026-06-14:2026-06-14T10:03:50.214776+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "268c18fb066678b7695916dee71424c2c57ceee31d0668abb696a8ac7fa49ddd", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "7685ddb24f49e9db20a43743e50f3801c5c1ffd2a4ede78202bceee1706357c4", "panel_denominator": 100000, "status_classes": { "status_2xx": 62431, "status_4xx": 12778, "status_5xx": 470, "transport_error": 24181, "other_status": 140 }, "unknown_reasons": { "transport_error": 24181, "server_error": 470, "redirect_or_other_status": 140, "missing_body": 724, "truncated_body": 379, "html_soft_200": 4330, "invalid_utf8": 232 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10350, "material_floor": 100, "DISALLOWED": 9974, "NOT_DISALLOWED_BY_TESTED_RULE": 59570, "UNKNOWN": 30456, "known_denominator": 69544, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 10, "unknown_pct_of_panel": 30.5, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8921, "material_floor": 100, "DISALLOWED": 8996, "NOT_DISALLOWED_BY_TESTED_RULE": 60548, "UNKNOWN": 30456, "known_denominator": 69544, "panel_denominator": 100000, "disallowed_pct_of_known": 12.9, "disallowed_pct_of_panel": 9, "unknown_pct_of_panel": 30.5, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8865, "material_floor": 100, "DISALLOWED": 9802, "NOT_DISALLOWED_BY_TESTED_RULE": 59742, "UNKNOWN": 30456, "known_denominator": 69544, "panel_denominator": 100000, "disallowed_pct_of_known": 14.1, "disallowed_pct_of_panel": 9.8, "unknown_pct_of_panel": 30.5, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8166, "material_floor": 100, "DISALLOWED": 8435, "NOT_DISALLOWED_BY_TESTED_RULE": 61109, "UNKNOWN": 30456, "known_denominator": 69544, "panel_denominator": 100000, "disallowed_pct_of_known": 12.1, "disallowed_pct_of_panel": 8.4, "unknown_pct_of_panel": 30.5, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4257, "material_floor": 100, "DISALLOWED": 4278, "NOT_DISALLOWED_BY_TESTED_RULE": 65266, "UNKNOWN": 30456, "known_denominator": 69544, "panel_denominator": 100000, "disallowed_pct_of_known": 6.2, "disallowed_pct_of_panel": 4.3, "unknown_pct_of_panel": 30.5, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-06-15", "collection_batch_sha256": "317d9f4293f13a215d51416198e8fad632e4fbd568bb759572c519a0d525dcd5", "collection_batch_parts": [ { "run_id": "2026-06-15:2026-06-15T10:09:53.968235+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "317d9f4293f13a215d51416198e8fad632e4fbd568bb759572c519a0d525dcd5", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "6c102be7c826a48c52f1828584b4406e3172103c9eafb0a8eae559931ee8b9e9", "panel_denominator": 100000, "status_classes": { "status_2xx": 62452, "status_4xx": 12828, "status_5xx": 477, "transport_error": 24102, "other_status": 141 }, "unknown_reasons": { "transport_error": 24102, "server_error": 477, "redirect_or_other_status": 141, "missing_body": 727, "truncated_body": 377, "html_soft_200": 4334, "invalid_utf8": 235 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10346, "material_floor": 100, "DISALLOWED": 9982, "NOT_DISALLOWED_BY_TESTED_RULE": 59625, "UNKNOWN": 30393, "known_denominator": 69607, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 10, "unknown_pct_of_panel": 30.4, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8932, "material_floor": 100, "DISALLOWED": 9010, "NOT_DISALLOWED_BY_TESTED_RULE": 60597, "UNKNOWN": 30393, "known_denominator": 69607, "panel_denominator": 100000, "disallowed_pct_of_known": 12.9, "disallowed_pct_of_panel": 9, "unknown_pct_of_panel": 30.4, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8876, "material_floor": 100, "DISALLOWED": 9816, "NOT_DISALLOWED_BY_TESTED_RULE": 59791, "UNKNOWN": 30393, "known_denominator": 69607, "panel_denominator": 100000, "disallowed_pct_of_known": 14.1, "disallowed_pct_of_panel": 9.8, "unknown_pct_of_panel": 30.4, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8179, "material_floor": 100, "DISALLOWED": 8449, "NOT_DISALLOWED_BY_TESTED_RULE": 61158, "UNKNOWN": 30393, "known_denominator": 69607, "panel_denominator": 100000, "disallowed_pct_of_known": 12.1, "disallowed_pct_of_panel": 8.4, "unknown_pct_of_panel": 30.4, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4250, "material_floor": 100, "DISALLOWED": 4278, "NOT_DISALLOWED_BY_TESTED_RULE": 65329, "UNKNOWN": 30393, "known_denominator": 69607, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.3, "unknown_pct_of_panel": 30.4, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-06-16", "collection_batch_sha256": "b878cdfccb0e3647d6a5dc0cb44a0ffabb0a6403b0f86c601cf18eb6632c23d2", "collection_batch_parts": [ { "run_id": "2026-06-16:2026-06-16T10:00:29.969310+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "b878cdfccb0e3647d6a5dc0cb44a0ffabb0a6403b0f86c601cf18eb6632c23d2", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "ea33919faab95641c21dd0c57119b5504722c2658a021396f9b0eb6e753a9827", "panel_denominator": 100000, "status_classes": { "status_2xx": 62386, "status_4xx": 12872, "status_5xx": 486, "transport_error": 24116, "other_status": 140 }, "unknown_reasons": { "transport_error": 24116, "server_error": 486, "redirect_or_other_status": 140, "missing_body": 732, "truncated_body": 376, "html_soft_200": 4333, "invalid_utf8": 234 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10364, "material_floor": 100, "DISALLOWED": 9983, "NOT_DISALLOWED_BY_TESTED_RULE": 59600, "UNKNOWN": 30417, "known_denominator": 69583, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 10, "unknown_pct_of_panel": 30.4, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8941, "material_floor": 100, "DISALLOWED": 9007, "NOT_DISALLOWED_BY_TESTED_RULE": 60576, "UNKNOWN": 30417, "known_denominator": 69583, "panel_denominator": 100000, "disallowed_pct_of_known": 12.9, "disallowed_pct_of_panel": 9, "unknown_pct_of_panel": 30.4, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8879, "material_floor": 100, "DISALLOWED": 9814, "NOT_DISALLOWED_BY_TESTED_RULE": 59769, "UNKNOWN": 30417, "known_denominator": 69583, "panel_denominator": 100000, "disallowed_pct_of_known": 14.1, "disallowed_pct_of_panel": 9.8, "unknown_pct_of_panel": 30.4, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8200, "material_floor": 100, "DISALLOWED": 8449, "NOT_DISALLOWED_BY_TESTED_RULE": 61134, "UNKNOWN": 30417, "known_denominator": 69583, "panel_denominator": 100000, "disallowed_pct_of_known": 12.1, "disallowed_pct_of_panel": 8.4, "unknown_pct_of_panel": 30.4, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4255, "material_floor": 100, "DISALLOWED": 4264, "NOT_DISALLOWED_BY_TESTED_RULE": 65319, "UNKNOWN": 30417, "known_denominator": 69583, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.3, "unknown_pct_of_panel": 30.4, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-06-17", "collection_batch_sha256": "babaac516af37d6810f2b1e41996a07e472edb1182552c881f9e9a27a4914acc", "collection_batch_parts": [ { "run_id": "2026-06-17:2026-06-17T10:02:09.863010+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "babaac516af37d6810f2b1e41996a07e472edb1182552c881f9e9a27a4914acc", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "cf27e1ce88ad99cf0f1c2a6a71b1b2fbc86d2522b3761cf9e82ac1544bdced2a", "panel_denominator": 100000, "status_classes": { "status_2xx": 62704, "status_4xx": 12795, "status_5xx": 493, "transport_error": 23864, "other_status": 144 }, "unknown_reasons": { "transport_error": 23864, "server_error": 493, "redirect_or_other_status": 144, "missing_body": 743, "truncated_body": 376, "html_soft_200": 4357, "invalid_utf8": 225 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10451, "material_floor": 100, "DISALLOWED": 10085, "NOT_DISALLOWED_BY_TESTED_RULE": 59713, "UNKNOWN": 30202, "known_denominator": 69798, "panel_denominator": 100000, "disallowed_pct_of_known": 14.4, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.2, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9014, "material_floor": 100, "DISALLOWED": 9111, "NOT_DISALLOWED_BY_TESTED_RULE": 60687, "UNKNOWN": 30202, "known_denominator": 69798, "panel_denominator": 100000, "disallowed_pct_of_known": 13.1, "disallowed_pct_of_panel": 9.1, "unknown_pct_of_panel": 30.2, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8957, "material_floor": 100, "DISALLOWED": 9913, "NOT_DISALLOWED_BY_TESTED_RULE": 59885, "UNKNOWN": 30202, "known_denominator": 69798, "panel_denominator": 100000, "disallowed_pct_of_known": 14.2, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.2, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8274, "material_floor": 100, "DISALLOWED": 8550, "NOT_DISALLOWED_BY_TESTED_RULE": 61248, "UNKNOWN": 30202, "known_denominator": 69798, "panel_denominator": 100000, "disallowed_pct_of_known": 12.2, "disallowed_pct_of_panel": 8.6, "unknown_pct_of_panel": 30.2, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4283, "material_floor": 100, "DISALLOWED": 4307, "NOT_DISALLOWED_BY_TESTED_RULE": 65491, "UNKNOWN": 30202, "known_denominator": 69798, "panel_denominator": 100000, "disallowed_pct_of_known": 6.2, "disallowed_pct_of_panel": 4.3, "unknown_pct_of_panel": 30.2, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-06-18", "collection_batch_sha256": "a42f12474e3b68b5fb25256dad0e096bb8b648cce371e2d8e0c997c6e82353ee", "collection_batch_parts": [ { "run_id": "2026-06-18:2026-06-18T10:09:36.967298+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "a42f12474e3b68b5fb25256dad0e096bb8b648cce371e2d8e0c997c6e82353ee", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "f887c5ca389aa43882bc21d65e97b0dfceda54756e430e3327a83fad9c6d5af5", "panel_denominator": 100000, "status_classes": { "status_2xx": 62556, "status_4xx": 12861, "status_5xx": 482, "transport_error": 23956, "other_status": 145 }, "unknown_reasons": { "transport_error": 23956, "server_error": 482, "redirect_or_other_status": 145, "missing_body": 739, "truncated_body": 383, "html_soft_200": 4342, "invalid_utf8": 227 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10391, "material_floor": 100, "DISALLOWED": 10015, "NOT_DISALLOWED_BY_TESTED_RULE": 59711, "UNKNOWN": 30274, "known_denominator": 69726, "panel_denominator": 100000, "disallowed_pct_of_known": 14.4, "disallowed_pct_of_panel": 10, "unknown_pct_of_panel": 30.3, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8972, "material_floor": 100, "DISALLOWED": 9057, "NOT_DISALLOWED_BY_TESTED_RULE": 60669, "UNKNOWN": 30274, "known_denominator": 69726, "panel_denominator": 100000, "disallowed_pct_of_known": 13, "disallowed_pct_of_panel": 9.1, "unknown_pct_of_panel": 30.3, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8909, "material_floor": 100, "DISALLOWED": 9854, "NOT_DISALLOWED_BY_TESTED_RULE": 59872, "UNKNOWN": 30274, "known_denominator": 69726, "panel_denominator": 100000, "disallowed_pct_of_known": 14.1, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.3, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8230, "material_floor": 100, "DISALLOWED": 8486, "NOT_DISALLOWED_BY_TESTED_RULE": 61240, "UNKNOWN": 30274, "known_denominator": 69726, "panel_denominator": 100000, "disallowed_pct_of_known": 12.2, "disallowed_pct_of_panel": 8.5, "unknown_pct_of_panel": 30.3, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4219, "material_floor": 100, "DISALLOWED": 4231, "NOT_DISALLOWED_BY_TESTED_RULE": 65495, "UNKNOWN": 30274, "known_denominator": 69726, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.3, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-06-20", "collection_batch_sha256": "93fe71e4a5ca33776d9ae2750bc1f8a22706f42a75a3b0c3d19d4fb1f31f63cf", "collection_batch_parts": [ { "run_id": "2026-06-20:2026-06-20T18:54:38.070473+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "93fe71e4a5ca33776d9ae2750bc1f8a22706f42a75a3b0c3d19d4fb1f31f63cf", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "ca97c4980ccc626e4f4cb9479a98035011c4421e6da186b739faa12537bf181a", "panel_denominator": 100000, "status_classes": { "status_2xx": 62495, "status_4xx": 12835, "status_5xx": 485, "transport_error": 24044, "other_status": 141 }, "unknown_reasons": { "transport_error": 24044, "server_error": 485, "redirect_or_other_status": 141, "missing_body": 732, "truncated_body": 387, "html_soft_200": 4346, "invalid_utf8": 229 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10384, "material_floor": 100, "DISALLOWED": 9944, "NOT_DISALLOWED_BY_TESTED_RULE": 59692, "UNKNOWN": 30364, "known_denominator": 69636, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.4, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8980, "material_floor": 100, "DISALLOWED": 8991, "NOT_DISALLOWED_BY_TESTED_RULE": 60645, "UNKNOWN": 30364, "known_denominator": 69636, "panel_denominator": 100000, "disallowed_pct_of_known": 12.9, "disallowed_pct_of_panel": 9, "unknown_pct_of_panel": 30.4, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8906, "material_floor": 100, "DISALLOWED": 9790, "NOT_DISALLOWED_BY_TESTED_RULE": 59846, "UNKNOWN": 30364, "known_denominator": 69636, "panel_denominator": 100000, "disallowed_pct_of_known": 14.1, "disallowed_pct_of_panel": 9.8, "unknown_pct_of_panel": 30.4, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8224, "material_floor": 100, "DISALLOWED": 8415, "NOT_DISALLOWED_BY_TESTED_RULE": 61221, "UNKNOWN": 30364, "known_denominator": 69636, "panel_denominator": 100000, "disallowed_pct_of_known": 12.1, "disallowed_pct_of_panel": 8.4, "unknown_pct_of_panel": 30.4, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4239, "material_floor": 100, "DISALLOWED": 4175, "NOT_DISALLOWED_BY_TESTED_RULE": 65461, "UNKNOWN": 30364, "known_denominator": 69636, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.4, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-06-21", "collection_batch_sha256": "7ff7e11f010ca5fd9e6eba2bde7287671562dbd24d78a8c52e292cef9fe7c165", "collection_batch_parts": [ { "run_id": "2026-06-21:2026-06-21T10:03:50.232034+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "7ff7e11f010ca5fd9e6eba2bde7287671562dbd24d78a8c52e292cef9fe7c165", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "ba17180e952bd7f5e78062aec86bf99a3ee2f245ec386f08972356120a6652e6", "panel_denominator": 100000, "status_classes": { "status_2xx": 62653, "status_4xx": 12840, "status_5xx": 486, "transport_error": 23878, "other_status": 143 }, "unknown_reasons": { "transport_error": 23878, "server_error": 486, "redirect_or_other_status": 143, "missing_body": 739, "truncated_body": 390, "html_soft_200": 4368, "invalid_utf8": 232 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10392, "material_floor": 100, "DISALLOWED": 9962, "NOT_DISALLOWED_BY_TESTED_RULE": 59802, "UNKNOWN": 30236, "known_denominator": 69764, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 10, "unknown_pct_of_panel": 30.2, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8983, "material_floor": 100, "DISALLOWED": 9007, "NOT_DISALLOWED_BY_TESTED_RULE": 60757, "UNKNOWN": 30236, "known_denominator": 69764, "panel_denominator": 100000, "disallowed_pct_of_known": 12.9, "disallowed_pct_of_panel": 9, "unknown_pct_of_panel": 30.2, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8909, "material_floor": 100, "DISALLOWED": 9810, "NOT_DISALLOWED_BY_TESTED_RULE": 59954, "UNKNOWN": 30236, "known_denominator": 69764, "panel_denominator": 100000, "disallowed_pct_of_known": 14.1, "disallowed_pct_of_panel": 9.8, "unknown_pct_of_panel": 30.2, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8227, "material_floor": 100, "DISALLOWED": 8430, "NOT_DISALLOWED_BY_TESTED_RULE": 61334, "UNKNOWN": 30236, "known_denominator": 69764, "panel_denominator": 100000, "disallowed_pct_of_known": 12.1, "disallowed_pct_of_panel": 8.4, "unknown_pct_of_panel": 30.2, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4250, "material_floor": 100, "DISALLOWED": 4197, "NOT_DISALLOWED_BY_TESTED_RULE": 65567, "UNKNOWN": 30236, "known_denominator": 69764, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.2, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-06-22", "collection_batch_sha256": "d44662e1ecfd6f505d117d7c9fb3e745856640c13d4491172907b1a02e86ac64", "collection_batch_parts": [ { "run_id": "2026-06-22:2026-06-22T10:08:46.955571+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "d44662e1ecfd6f505d117d7c9fb3e745856640c13d4491172907b1a02e86ac64", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "6101f4bb149613eea63cb21f78b0baef1c754fb903cea96c0f3876a9946a3b94", "panel_denominator": 100000, "status_classes": { "status_2xx": 62657, "status_4xx": 12817, "status_5xx": 499, "transport_error": 23882, "other_status": 145 }, "unknown_reasons": { "transport_error": 23882, "server_error": 499, "redirect_or_other_status": 145, "missing_body": 737, "truncated_body": 394, "html_soft_200": 4370, "invalid_utf8": 235 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10416, "material_floor": 100, "DISALLOWED": 9992, "NOT_DISALLOWED_BY_TESTED_RULE": 59746, "UNKNOWN": 30262, "known_denominator": 69738, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 10, "unknown_pct_of_panel": 30.3, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9007, "material_floor": 100, "DISALLOWED": 9034, "NOT_DISALLOWED_BY_TESTED_RULE": 60704, "UNKNOWN": 30262, "known_denominator": 69738, "panel_denominator": 100000, "disallowed_pct_of_known": 13, "disallowed_pct_of_panel": 9, "unknown_pct_of_panel": 30.3, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8929, "material_floor": 100, "DISALLOWED": 9834, "NOT_DISALLOWED_BY_TESTED_RULE": 59904, "UNKNOWN": 30262, "known_denominator": 69738, "panel_denominator": 100000, "disallowed_pct_of_known": 14.1, "disallowed_pct_of_panel": 9.8, "unknown_pct_of_panel": 30.3, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8245, "material_floor": 100, "DISALLOWED": 8449, "NOT_DISALLOWED_BY_TESTED_RULE": 61289, "UNKNOWN": 30262, "known_denominator": 69738, "panel_denominator": 100000, "disallowed_pct_of_known": 12.1, "disallowed_pct_of_panel": 8.4, "unknown_pct_of_panel": 30.3, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4256, "material_floor": 100, "DISALLOWED": 4209, "NOT_DISALLOWED_BY_TESTED_RULE": 65529, "UNKNOWN": 30262, "known_denominator": 69738, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.3, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-06-23", "collection_batch_sha256": "02fa50a658b13656f7eb2ff6c893128273f377a8a2b7b5b9e5c88cd46dfd5a72", "collection_batch_parts": [ { "run_id": "2026-06-23:2026-06-23T10:05:50.212696+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "02fa50a658b13656f7eb2ff6c893128273f377a8a2b7b5b9e5c88cd46dfd5a72", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "49797437301e20022904ea9db28a63e8e1ef7a0620f1b3046fb024ff3c616374", "panel_denominator": 100000, "status_classes": { "status_2xx": 62619, "status_4xx": 12836, "status_5xx": 478, "transport_error": 23919, "other_status": 148 }, "unknown_reasons": { "transport_error": 23919, "server_error": 478, "redirect_or_other_status": 148, "missing_body": 733, "truncated_body": 391, "html_soft_200": 4363, "invalid_utf8": 226 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10433, "material_floor": 100, "DISALLOWED": 10014, "NOT_DISALLOWED_BY_TESTED_RULE": 59728, "UNKNOWN": 30258, "known_denominator": 69742, "panel_denominator": 100000, "disallowed_pct_of_known": 14.4, "disallowed_pct_of_panel": 10, "unknown_pct_of_panel": 30.3, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9028, "material_floor": 100, "DISALLOWED": 9059, "NOT_DISALLOWED_BY_TESTED_RULE": 60683, "UNKNOWN": 30258, "known_denominator": 69742, "panel_denominator": 100000, "disallowed_pct_of_known": 13, "disallowed_pct_of_panel": 9.1, "unknown_pct_of_panel": 30.3, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8942, "material_floor": 100, "DISALLOWED": 9853, "NOT_DISALLOWED_BY_TESTED_RULE": 59889, "UNKNOWN": 30258, "known_denominator": 69742, "panel_denominator": 100000, "disallowed_pct_of_known": 14.1, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.3, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8264, "material_floor": 100, "DISALLOWED": 8476, "NOT_DISALLOWED_BY_TESTED_RULE": 61266, "UNKNOWN": 30258, "known_denominator": 69742, "panel_denominator": 100000, "disallowed_pct_of_known": 12.2, "disallowed_pct_of_panel": 8.5, "unknown_pct_of_panel": 30.3, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4261, "material_floor": 100, "DISALLOWED": 4222, "NOT_DISALLOWED_BY_TESTED_RULE": 65520, "UNKNOWN": 30258, "known_denominator": 69742, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.3, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-06-24", "collection_batch_sha256": "df6fcd647d3c80beec4796656d084acad99614e3ed1aea23786fef13be39b367", "collection_batch_parts": [ { "run_id": "2026-06-24:2026-06-24T10:09:50.212662+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "df6fcd647d3c80beec4796656d084acad99614e3ed1aea23786fef13be39b367", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "60a2952f55d94278e09f25b05efe204b9bf8e0779f12fbf3904ce54dcf9f756b", "panel_denominator": 100000, "status_classes": { "status_2xx": 62398, "status_4xx": 12803, "status_5xx": 494, "transport_error": 24169, "other_status": 136 }, "unknown_reasons": { "transport_error": 24169, "server_error": 494, "redirect_or_other_status": 136, "missing_body": 731, "truncated_body": 395, "html_soft_200": 4328, "invalid_utf8": 227 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10417, "material_floor": 100, "DISALLOWED": 9999, "NOT_DISALLOWED_BY_TESTED_RULE": 59521, "UNKNOWN": 30480, "known_denominator": 69520, "panel_denominator": 100000, "disallowed_pct_of_known": 14.4, "disallowed_pct_of_panel": 10, "unknown_pct_of_panel": 30.5, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9033, "material_floor": 100, "DISALLOWED": 9059, "NOT_DISALLOWED_BY_TESTED_RULE": 60461, "UNKNOWN": 30480, "known_denominator": 69520, "panel_denominator": 100000, "disallowed_pct_of_known": 13, "disallowed_pct_of_panel": 9.1, "unknown_pct_of_panel": 30.5, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8935, "material_floor": 100, "DISALLOWED": 9849, "NOT_DISALLOWED_BY_TESTED_RULE": 59671, "UNKNOWN": 30480, "known_denominator": 69520, "panel_denominator": 100000, "disallowed_pct_of_known": 14.2, "disallowed_pct_of_panel": 9.8, "unknown_pct_of_panel": 30.5, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8259, "material_floor": 100, "DISALLOWED": 8469, "NOT_DISALLOWED_BY_TESTED_RULE": 61051, "UNKNOWN": 30480, "known_denominator": 69520, "panel_denominator": 100000, "disallowed_pct_of_known": 12.2, "disallowed_pct_of_panel": 8.5, "unknown_pct_of_panel": 30.5, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4250, "material_floor": 100, "DISALLOWED": 4210, "NOT_DISALLOWED_BY_TESTED_RULE": 65310, "UNKNOWN": 30480, "known_denominator": 69520, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.5, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-06-25", "collection_batch_sha256": "f90bf932f8c424633d7582981168ea87481a135227432e5b88d7cde15618d4cb", "collection_batch_parts": [ { "run_id": "2026-06-25:2026-06-25T10:09:50.210134+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "f90bf932f8c424633d7582981168ea87481a135227432e5b88d7cde15618d4cb", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "c82511b7407daacef9f3c88335a017c8ae17ee54c6791e6ba14145cdd2af8c73", "panel_denominator": 100000, "status_classes": { "status_2xx": 62615, "status_4xx": 12874, "status_5xx": 480, "transport_error": 23888, "other_status": 143 }, "unknown_reasons": { "transport_error": 23888, "server_error": 480, "redirect_or_other_status": 143, "missing_body": 739, "truncated_body": 391, "html_soft_200": 4353, "invalid_utf8": 225 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10508, "material_floor": 100, "DISALLOWED": 10064, "NOT_DISALLOWED_BY_TESTED_RULE": 59717, "UNKNOWN": 30219, "known_denominator": 69781, "panel_denominator": 100000, "disallowed_pct_of_known": 14.4, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.2, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9128, "material_floor": 100, "DISALLOWED": 9121, "NOT_DISALLOWED_BY_TESTED_RULE": 60660, "UNKNOWN": 30219, "known_denominator": 69781, "panel_denominator": 100000, "disallowed_pct_of_known": 13.1, "disallowed_pct_of_panel": 9.1, "unknown_pct_of_panel": 30.2, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9015, "material_floor": 100, "DISALLOWED": 9918, "NOT_DISALLOWED_BY_TESTED_RULE": 59863, "UNKNOWN": 30219, "known_denominator": 69781, "panel_denominator": 100000, "disallowed_pct_of_known": 14.2, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.2, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8348, "material_floor": 100, "DISALLOWED": 8535, "NOT_DISALLOWED_BY_TESTED_RULE": 61246, "UNKNOWN": 30219, "known_denominator": 69781, "panel_denominator": 100000, "disallowed_pct_of_known": 12.2, "disallowed_pct_of_panel": 8.5, "unknown_pct_of_panel": 30.2, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4329, "material_floor": 100, "DISALLOWED": 4266, "NOT_DISALLOWED_BY_TESTED_RULE": 65515, "UNKNOWN": 30219, "known_denominator": 69781, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.3, "unknown_pct_of_panel": 30.2, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-06-26", "collection_batch_sha256": "db2ad74c1f6c6b2c125cfde752e738bbc2117edd6aa9618ad328c1a8f952ca19", "collection_batch_parts": [ { "run_id": "2026-06-26:2026-06-26T10:00:42.990277+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "db2ad74c1f6c6b2c125cfde752e738bbc2117edd6aa9618ad328c1a8f952ca19", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "d86870a5838f809c8ee09832a731484a1f6fd4264e7478692794e8d770cce2ec", "panel_denominator": 100000, "status_classes": { "status_2xx": 62595, "status_4xx": 12835, "status_5xx": 483, "transport_error": 23945, "other_status": 142 }, "unknown_reasons": { "transport_error": 23945, "server_error": 483, "redirect_or_other_status": 142, "missing_body": 736, "truncated_body": 392, "html_soft_200": 4354, "invalid_utf8": 226 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10542, "material_floor": 100, "DISALLOWED": 10061, "NOT_DISALLOWED_BY_TESTED_RULE": 59661, "UNKNOWN": 30278, "known_denominator": 69722, "panel_denominator": 100000, "disallowed_pct_of_known": 14.4, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.3, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9165, "material_floor": 100, "DISALLOWED": 9124, "NOT_DISALLOWED_BY_TESTED_RULE": 60598, "UNKNOWN": 30278, "known_denominator": 69722, "panel_denominator": 100000, "disallowed_pct_of_known": 13.1, "disallowed_pct_of_panel": 9.1, "unknown_pct_of_panel": 30.3, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9049, "material_floor": 100, "DISALLOWED": 9923, "NOT_DISALLOWED_BY_TESTED_RULE": 59799, "UNKNOWN": 30278, "known_denominator": 69722, "panel_denominator": 100000, "disallowed_pct_of_known": 14.2, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.3, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8381, "material_floor": 100, "DISALLOWED": 8532, "NOT_DISALLOWED_BY_TESTED_RULE": 61190, "UNKNOWN": 30278, "known_denominator": 69722, "panel_denominator": 100000, "disallowed_pct_of_known": 12.2, "disallowed_pct_of_panel": 8.5, "unknown_pct_of_panel": 30.3, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4360, "material_floor": 100, "DISALLOWED": 4257, "NOT_DISALLOWED_BY_TESTED_RULE": 65465, "UNKNOWN": 30278, "known_denominator": 69722, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.3, "unknown_pct_of_panel": 30.3, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-06-27", "collection_batch_sha256": "f3f128a99ec6eb26eebc2fce525790933364e182c480e827fccc19d7b093b4bf", "collection_batch_parts": [ { "run_id": "2026-06-27:2026-06-27T10:05:37.213904+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "f3f128a99ec6eb26eebc2fce525790933364e182c480e827fccc19d7b093b4bf", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "ee90132a61182b3a3a769d49e4c81c0b531c91b8d0c4213608bbd0eb36fe7939", "panel_denominator": 100000, "status_classes": { "status_2xx": 62595, "status_4xx": 12839, "status_5xx": 489, "transport_error": 23933, "other_status": 144 }, "unknown_reasons": { "transport_error": 23933, "server_error": 489, "redirect_or_other_status": 144, "missing_body": 738, "truncated_body": 386, "html_soft_200": 4372, "invalid_utf8": 233 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10543, "material_floor": 100, "DISALLOWED": 10066, "NOT_DISALLOWED_BY_TESTED_RULE": 59639, "UNKNOWN": 30295, "known_denominator": 69705, "panel_denominator": 100000, "disallowed_pct_of_known": 14.4, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.3, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9166, "material_floor": 100, "DISALLOWED": 9129, "NOT_DISALLOWED_BY_TESTED_RULE": 60576, "UNKNOWN": 30295, "known_denominator": 69705, "panel_denominator": 100000, "disallowed_pct_of_known": 13.1, "disallowed_pct_of_panel": 9.1, "unknown_pct_of_panel": 30.3, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9043, "material_floor": 100, "DISALLOWED": 9933, "NOT_DISALLOWED_BY_TESTED_RULE": 59772, "UNKNOWN": 30295, "known_denominator": 69705, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.3, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8383, "material_floor": 100, "DISALLOWED": 8543, "NOT_DISALLOWED_BY_TESTED_RULE": 61162, "UNKNOWN": 30295, "known_denominator": 69705, "panel_denominator": 100000, "disallowed_pct_of_known": 12.3, "disallowed_pct_of_panel": 8.5, "unknown_pct_of_panel": 30.3, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4354, "material_floor": 100, "DISALLOWED": 4260, "NOT_DISALLOWED_BY_TESTED_RULE": 65445, "UNKNOWN": 30295, "known_denominator": 69705, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.3, "unknown_pct_of_panel": 30.3, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-06-28", "collection_batch_sha256": "469c8e60c5453cf16dacb3f1606b73b685121efa116ab60110dc2c358f0ec044", "collection_batch_parts": [ { "run_id": "2026-06-28:2026-06-28T10:09:52.717614+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "469c8e60c5453cf16dacb3f1606b73b685121efa116ab60110dc2c358f0ec044", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "64fd583aad5106fd9e0efb177d69831aa6ed3c68dceef745ef13074fc5d39213", "panel_denominator": 100000, "status_classes": { "status_2xx": 62588, "status_4xx": 12822, "status_5xx": 495, "transport_error": 23948, "other_status": 147 }, "unknown_reasons": { "transport_error": 23948, "server_error": 495, "redirect_or_other_status": 147, "missing_body": 745, "truncated_body": 387, "html_soft_200": 4376, "invalid_utf8": 231 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10560, "material_floor": 100, "DISALLOWED": 10080, "NOT_DISALLOWED_BY_TESTED_RULE": 59591, "UNKNOWN": 30329, "known_denominator": 69671, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.3, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9180, "material_floor": 100, "DISALLOWED": 9144, "NOT_DISALLOWED_BY_TESTED_RULE": 60527, "UNKNOWN": 30329, "known_denominator": 69671, "panel_denominator": 100000, "disallowed_pct_of_known": 13.1, "disallowed_pct_of_panel": 9.1, "unknown_pct_of_panel": 30.3, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9064, "material_floor": 100, "DISALLOWED": 9941, "NOT_DISALLOWED_BY_TESTED_RULE": 59730, "UNKNOWN": 30329, "known_denominator": 69671, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.3, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8401, "material_floor": 100, "DISALLOWED": 8556, "NOT_DISALLOWED_BY_TESTED_RULE": 61115, "UNKNOWN": 30329, "known_denominator": 69671, "panel_denominator": 100000, "disallowed_pct_of_known": 12.3, "disallowed_pct_of_panel": 8.6, "unknown_pct_of_panel": 30.3, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4352, "material_floor": 100, "DISALLOWED": 4255, "NOT_DISALLOWED_BY_TESTED_RULE": 65416, "UNKNOWN": 30329, "known_denominator": 69671, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.3, "unknown_pct_of_panel": 30.3, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-06-29", "collection_batch_sha256": "acdb51b2a95418203ad4b5165e6f50f28f813a552ff6e93c0d41350a00ba122e", "collection_batch_parts": [ { "run_id": "2026-06-29:2026-06-29T10:02:39.718325+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "acdb51b2a95418203ad4b5165e6f50f28f813a552ff6e93c0d41350a00ba122e", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "e5dab2494ef941c2cef743b58b8a795ee04438bb7a53c89f5a910a85583e94be", "panel_denominator": 100000, "status_classes": { "status_2xx": 62566, "status_4xx": 12873, "status_5xx": 489, "transport_error": 23927, "other_status": 145 }, "unknown_reasons": { "transport_error": 23927, "server_error": 489, "redirect_or_other_status": 145, "missing_body": 743, "truncated_body": 385, "html_soft_200": 4359, "invalid_utf8": 240 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10560, "material_floor": 100, "DISALLOWED": 10072, "NOT_DISALLOWED_BY_TESTED_RULE": 59640, "UNKNOWN": 30288, "known_denominator": 69712, "panel_denominator": 100000, "disallowed_pct_of_known": 14.4, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.3, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9190, "material_floor": 100, "DISALLOWED": 9140, "NOT_DISALLOWED_BY_TESTED_RULE": 60572, "UNKNOWN": 30288, "known_denominator": 69712, "panel_denominator": 100000, "disallowed_pct_of_known": 13.1, "disallowed_pct_of_panel": 9.1, "unknown_pct_of_panel": 30.3, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9064, "material_floor": 100, "DISALLOWED": 9937, "NOT_DISALLOWED_BY_TESTED_RULE": 59775, "UNKNOWN": 30288, "known_denominator": 69712, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.3, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8398, "material_floor": 100, "DISALLOWED": 8542, "NOT_DISALLOWED_BY_TESTED_RULE": 61170, "UNKNOWN": 30288, "known_denominator": 69712, "panel_denominator": 100000, "disallowed_pct_of_known": 12.3, "disallowed_pct_of_panel": 8.5, "unknown_pct_of_panel": 30.3, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4366, "material_floor": 100, "DISALLOWED": 4249, "NOT_DISALLOWED_BY_TESTED_RULE": 65463, "UNKNOWN": 30288, "known_denominator": 69712, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.3, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-06-30", "collection_batch_sha256": "1a1bad3abd226204c18274c738720c40e4616ff5e0e742dc4e7d88fde163d744", "collection_batch_parts": [ { "run_id": "2026-06-30:2026-06-30T10:05:35.698059+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "1a1bad3abd226204c18274c738720c40e4616ff5e0e742dc4e7d88fde163d744", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "c4357a9b0d0304a20a1260faddac60ff145f2eae435c1de87bbc46b9792b71a2", "panel_denominator": 100000, "status_classes": { "status_2xx": 62323, "status_4xx": 12824, "status_5xx": 491, "transport_error": 24220, "other_status": 142 }, "unknown_reasons": { "transport_error": 24220, "server_error": 491, "redirect_or_other_status": 142, "missing_body": 725, "truncated_body": 391, "html_soft_200": 4332, "invalid_utf8": 240 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10558, "material_floor": 100, "DISALLOWED": 10028, "NOT_DISALLOWED_BY_TESTED_RULE": 59431, "UNKNOWN": 30541, "known_denominator": 69459, "panel_denominator": 100000, "disallowed_pct_of_known": 14.4, "disallowed_pct_of_panel": 10, "unknown_pct_of_panel": 30.5, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9179, "material_floor": 100, "DISALLOWED": 9082, "NOT_DISALLOWED_BY_TESTED_RULE": 60377, "UNKNOWN": 30541, "known_denominator": 69459, "panel_denominator": 100000, "disallowed_pct_of_known": 13.1, "disallowed_pct_of_panel": 9.1, "unknown_pct_of_panel": 30.5, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9050, "material_floor": 100, "DISALLOWED": 9886, "NOT_DISALLOWED_BY_TESTED_RULE": 59573, "UNKNOWN": 30541, "known_denominator": 69459, "panel_denominator": 100000, "disallowed_pct_of_known": 14.2, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.5, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8377, "material_floor": 100, "DISALLOWED": 8483, "NOT_DISALLOWED_BY_TESTED_RULE": 60976, "UNKNOWN": 30541, "known_denominator": 69459, "panel_denominator": 100000, "disallowed_pct_of_known": 12.2, "disallowed_pct_of_panel": 8.5, "unknown_pct_of_panel": 30.5, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4378, "material_floor": 100, "DISALLOWED": 4222, "NOT_DISALLOWED_BY_TESTED_RULE": 65237, "UNKNOWN": 30541, "known_denominator": 69459, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.5, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-01", "collection_batch_sha256": "b2677582add89996fffa9694b9b5b1b4c2bf452b00517a739ce98396f9980146", "collection_batch_parts": [ { "run_id": "2026-07-01:2026-07-01T10:06:35.701002+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "b2677582add89996fffa9694b9b5b1b4c2bf452b00517a739ce98396f9980146", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "c62fb2dc643e27a65d7d2d92054ad3a402d518795960d8bb3a8454af4d9a3b6e", "panel_denominator": 100000, "status_classes": { "status_2xx": 62340, "status_4xx": 12799, "status_5xx": 476, "transport_error": 24239, "other_status": 146 }, "unknown_reasons": { "transport_error": 24239, "server_error": 476, "redirect_or_other_status": 146, "missing_body": 729, "truncated_body": 387, "html_soft_200": 4350, "invalid_utf8": 228 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10544, "material_floor": 100, "DISALLOWED": 10005, "NOT_DISALLOWED_BY_TESTED_RULE": 59440, "UNKNOWN": 30555, "known_denominator": 69445, "panel_denominator": 100000, "disallowed_pct_of_known": 14.4, "disallowed_pct_of_panel": 10, "unknown_pct_of_panel": 30.6, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9172, "material_floor": 100, "DISALLOWED": 9060, "NOT_DISALLOWED_BY_TESTED_RULE": 60385, "UNKNOWN": 30555, "known_denominator": 69445, "panel_denominator": 100000, "disallowed_pct_of_known": 13, "disallowed_pct_of_panel": 9.1, "unknown_pct_of_panel": 30.6, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9038, "material_floor": 100, "DISALLOWED": 9866, "NOT_DISALLOWED_BY_TESTED_RULE": 59579, "UNKNOWN": 30555, "known_denominator": 69445, "panel_denominator": 100000, "disallowed_pct_of_known": 14.2, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.6, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8381, "material_floor": 100, "DISALLOWED": 8465, "NOT_DISALLOWED_BY_TESTED_RULE": 60980, "UNKNOWN": 30555, "known_denominator": 69445, "panel_denominator": 100000, "disallowed_pct_of_known": 12.2, "disallowed_pct_of_panel": 8.5, "unknown_pct_of_panel": 30.6, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4379, "material_floor": 100, "DISALLOWED": 4210, "NOT_DISALLOWED_BY_TESTED_RULE": 65235, "UNKNOWN": 30555, "known_denominator": 69445, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.6, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-02", "collection_batch_sha256": "1d5cec88cbe55eed48786c8ac58491a275fe1473cec437cd83bf3b2ea177f54a", "collection_batch_parts": [ { "run_id": "2026-07-02:2026-07-02T10:05:29.705905+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "1d5cec88cbe55eed48786c8ac58491a275fe1473cec437cd83bf3b2ea177f54a", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "a79a43d8829f7ea89d0a11ea23ed0032ff89e4622bf8130191ee650b37a154cb", "panel_denominator": 100000, "status_classes": { "status_2xx": 62287, "status_4xx": 12815, "status_5xx": 480, "transport_error": 24274, "other_status": 144 }, "unknown_reasons": { "transport_error": 24274, "server_error": 480, "redirect_or_other_status": 144, "missing_body": 724, "truncated_body": 397, "html_soft_200": 4341, "invalid_utf8": 230 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10583, "material_floor": 100, "DISALLOWED": 10028, "NOT_DISALLOWED_BY_TESTED_RULE": 59382, "UNKNOWN": 30590, "known_denominator": 69410, "panel_denominator": 100000, "disallowed_pct_of_known": 14.4, "disallowed_pct_of_panel": 10, "unknown_pct_of_panel": 30.6, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9220, "material_floor": 100, "DISALLOWED": 9092, "NOT_DISALLOWED_BY_TESTED_RULE": 60318, "UNKNOWN": 30590, "known_denominator": 69410, "panel_denominator": 100000, "disallowed_pct_of_known": 13.1, "disallowed_pct_of_panel": 9.1, "unknown_pct_of_panel": 30.6, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9075, "material_floor": 100, "DISALLOWED": 9892, "NOT_DISALLOWED_BY_TESTED_RULE": 59518, "UNKNOWN": 30590, "known_denominator": 69410, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.6, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8426, "material_floor": 100, "DISALLOWED": 8414, "NOT_DISALLOWED_BY_TESTED_RULE": 60996, "UNKNOWN": 30590, "known_denominator": 69410, "panel_denominator": 100000, "disallowed_pct_of_known": 12.1, "disallowed_pct_of_panel": 8.4, "unknown_pct_of_panel": 30.6, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4385, "material_floor": 100, "DISALLOWED": 4194, "NOT_DISALLOWED_BY_TESTED_RULE": 65216, "UNKNOWN": 30590, "known_denominator": 69410, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.6, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-03", "collection_batch_sha256": "c740e9593f61328d38196ef362fe185cae46251203af38cde4bf332846956aed", "collection_batch_parts": [ { "run_id": "2026-07-03:2026-07-03T15:39:40.497797+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "c740e9593f61328d38196ef362fe185cae46251203af38cde4bf332846956aed", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "f710206993af6c449c96ec6cc8a740813eb21fbae53cc2b614367fe8a3412bd9", "panel_denominator": 100000, "status_classes": { "status_2xx": 62268, "status_4xx": 12793, "status_5xx": 477, "transport_error": 24323, "other_status": 139 }, "unknown_reasons": { "transport_error": 24323, "server_error": 477, "redirect_or_other_status": 139, "missing_body": 722, "truncated_body": 397, "html_soft_200": 4345, "invalid_utf8": 225 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10606, "material_floor": 100, "DISALLOWED": 10052, "NOT_DISALLOWED_BY_TESTED_RULE": 59320, "UNKNOWN": 30628, "known_denominator": 69372, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.6, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9247, "material_floor": 100, "DISALLOWED": 9115, "NOT_DISALLOWED_BY_TESTED_RULE": 60257, "UNKNOWN": 30628, "known_denominator": 69372, "panel_denominator": 100000, "disallowed_pct_of_known": 13.1, "disallowed_pct_of_panel": 9.1, "unknown_pct_of_panel": 30.6, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9099, "material_floor": 100, "DISALLOWED": 9915, "NOT_DISALLOWED_BY_TESTED_RULE": 59457, "UNKNOWN": 30628, "known_denominator": 69372, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.6, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8453, "material_floor": 100, "DISALLOWED": 8523, "NOT_DISALLOWED_BY_TESTED_RULE": 60849, "UNKNOWN": 30628, "known_denominator": 69372, "panel_denominator": 100000, "disallowed_pct_of_known": 12.3, "disallowed_pct_of_panel": 8.5, "unknown_pct_of_panel": 30.6, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4395, "material_floor": 100, "DISALLOWED": 4200, "NOT_DISALLOWED_BY_TESTED_RULE": 65172, "UNKNOWN": 30628, "known_denominator": 69372, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.6, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-05", "collection_batch_sha256": "c638bd35f61279801c9cf92337a90c52bf60c7a56aca246cb62f442b947ebad8", "collection_batch_parts": [ { "run_id": "2026-07-05:2026-07-05T20:53:29.055499+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "c638bd35f61279801c9cf92337a90c52bf60c7a56aca246cb62f442b947ebad8", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "758100b9a0d8a05aeebc5cb9e3f4bac51c5386d7b86b65ad4a17964c76c4307e", "panel_denominator": 100000, "status_classes": { "status_2xx": 62284, "status_4xx": 12804, "status_5xx": 478, "transport_error": 24290, "other_status": 144 }, "unknown_reasons": { "transport_error": 24290, "server_error": 478, "redirect_or_other_status": 144, "missing_body": 713, "truncated_body": 391, "html_soft_200": 4345, "invalid_utf8": 232 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10606, "material_floor": 100, "DISALLOWED": 10042, "NOT_DISALLOWED_BY_TESTED_RULE": 59365, "UNKNOWN": 30593, "known_denominator": 69407, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10, "unknown_pct_of_panel": 30.6, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9247, "material_floor": 100, "DISALLOWED": 9105, "NOT_DISALLOWED_BY_TESTED_RULE": 60302, "UNKNOWN": 30593, "known_denominator": 69407, "panel_denominator": 100000, "disallowed_pct_of_known": 13.1, "disallowed_pct_of_panel": 9.1, "unknown_pct_of_panel": 30.6, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9099, "material_floor": 100, "DISALLOWED": 9905, "NOT_DISALLOWED_BY_TESTED_RULE": 59502, "UNKNOWN": 30593, "known_denominator": 69407, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.6, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8452, "material_floor": 100, "DISALLOWED": 8506, "NOT_DISALLOWED_BY_TESTED_RULE": 60901, "UNKNOWN": 30593, "known_denominator": 69407, "panel_denominator": 100000, "disallowed_pct_of_known": 12.3, "disallowed_pct_of_panel": 8.5, "unknown_pct_of_panel": 30.6, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4395, "material_floor": 100, "DISALLOWED": 4185, "NOT_DISALLOWED_BY_TESTED_RULE": 65222, "UNKNOWN": 30593, "known_denominator": 69407, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.6, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-06", "collection_batch_sha256": "c4c1ae0835cd09f694e0a3794514b6f5c78f3096b8397e9136ac46920e7335eb", "collection_batch_parts": [ { "run_id": "2026-07-06:2026-07-06T10:01:57.762777+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "c4c1ae0835cd09f694e0a3794514b6f5c78f3096b8397e9136ac46920e7335eb", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "3a248c37e6e08c2c7bbee8f7618d345fcb2a5b47020dbdb89e9b3f8dca963b53", "panel_denominator": 100000, "status_classes": { "status_2xx": 62189, "status_4xx": 12821, "status_5xx": 467, "transport_error": 24379, "other_status": 144 }, "unknown_reasons": { "transport_error": 24379, "server_error": 467, "redirect_or_other_status": 144, "missing_body": 725, "truncated_body": 397, "html_soft_200": 4330, "invalid_utf8": 232 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10605, "material_floor": 100, "DISALLOWED": 10042, "NOT_DISALLOWED_BY_TESTED_RULE": 59284, "UNKNOWN": 30674, "known_denominator": 69326, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10, "unknown_pct_of_panel": 30.7, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9249, "material_floor": 100, "DISALLOWED": 9105, "NOT_DISALLOWED_BY_TESTED_RULE": 60221, "UNKNOWN": 30674, "known_denominator": 69326, "panel_denominator": 100000, "disallowed_pct_of_known": 13.1, "disallowed_pct_of_panel": 9.1, "unknown_pct_of_panel": 30.7, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9101, "material_floor": 100, "DISALLOWED": 9905, "NOT_DISALLOWED_BY_TESTED_RULE": 59421, "UNKNOWN": 30674, "known_denominator": 69326, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.7, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8451, "material_floor": 100, "DISALLOWED": 8506, "NOT_DISALLOWED_BY_TESTED_RULE": 60820, "UNKNOWN": 30674, "known_denominator": 69326, "panel_denominator": 100000, "disallowed_pct_of_known": 12.3, "disallowed_pct_of_panel": 8.5, "unknown_pct_of_panel": 30.7, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4396, "material_floor": 100, "DISALLOWED": 4186, "NOT_DISALLOWED_BY_TESTED_RULE": 65140, "UNKNOWN": 30674, "known_denominator": 69326, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.7, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-07", "collection_batch_sha256": "0e9f54e4ccbc4d0b4ed6787ccd112da1c4453267034c2d3226f25267d5ba4c9a", "collection_batch_parts": [ { "run_id": "2026-07-07:2026-07-07T10:03:07.756574+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "0e9f54e4ccbc4d0b4ed6787ccd112da1c4453267034c2d3226f25267d5ba4c9a", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "9eabae79b3bb33422f4985955e1ebfb23063c6f899169f90c4fad7dd91377e35", "panel_denominator": 100000, "status_classes": { "status_2xx": 62224, "status_4xx": 12809, "status_5xx": 494, "transport_error": 24333, "other_status": 140 }, "unknown_reasons": { "transport_error": 24333, "server_error": 494, "redirect_or_other_status": 140, "missing_body": 723, "truncated_body": 393, "html_soft_200": 4339, "invalid_utf8": 236 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10619, "material_floor": 100, "DISALLOWED": 10032, "NOT_DISALLOWED_BY_TESTED_RULE": 59310, "UNKNOWN": 30658, "known_denominator": 69342, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10, "unknown_pct_of_panel": 30.7, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9256, "material_floor": 100, "DISALLOWED": 9102, "NOT_DISALLOWED_BY_TESTED_RULE": 60240, "UNKNOWN": 30658, "known_denominator": 69342, "panel_denominator": 100000, "disallowed_pct_of_known": 13.1, "disallowed_pct_of_panel": 9.1, "unknown_pct_of_panel": 30.7, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9094, "material_floor": 100, "DISALLOWED": 9891, "NOT_DISALLOWED_BY_TESTED_RULE": 59451, "UNKNOWN": 30658, "known_denominator": 69342, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.7, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8450, "material_floor": 100, "DISALLOWED": 8494, "NOT_DISALLOWED_BY_TESTED_RULE": 60848, "UNKNOWN": 30658, "known_denominator": 69342, "panel_denominator": 100000, "disallowed_pct_of_known": 12.2, "disallowed_pct_of_panel": 8.5, "unknown_pct_of_panel": 30.7, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4406, "material_floor": 100, "DISALLOWED": 4186, "NOT_DISALLOWED_BY_TESTED_RULE": 65156, "UNKNOWN": 30658, "known_denominator": 69342, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.7, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-08", "collection_batch_sha256": "713d283c1377d06a7d992b04e9182a64ee37a70ba0f70f01350e118ee1b92859", "collection_batch_parts": [ { "run_id": "2026-07-08:2026-07-08T10:04:20.795743+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "713d283c1377d06a7d992b04e9182a64ee37a70ba0f70f01350e118ee1b92859", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "0e7ad824dd22dedef4c3e6a5422ec1535b3487df1a9cbd45ac3eb68519be6dab", "panel_denominator": 100000, "status_classes": { "status_2xx": 62090, "status_4xx": 12824, "status_5xx": 478, "transport_error": 24472, "other_status": 136 }, "unknown_reasons": { "transport_error": 24472, "server_error": 478, "redirect_or_other_status": 136, "missing_body": 718, "truncated_body": 392, "html_soft_200": 4346, "invalid_utf8": 233 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10615, "material_floor": 100, "DISALLOWED": 10044, "NOT_DISALLOWED_BY_TESTED_RULE": 59181, "UNKNOWN": 30775, "known_denominator": 69225, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10, "unknown_pct_of_panel": 30.8, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9279, "material_floor": 100, "DISALLOWED": 9125, "NOT_DISALLOWED_BY_TESTED_RULE": 60100, "UNKNOWN": 30775, "known_denominator": 69225, "panel_denominator": 100000, "disallowed_pct_of_known": 13.2, "disallowed_pct_of_panel": 9.1, "unknown_pct_of_panel": 30.8, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9108, "material_floor": 100, "DISALLOWED": 9909, "NOT_DISALLOWED_BY_TESTED_RULE": 59316, "UNKNOWN": 30775, "known_denominator": 69225, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.8, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8478, "material_floor": 100, "DISALLOWED": 8522, "NOT_DISALLOWED_BY_TESTED_RULE": 60703, "UNKNOWN": 30775, "known_denominator": 69225, "panel_denominator": 100000, "disallowed_pct_of_known": 12.3, "disallowed_pct_of_panel": 8.5, "unknown_pct_of_panel": 30.8, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4406, "material_floor": 100, "DISALLOWED": 4185, "NOT_DISALLOWED_BY_TESTED_RULE": 65040, "UNKNOWN": 30775, "known_denominator": 69225, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.8, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-09", "collection_batch_sha256": "4d03ffd99581099cb9220fdc5b05ab6d55b24f5cb086418f3ec0cc11ba955f2c", "collection_batch_parts": [ { "run_id": "2026-07-09:2026-07-09T10:06:42.181604+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "4d03ffd99581099cb9220fdc5b05ab6d55b24f5cb086418f3ec0cc11ba955f2c", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "72c2c768a74b8fda43e7b4513c0ad8090a3336e8c0c70ffbbc2dac29841cd558", "panel_denominator": 100000, "status_classes": { "status_2xx": 62218, "status_4xx": 12901, "status_5xx": 479, "transport_error": 24254, "other_status": 148 }, "unknown_reasons": { "transport_error": 24254, "server_error": 479, "redirect_or_other_status": 148, "missing_body": 713, "truncated_body": 389, "html_soft_200": 4353, "invalid_utf8": 223 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10667, "material_floor": 100, "DISALLOWED": 10064, "NOT_DISALLOWED_BY_TESTED_RULE": 59377, "UNKNOWN": 30559, "known_denominator": 69441, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.6, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9316, "material_floor": 100, "DISALLOWED": 9141, "NOT_DISALLOWED_BY_TESTED_RULE": 60300, "UNKNOWN": 30559, "known_denominator": 69441, "panel_denominator": 100000, "disallowed_pct_of_known": 13.2, "disallowed_pct_of_panel": 9.1, "unknown_pct_of_panel": 30.6, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9140, "material_floor": 100, "DISALLOWED": 9920, "NOT_DISALLOWED_BY_TESTED_RULE": 59521, "UNKNOWN": 30559, "known_denominator": 69441, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.6, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8504, "material_floor": 100, "DISALLOWED": 8531, "NOT_DISALLOWED_BY_TESTED_RULE": 60910, "UNKNOWN": 30559, "known_denominator": 69441, "panel_denominator": 100000, "disallowed_pct_of_known": 12.3, "disallowed_pct_of_panel": 8.5, "unknown_pct_of_panel": 30.6, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4432, "material_floor": 100, "DISALLOWED": 4190, "NOT_DISALLOWED_BY_TESTED_RULE": 65251, "UNKNOWN": 30559, "known_denominator": 69441, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.6, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-10", "collection_batch_sha256": "cf3467c0ec8b044dc668233c31ac9719d2e60be448efb7723a742e2e79806ba9", "collection_batch_parts": [ { "run_id": "2026-07-10:2026-07-10T13:38:29.345469+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "cf3467c0ec8b044dc668233c31ac9719d2e60be448efb7723a742e2e79806ba9", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "b388dd4e2c6171357a313783de29671c0e97125cde539f80445a3b4cc037ae97", "panel_denominator": 100000, "status_classes": { "status_2xx": 62150, "status_4xx": 12835, "status_5xx": 485, "transport_error": 24389, "other_status": 141 }, "unknown_reasons": { "transport_error": 24389, "server_error": 485, "redirect_or_other_status": 141, "missing_body": 714, "truncated_body": 392, "html_soft_200": 4342, "invalid_utf8": 223 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10679, "material_floor": 100, "DISALLOWED": 10082, "NOT_DISALLOWED_BY_TESTED_RULE": 59232, "UNKNOWN": 30686, "known_denominator": 69314, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.7, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9337, "material_floor": 100, "DISALLOWED": 9159, "NOT_DISALLOWED_BY_TESTED_RULE": 60155, "UNKNOWN": 30686, "known_denominator": 69314, "panel_denominator": 100000, "disallowed_pct_of_known": 13.2, "disallowed_pct_of_panel": 9.2, "unknown_pct_of_panel": 30.7, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9163, "material_floor": 100, "DISALLOWED": 9938, "NOT_DISALLOWED_BY_TESTED_RULE": 59376, "UNKNOWN": 30686, "known_denominator": 69314, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.7, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8523, "material_floor": 100, "DISALLOWED": 8548, "NOT_DISALLOWED_BY_TESTED_RULE": 60766, "UNKNOWN": 30686, "known_denominator": 69314, "panel_denominator": 100000, "disallowed_pct_of_known": 12.3, "disallowed_pct_of_panel": 8.5, "unknown_pct_of_panel": 30.7, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4441, "material_floor": 100, "DISALLOWED": 4192, "NOT_DISALLOWED_BY_TESTED_RULE": 65122, "UNKNOWN": 30686, "known_denominator": 69314, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.7, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-11", "collection_batch_sha256": "f1861637263e6cf385d86b78b2cd46adbd12598a6f5e0fe48985bd43b3303be1", "collection_batch_parts": [ { "run_id": "2026-07-11:2026-07-11T10:06:57.590078+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "f1861637263e6cf385d86b78b2cd46adbd12598a6f5e0fe48985bd43b3303be1", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "2b2051acc5cdee3cfaabbcc4b91e0869bc81f2f4f2cc5f43b53a26c76f694db7", "panel_denominator": 100000, "status_classes": { "status_2xx": 62247, "status_4xx": 12863, "status_5xx": 491, "transport_error": 24255, "other_status": 144 }, "unknown_reasons": { "transport_error": 24255, "server_error": 491, "redirect_or_other_status": 144, "missing_body": 711, "truncated_body": 395, "html_soft_200": 4359, "invalid_utf8": 227 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10679, "material_floor": 100, "DISALLOWED": 10070, "NOT_DISALLOWED_BY_TESTED_RULE": 59348, "UNKNOWN": 30582, "known_denominator": 69418, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.6, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9347, "material_floor": 100, "DISALLOWED": 9157, "NOT_DISALLOWED_BY_TESTED_RULE": 60261, "UNKNOWN": 30582, "known_denominator": 69418, "panel_denominator": 100000, "disallowed_pct_of_known": 13.2, "disallowed_pct_of_panel": 9.2, "unknown_pct_of_panel": 30.6, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9161, "material_floor": 100, "DISALLOWED": 9927, "NOT_DISALLOWED_BY_TESTED_RULE": 59491, "UNKNOWN": 30582, "known_denominator": 69418, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.6, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8527, "material_floor": 100, "DISALLOWED": 8539, "NOT_DISALLOWED_BY_TESTED_RULE": 60879, "UNKNOWN": 30582, "known_denominator": 69418, "panel_denominator": 100000, "disallowed_pct_of_known": 12.3, "disallowed_pct_of_panel": 8.5, "unknown_pct_of_panel": 30.6, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4441, "material_floor": 100, "DISALLOWED": 4183, "NOT_DISALLOWED_BY_TESTED_RULE": 65235, "UNKNOWN": 30582, "known_denominator": 69418, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.6, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-12", "collection_batch_sha256": "907a32c7747d4c995ef778fd22f08ebf57277350e3443314090b2c3aeca05777", "collection_batch_parts": [ { "run_id": "2026-07-12:2026-07-12T10:07:27.574580+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "907a32c7747d4c995ef778fd22f08ebf57277350e3443314090b2c3aeca05777", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "fd99d90c18086f55de840f2bb70b1a40b4d752fb686cd763b14f7cb41e8892a5", "panel_denominator": 100000, "status_classes": { "status_2xx": 62242, "status_4xx": 12887, "status_5xx": 497, "transport_error": 24228, "other_status": 146 }, "unknown_reasons": { "transport_error": 24228, "server_error": 497, "redirect_or_other_status": 146, "missing_body": 713, "truncated_body": 392, "html_soft_200": 4352, "invalid_utf8": 226 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10687, "material_floor": 100, "DISALLOWED": 10076, "NOT_DISALLOWED_BY_TESTED_RULE": 59370, "UNKNOWN": 30554, "known_denominator": 69446, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.6, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9353, "material_floor": 100, "DISALLOWED": 9163, "NOT_DISALLOWED_BY_TESTED_RULE": 60283, "UNKNOWN": 30554, "known_denominator": 69446, "panel_denominator": 100000, "disallowed_pct_of_known": 13.2, "disallowed_pct_of_panel": 9.2, "unknown_pct_of_panel": 30.6, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9163, "material_floor": 100, "DISALLOWED": 9933, "NOT_DISALLOWED_BY_TESTED_RULE": 59513, "UNKNOWN": 30554, "known_denominator": 69446, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.6, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8523, "material_floor": 100, "DISALLOWED": 8538, "NOT_DISALLOWED_BY_TESTED_RULE": 60908, "UNKNOWN": 30554, "known_denominator": 69446, "panel_denominator": 100000, "disallowed_pct_of_known": 12.3, "disallowed_pct_of_panel": 8.5, "unknown_pct_of_panel": 30.6, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4444, "material_floor": 100, "DISALLOWED": 4183, "NOT_DISALLOWED_BY_TESTED_RULE": 65263, "UNKNOWN": 30554, "known_denominator": 69446, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.6, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-13", "collection_batch_sha256": "69360cbdf446b5d3ccdeb65585294f72d4643ecc0cad4ae7cbfdca94f544f146", "collection_batch_parts": [ { "run_id": "2026-07-13:2026-07-13T10:03:02.565633+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "69360cbdf446b5d3ccdeb65585294f72d4643ecc0cad4ae7cbfdca94f544f146", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "3ce4b21f8029deb6d7f8b79be18524a4c7bee0813cbac2530d4424e2a0495628", "panel_denominator": 100000, "status_classes": { "status_2xx": 62175, "status_4xx": 12861, "status_5xx": 491, "transport_error": 24328, "other_status": 145 }, "unknown_reasons": { "transport_error": 24328, "server_error": 491, "redirect_or_other_status": 145, "missing_body": 718, "truncated_body": 394, "html_soft_200": 4353, "invalid_utf8": 231 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10666, "material_floor": 100, "DISALLOWED": 10038, "NOT_DISALLOWED_BY_TESTED_RULE": 59302, "UNKNOWN": 30660, "known_denominator": 69340, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10, "unknown_pct_of_panel": 30.7, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9335, "material_floor": 100, "DISALLOWED": 9129, "NOT_DISALLOWED_BY_TESTED_RULE": 60211, "UNKNOWN": 30660, "known_denominator": 69340, "panel_denominator": 100000, "disallowed_pct_of_known": 13.2, "disallowed_pct_of_panel": 9.1, "unknown_pct_of_panel": 30.7, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9161, "material_floor": 100, "DISALLOWED": 9906, "NOT_DISALLOWED_BY_TESTED_RULE": 59434, "UNKNOWN": 30660, "known_denominator": 69340, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.7, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8521, "material_floor": 100, "DISALLOWED": 8515, "NOT_DISALLOWED_BY_TESTED_RULE": 60825, "UNKNOWN": 30660, "known_denominator": 69340, "panel_denominator": 100000, "disallowed_pct_of_known": 12.3, "disallowed_pct_of_panel": 8.5, "unknown_pct_of_panel": 30.7, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4438, "material_floor": 100, "DISALLOWED": 4160, "NOT_DISALLOWED_BY_TESTED_RULE": 65180, "UNKNOWN": 30660, "known_denominator": 69340, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.7, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-14", "collection_batch_sha256": "1f4398d1aff3977cd9d43b130628ef10b7082b8519f8c46461a14bd4c10ce6a5", "collection_batch_parts": [ { "run_id": "2026-07-14:2026-07-14T10:05:29.280780+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "1f4398d1aff3977cd9d43b130628ef10b7082b8519f8c46461a14bd4c10ce6a5", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "1ed6ba5facd4205fc07164f8d8ad6dfc5eed00e4fbe235ad8ee37e4c8ee9b387", "panel_denominator": 100000, "status_classes": { "status_2xx": 62267, "status_4xx": 12815, "status_5xx": 479, "transport_error": 24304, "other_status": 135 }, "unknown_reasons": { "transport_error": 24304, "server_error": 479, "redirect_or_other_status": 135, "missing_body": 717, "truncated_body": 395, "html_soft_200": 4339, "invalid_utf8": 227 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10690, "material_floor": 100, "DISALLOWED": 10053, "NOT_DISALLOWED_BY_TESTED_RULE": 59351, "UNKNOWN": 30596, "known_denominator": 69404, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.6, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9356, "material_floor": 100, "DISALLOWED": 9141, "NOT_DISALLOWED_BY_TESTED_RULE": 60263, "UNKNOWN": 30596, "known_denominator": 69404, "panel_denominator": 100000, "disallowed_pct_of_known": 13.2, "disallowed_pct_of_panel": 9.1, "unknown_pct_of_panel": 30.6, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9178, "material_floor": 100, "DISALLOWED": 9920, "NOT_DISALLOWED_BY_TESTED_RULE": 59484, "UNKNOWN": 30596, "known_denominator": 69404, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 9.9, "unknown_pct_of_panel": 30.6, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8550, "material_floor": 100, "DISALLOWED": 8535, "NOT_DISALLOWED_BY_TESTED_RULE": 60869, "UNKNOWN": 30596, "known_denominator": 69404, "panel_denominator": 100000, "disallowed_pct_of_known": 12.3, "disallowed_pct_of_panel": 8.5, "unknown_pct_of_panel": 30.6, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4450, "material_floor": 100, "DISALLOWED": 4158, "NOT_DISALLOWED_BY_TESTED_RULE": 65246, "UNKNOWN": 30596, "known_denominator": 69404, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.6, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-16", "collection_batch_sha256": "1cb48c890b1bc7acd62b9a3f43ad8bb1ee39fc1850f4da6254f734b816741db6", "collection_batch_parts": [ { "run_id": "2026-07-16:2026-07-16T00:50:07.723772+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "1cb48c890b1bc7acd62b9a3f43ad8bb1ee39fc1850f4da6254f734b816741db6", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "a672d4582f3a58bd6775296970c6dc84f5262a7f87aad2d9aa82d56e4e71db96", "panel_denominator": 100000, "status_classes": { "status_2xx": 62244, "status_4xx": 12860, "status_5xx": 486, "transport_error": 24276, "other_status": 134 }, "unknown_reasons": { "transport_error": 24276, "server_error": 486, "redirect_or_other_status": 134, "missing_body": 708, "truncated_body": 394, "html_soft_200": 4353, "invalid_utf8": 232 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10743, "material_floor": 100, "DISALLOWED": 10091, "NOT_DISALLOWED_BY_TESTED_RULE": 59326, "UNKNOWN": 30583, "known_denominator": 69417, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.6, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9408, "material_floor": 100, "DISALLOWED": 9178, "NOT_DISALLOWED_BY_TESTED_RULE": 60239, "UNKNOWN": 30583, "known_denominator": 69417, "panel_denominator": 100000, "disallowed_pct_of_known": 13.2, "disallowed_pct_of_panel": 9.2, "unknown_pct_of_panel": 30.6, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9216, "material_floor": 100, "DISALLOWED": 9956, "NOT_DISALLOWED_BY_TESTED_RULE": 59461, "UNKNOWN": 30583, "known_denominator": 69417, "panel_denominator": 100000, "disallowed_pct_of_known": 14.3, "disallowed_pct_of_panel": 10, "unknown_pct_of_panel": 30.6, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8586, "material_floor": 100, "DISALLOWED": 8568, "NOT_DISALLOWED_BY_TESTED_RULE": 60849, "UNKNOWN": 30583, "known_denominator": 69417, "panel_denominator": 100000, "disallowed_pct_of_known": 12.3, "disallowed_pct_of_panel": 8.6, "unknown_pct_of_panel": 30.6, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4474, "material_floor": 100, "DISALLOWED": 4167, "NOT_DISALLOWED_BY_TESTED_RULE": 65250, "UNKNOWN": 30583, "known_denominator": 69417, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.6, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-17", "collection_batch_sha256": "1ed53c28b08d76c04f80c56027d5288af58fdb99f452b85ea813d6477b568fdd", "collection_batch_parts": [ { "run_id": "2026-07-17:2026-07-17T10:07:33.252220+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "1ed53c28b08d76c04f80c56027d5288af58fdb99f452b85ea813d6477b568fdd", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "0a49bbbc6f0b8364fe8038c39f1682d4add78b0a4308e8735e92f36439fd5b40", "panel_denominator": 100000, "status_classes": { "status_2xx": 62392, "status_4xx": 12829, "status_5xx": 521, "transport_error": 24112, "other_status": 146 }, "unknown_reasons": { "transport_error": 24112, "server_error": 521, "redirect_or_other_status": 146, "missing_body": 724, "truncated_body": 395, "html_soft_200": 4345, "invalid_utf8": 231 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10786, "material_floor": 100, "DISALLOWED": 10156, "NOT_DISALLOWED_BY_TESTED_RULE": 59370, "UNKNOWN": 30474, "known_denominator": 69526, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.5, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9474, "material_floor": 100, "DISALLOWED": 9261, "NOT_DISALLOWED_BY_TESTED_RULE": 60265, "UNKNOWN": 30474, "known_denominator": 69526, "panel_denominator": 100000, "disallowed_pct_of_known": 13.3, "disallowed_pct_of_panel": 9.3, "unknown_pct_of_panel": 30.5, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9263, "material_floor": 100, "DISALLOWED": 10037, "NOT_DISALLOWED_BY_TESTED_RULE": 59489, "UNKNOWN": 30474, "known_denominator": 69526, "panel_denominator": 100000, "disallowed_pct_of_known": 14.4, "disallowed_pct_of_panel": 10, "unknown_pct_of_panel": 30.5, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8634, "material_floor": 100, "DISALLOWED": 8635, "NOT_DISALLOWED_BY_TESTED_RULE": 60891, "UNKNOWN": 30474, "known_denominator": 69526, "panel_denominator": 100000, "disallowed_pct_of_known": 12.4, "disallowed_pct_of_panel": 8.6, "unknown_pct_of_panel": 30.5, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4494, "material_floor": 100, "DISALLOWED": 4209, "NOT_DISALLOWED_BY_TESTED_RULE": 65317, "UNKNOWN": 30474, "known_denominator": 69526, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.5, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-18", "collection_batch_sha256": "44b204eadfa4937f9c76eee8ff73c24486ab4bb52dc85a555b6fcbac44fb41be", "collection_batch_parts": [ { "run_id": "2026-07-18:2026-07-18T10:06:13.251277+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "44b204eadfa4937f9c76eee8ff73c24486ab4bb52dc85a555b6fcbac44fb41be", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "06749891f45e45c2f52e9466ef40861d6a31787d103c56889e304dd46d5782c5", "panel_denominator": 100000, "status_classes": { "status_2xx": 62515, "status_4xx": 12884, "status_5xx": 510, "transport_error": 23943, "other_status": 148 }, "unknown_reasons": { "transport_error": 23943, "server_error": 510, "redirect_or_other_status": 148, "missing_body": 724, "truncated_body": 396, "html_soft_200": 4356, "invalid_utf8": 229 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10816, "material_floor": 100, "DISALLOWED": 10189, "NOT_DISALLOWED_BY_TESTED_RULE": 59505, "UNKNOWN": 30306, "known_denominator": 69694, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.3, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9498, "material_floor": 100, "DISALLOWED": 9286, "NOT_DISALLOWED_BY_TESTED_RULE": 60408, "UNKNOWN": 30306, "known_denominator": 69694, "panel_denominator": 100000, "disallowed_pct_of_known": 13.3, "disallowed_pct_of_panel": 9.3, "unknown_pct_of_panel": 30.3, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9288, "material_floor": 100, "DISALLOWED": 10068, "NOT_DISALLOWED_BY_TESTED_RULE": 59626, "UNKNOWN": 30306, "known_denominator": 69694, "panel_denominator": 100000, "disallowed_pct_of_known": 14.4, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.3, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8663, "material_floor": 100, "DISALLOWED": 8664, "NOT_DISALLOWED_BY_TESTED_RULE": 61030, "UNKNOWN": 30306, "known_denominator": 69694, "panel_denominator": 100000, "disallowed_pct_of_known": 12.4, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.3, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4518, "material_floor": 100, "DISALLOWED": 4221, "NOT_DISALLOWED_BY_TESTED_RULE": 65473, "UNKNOWN": 30306, "known_denominator": 69694, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.3, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-19", "collection_batch_sha256": "90ab798275a0235b36fa3d03f0fa01264126e41594ad030d3aa84c9323c4ddb3", "collection_batch_parts": [ { "run_id": "2026-07-19:2026-07-19T10:09:23.222525+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "90ab798275a0235b36fa3d03f0fa01264126e41594ad030d3aa84c9323c4ddb3", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "481ce80cf80a4cf01f3b5b6671a51ffed21b10fc713c99f236317ce95fb85e21", "panel_denominator": 100000, "status_classes": { "status_2xx": 62193, "status_4xx": 12793, "status_5xx": 505, "transport_error": 24369, "other_status": 140 }, "unknown_reasons": { "transport_error": 24369, "server_error": 505, "redirect_or_other_status": 140, "missing_body": 732, "truncated_body": 389, "html_soft_200": 4339, "invalid_utf8": 229 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10789, "material_floor": 100, "DISALLOWED": 10156, "NOT_DISALLOWED_BY_TESTED_RULE": 59141, "UNKNOWN": 30703, "known_denominator": 69297, "panel_denominator": 100000, "disallowed_pct_of_known": 14.7, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.7, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9484, "material_floor": 100, "DISALLOWED": 9257, "NOT_DISALLOWED_BY_TESTED_RULE": 60040, "UNKNOWN": 30703, "known_denominator": 69297, "panel_denominator": 100000, "disallowed_pct_of_known": 13.4, "disallowed_pct_of_panel": 9.3, "unknown_pct_of_panel": 30.7, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9272, "material_floor": 100, "DISALLOWED": 10035, "NOT_DISALLOWED_BY_TESTED_RULE": 59262, "UNKNOWN": 30703, "known_denominator": 69297, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10, "unknown_pct_of_panel": 30.7, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8645, "material_floor": 100, "DISALLOWED": 8634, "NOT_DISALLOWED_BY_TESTED_RULE": 60663, "UNKNOWN": 30703, "known_denominator": 69297, "panel_denominator": 100000, "disallowed_pct_of_known": 12.5, "disallowed_pct_of_panel": 8.6, "unknown_pct_of_panel": 30.7, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4498, "material_floor": 100, "DISALLOWED": 4198, "NOT_DISALLOWED_BY_TESTED_RULE": 65099, "UNKNOWN": 30703, "known_denominator": 69297, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.7, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-20", "collection_batch_sha256": "70408754844037a6e54d7fc506ebb11856481c01db987c645458f530f90d49f9", "collection_batch_parts": [ { "run_id": "2026-07-20:2026-07-20T10:03:16.229058+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "70408754844037a6e54d7fc506ebb11856481c01db987c645458f530f90d49f9", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "91d4aef1226abd21223bb8cebd15314f9b367c0292c72663441b2cb6399104d2", "panel_denominator": 100000, "status_classes": { "status_2xx": 62398, "status_4xx": 12837, "status_5xx": 503, "transport_error": 24116, "other_status": 146 }, "unknown_reasons": { "transport_error": 24116, "server_error": 503, "redirect_or_other_status": 146, "missing_body": 724, "truncated_body": 389, "html_soft_200": 4365, "invalid_utf8": 232 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10806, "material_floor": 100, "DISALLOWED": 10172, "NOT_DISALLOWED_BY_TESTED_RULE": 59353, "UNKNOWN": 30475, "known_denominator": 69525, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.5, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9500, "material_floor": 100, "DISALLOWED": 9271, "NOT_DISALLOWED_BY_TESTED_RULE": 60254, "UNKNOWN": 30475, "known_denominator": 69525, "panel_denominator": 100000, "disallowed_pct_of_known": 13.3, "disallowed_pct_of_panel": 9.3, "unknown_pct_of_panel": 30.5, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9285, "material_floor": 100, "DISALLOWED": 10053, "NOT_DISALLOWED_BY_TESTED_RULE": 59472, "UNKNOWN": 30475, "known_denominator": 69525, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.5, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8654, "material_floor": 100, "DISALLOWED": 8645, "NOT_DISALLOWED_BY_TESTED_RULE": 60880, "UNKNOWN": 30475, "known_denominator": 69525, "panel_denominator": 100000, "disallowed_pct_of_known": 12.4, "disallowed_pct_of_panel": 8.6, "unknown_pct_of_panel": 30.5, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4512, "material_floor": 100, "DISALLOWED": 4203, "NOT_DISALLOWED_BY_TESTED_RULE": 65322, "UNKNOWN": 30475, "known_denominator": 69525, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.5, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-21", "collection_batch_sha256": "52d651a4eaf711be5ba61950015c1d368dcd2b0b348c93e7278f25d8c4ca0f48", "collection_batch_parts": [ { "run_id": "2026-07-21:2026-07-21T10:02:06.252577+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "52d651a4eaf711be5ba61950015c1d368dcd2b0b348c93e7278f25d8c4ca0f48", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "8d0fe644856b7e6620c3c1dfbf126490cc03ae4c79243c044e6a5fd830103286", "panel_denominator": 100000, "status_classes": { "status_2xx": 62288, "status_4xx": 12819, "status_5xx": 490, "transport_error": 24260, "other_status": 143 }, "unknown_reasons": { "transport_error": 24260, "server_error": 490, "redirect_or_other_status": 143, "missing_body": 713, "truncated_body": 396, "html_soft_200": 4352, "invalid_utf8": 227 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10762, "material_floor": 100, "DISALLOWED": 10111, "NOT_DISALLOWED_BY_TESTED_RULE": 59308, "UNKNOWN": 30581, "known_denominator": 69419, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.6, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9451, "material_floor": 100, "DISALLOWED": 9210, "NOT_DISALLOWED_BY_TESTED_RULE": 60209, "UNKNOWN": 30581, "known_denominator": 69419, "panel_denominator": 100000, "disallowed_pct_of_known": 13.3, "disallowed_pct_of_panel": 9.2, "unknown_pct_of_panel": 30.6, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9239, "material_floor": 100, "DISALLOWED": 9989, "NOT_DISALLOWED_BY_TESTED_RULE": 59430, "UNKNOWN": 30581, "known_denominator": 69419, "panel_denominator": 100000, "disallowed_pct_of_known": 14.4, "disallowed_pct_of_panel": 10, "unknown_pct_of_panel": 30.6, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8612, "material_floor": 100, "DISALLOWED": 8581, "NOT_DISALLOWED_BY_TESTED_RULE": 60838, "UNKNOWN": 30581, "known_denominator": 69419, "panel_denominator": 100000, "disallowed_pct_of_known": 12.4, "disallowed_pct_of_panel": 8.6, "unknown_pct_of_panel": 30.6, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4509, "material_floor": 100, "DISALLOWED": 4178, "NOT_DISALLOWED_BY_TESTED_RULE": 65241, "UNKNOWN": 30581, "known_denominator": 69419, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.6, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-22", "collection_batch_sha256": "124d98884d8a4e8c43a0b09be5cf58bc277cec59ee5fbb11dacc42bf7c01c1dc", "collection_batch_parts": [ { "run_id": "2026-07-22:2026-07-22T10:02:47.092954+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "124d98884d8a4e8c43a0b09be5cf58bc277cec59ee5fbb11dacc42bf7c01c1dc", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "91634c353a614e155089c33fc990bd782e6e662c35ed91e0fbed4587cf71c4e0", "panel_denominator": 100000, "status_classes": { "status_2xx": 62486, "status_4xx": 12836, "status_5xx": 486, "transport_error": 24049, "other_status": 143 }, "unknown_reasons": { "transport_error": 24049, "server_error": 486, "redirect_or_other_status": 143, "missing_body": 734, "truncated_body": 397, "html_soft_200": 4346, "invalid_utf8": 220 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10824, "material_floor": 100, "DISALLOWED": 10170, "NOT_DISALLOWED_BY_TESTED_RULE": 59455, "UNKNOWN": 30375, "known_denominator": 69625, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.4, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9515, "material_floor": 100, "DISALLOWED": 9270, "NOT_DISALLOWED_BY_TESTED_RULE": 60355, "UNKNOWN": 30375, "known_denominator": 69625, "panel_denominator": 100000, "disallowed_pct_of_known": 13.3, "disallowed_pct_of_panel": 9.3, "unknown_pct_of_panel": 30.4, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9298, "material_floor": 100, "DISALLOWED": 10053, "NOT_DISALLOWED_BY_TESTED_RULE": 59572, "UNKNOWN": 30375, "known_denominator": 69625, "panel_denominator": 100000, "disallowed_pct_of_known": 14.4, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.4, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8671, "material_floor": 100, "DISALLOWED": 8640, "NOT_DISALLOWED_BY_TESTED_RULE": 60985, "UNKNOWN": 30375, "known_denominator": 69625, "panel_denominator": 100000, "disallowed_pct_of_known": 12.4, "disallowed_pct_of_panel": 8.6, "unknown_pct_of_panel": 30.4, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4537, "material_floor": 100, "DISALLOWED": 4200, "NOT_DISALLOWED_BY_TESTED_RULE": 65425, "UNKNOWN": 30375, "known_denominator": 69625, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.4, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-23", "collection_batch_sha256": "4f74ac0979816572e1e26d860042629a5ade31c85ca566665a4771e498b59ddd", "collection_batch_parts": [ { "run_id": "2026-07-23:2026-07-23T10:01:51.376304+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "4f74ac0979816572e1e26d860042629a5ade31c85ca566665a4771e498b59ddd", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "ee35f025765b2607009b71007826e8d40f014d7bfa07f79ba611e594f3301a72", "panel_denominator": 100000, "status_classes": { "status_2xx": 62508, "status_4xx": 12865, "status_5xx": 484, "transport_error": 23999, "other_status": 144 }, "unknown_reasons": { "transport_error": 23999, "server_error": 484, "redirect_or_other_status": 144, "missing_body": 740, "truncated_body": 400, "html_soft_200": 4351, "invalid_utf8": 234 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10839, "material_floor": 100, "DISALLOWED": 10197, "NOT_DISALLOWED_BY_TESTED_RULE": 59451, "UNKNOWN": 30352, "known_denominator": 69648, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.4, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9536, "material_floor": 100, "DISALLOWED": 9298, "NOT_DISALLOWED_BY_TESTED_RULE": 60350, "UNKNOWN": 30352, "known_denominator": 69648, "panel_denominator": 100000, "disallowed_pct_of_known": 13.3, "disallowed_pct_of_panel": 9.3, "unknown_pct_of_panel": 30.4, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9320, "material_floor": 100, "DISALLOWED": 10078, "NOT_DISALLOWED_BY_TESTED_RULE": 59570, "UNKNOWN": 30352, "known_denominator": 69648, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.4, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8690, "material_floor": 100, "DISALLOWED": 8671, "NOT_DISALLOWED_BY_TESTED_RULE": 60977, "UNKNOWN": 30352, "known_denominator": 69648, "panel_denominator": 100000, "disallowed_pct_of_known": 12.4, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.4, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4528, "material_floor": 100, "DISALLOWED": 4202, "NOT_DISALLOWED_BY_TESTED_RULE": 65446, "UNKNOWN": 30352, "known_denominator": 69648, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.4, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-24", "collection_batch_sha256": "e22377b94ed805c58ad653e661650e246c57e84e29c87a2ec57a03d012d2c017", "collection_batch_parts": [ { "run_id": "2026-07-24:2026-07-24T10:10:27.119442+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "e22377b94ed805c58ad653e661650e246c57e84e29c87a2ec57a03d012d2c017", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "5c340385772b32e8078a75898abc77fad58610207eb5c9e0be088dddd630ce21", "panel_denominator": 100000, "status_classes": { "status_2xx": 62311, "status_4xx": 12851, "status_5xx": 486, "transport_error": 24211, "other_status": 141 }, "unknown_reasons": { "transport_error": 24211, "server_error": 486, "redirect_or_other_status": 141, "missing_body": 726, "truncated_body": 394, "html_soft_200": 4329, "invalid_utf8": 232 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10827, "material_floor": 100, "DISALLOWED": 10177, "NOT_DISALLOWED_BY_TESTED_RULE": 59304, "UNKNOWN": 30519, "known_denominator": 69481, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.5, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9540, "material_floor": 100, "DISALLOWED": 9290, "NOT_DISALLOWED_BY_TESTED_RULE": 60191, "UNKNOWN": 30519, "known_denominator": 69481, "panel_denominator": 100000, "disallowed_pct_of_known": 13.4, "disallowed_pct_of_panel": 9.3, "unknown_pct_of_panel": 30.5, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9313, "material_floor": 100, "DISALLOWED": 10056, "NOT_DISALLOWED_BY_TESTED_RULE": 59425, "UNKNOWN": 30519, "known_denominator": 69481, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.5, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8694, "material_floor": 100, "DISALLOWED": 8665, "NOT_DISALLOWED_BY_TESTED_RULE": 60816, "UNKNOWN": 30519, "known_denominator": 69481, "panel_denominator": 100000, "disallowed_pct_of_known": 12.5, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.5, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4522, "material_floor": 100, "DISALLOWED": 4182, "NOT_DISALLOWED_BY_TESTED_RULE": 65299, "UNKNOWN": 30519, "known_denominator": 69481, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.5, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-25", "collection_batch_sha256": "f973efd65f0cefbeda9b5d3f719d8e8999407285ec991e78592a0c505b948b10", "collection_batch_parts": [ { "run_id": "2026-07-25:2026-07-25T10:06:42.394921+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "f973efd65f0cefbeda9b5d3f719d8e8999407285ec991e78592a0c505b948b10", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "d86eef02750f76ecfb9c7229461f220c969b317fc28e1f096ee717248ecb2e96", "panel_denominator": 100000, "status_classes": { "status_2xx": 62319, "status_4xx": 12916, "status_5xx": 533, "transport_error": 24085, "other_status": 147 }, "unknown_reasons": { "transport_error": 24085, "server_error": 533, "redirect_or_other_status": 147, "missing_body": 726, "truncated_body": 392, "html_soft_200": 4327, "invalid_utf8": 229 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10852, "material_floor": 100, "DISALLOWED": 10207, "NOT_DISALLOWED_BY_TESTED_RULE": 59354, "UNKNOWN": 30439, "known_denominator": 69561, "panel_denominator": 100000, "disallowed_pct_of_known": 14.7, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.4, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9564, "material_floor": 100, "DISALLOWED": 9313, "NOT_DISALLOWED_BY_TESTED_RULE": 60248, "UNKNOWN": 30439, "known_denominator": 69561, "panel_denominator": 100000, "disallowed_pct_of_known": 13.4, "disallowed_pct_of_panel": 9.3, "unknown_pct_of_panel": 30.4, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9335, "material_floor": 100, "DISALLOWED": 10086, "NOT_DISALLOWED_BY_TESTED_RULE": 59475, "UNKNOWN": 30439, "known_denominator": 69561, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.4, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8715, "material_floor": 100, "DISALLOWED": 8689, "NOT_DISALLOWED_BY_TESTED_RULE": 60872, "UNKNOWN": 30439, "known_denominator": 69561, "panel_denominator": 100000, "disallowed_pct_of_known": 12.5, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.4, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4529, "material_floor": 100, "DISALLOWED": 4189, "NOT_DISALLOWED_BY_TESTED_RULE": 65372, "UNKNOWN": 30439, "known_denominator": 69561, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.4, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-26", "collection_batch_sha256": "897d4ac668fe3b851624b3c1b16c75b7b740f12c9e2ccd6f0608b80f1d7e7637", "collection_batch_parts": [ { "run_id": "2026-07-26:2026-07-26T10:01:42.861176+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "897d4ac668fe3b851624b3c1b16c75b7b740f12c9e2ccd6f0608b80f1d7e7637", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "44216a71e230e1f505cddf1aff380cb9ddab24ce03c840f5038df8322759c1bc", "panel_denominator": 100000, "status_classes": { "status_2xx": 62308, "status_4xx": 12874, "status_5xx": 497, "transport_error": 24174, "other_status": 147 }, "unknown_reasons": { "transport_error": 24174, "server_error": 497, "redirect_or_other_status": 147, "missing_body": 726, "truncated_body": 395, "html_soft_200": 4321, "invalid_utf8": 235 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10833, "material_floor": 100, "DISALLOWED": 10194, "NOT_DISALLOWED_BY_TESTED_RULE": 59311, "UNKNOWN": 30495, "known_denominator": 69505, "panel_denominator": 100000, "disallowed_pct_of_known": 14.7, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.5, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9553, "material_floor": 100, "DISALLOWED": 9305, "NOT_DISALLOWED_BY_TESTED_RULE": 60200, "UNKNOWN": 30495, "known_denominator": 69505, "panel_denominator": 100000, "disallowed_pct_of_known": 13.4, "disallowed_pct_of_panel": 9.3, "unknown_pct_of_panel": 30.5, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9332, "material_floor": 100, "DISALLOWED": 10080, "NOT_DISALLOWED_BY_TESTED_RULE": 59425, "UNKNOWN": 30495, "known_denominator": 69505, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.5, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8708, "material_floor": 100, "DISALLOWED": 8681, "NOT_DISALLOWED_BY_TESTED_RULE": 60824, "UNKNOWN": 30495, "known_denominator": 69505, "panel_denominator": 100000, "disallowed_pct_of_known": 12.5, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.5, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4525, "material_floor": 100, "DISALLOWED": 4185, "NOT_DISALLOWED_BY_TESTED_RULE": 65320, "UNKNOWN": 30495, "known_denominator": 69505, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.5, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-27", "collection_batch_sha256": "8d26f46e738489ba350a8c9109f13ed236d4d977a0aa7006e52974029c4684d1", "collection_batch_parts": [ { "run_id": "2026-07-27:2026-07-27T10:05:43.117517+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "8d26f46e738489ba350a8c9109f13ed236d4d977a0aa7006e52974029c4684d1", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "0fef6858ac0d8c5e4f7e836650091937e2538c8df6a33072d99e9131afb5a9fa", "panel_denominator": 100000, "status_classes": { "status_2xx": 62243, "status_4xx": 12828, "status_5xx": 480, "transport_error": 24307, "other_status": 142 }, "unknown_reasons": { "transport_error": 24307, "server_error": 480, "redirect_or_other_status": 142, "missing_body": 727, "truncated_body": 390, "html_soft_200": 4398, "invalid_utf8": 227 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10833, "material_floor": 100, "DISALLOWED": 10182, "NOT_DISALLOWED_BY_TESTED_RULE": 59147, "UNKNOWN": 30671, "known_denominator": 69329, "panel_denominator": 100000, "disallowed_pct_of_known": 14.7, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.7, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9563, "material_floor": 100, "DISALLOWED": 9302, "NOT_DISALLOWED_BY_TESTED_RULE": 60027, "UNKNOWN": 30671, "known_denominator": 69329, "panel_denominator": 100000, "disallowed_pct_of_known": 13.4, "disallowed_pct_of_panel": 9.3, "unknown_pct_of_panel": 30.7, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9343, "material_floor": 100, "DISALLOWED": 10082, "NOT_DISALLOWED_BY_TESTED_RULE": 59247, "UNKNOWN": 30671, "known_denominator": 69329, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.7, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8719, "material_floor": 100, "DISALLOWED": 8680, "NOT_DISALLOWED_BY_TESTED_RULE": 60649, "UNKNOWN": 30671, "known_denominator": 69329, "panel_denominator": 100000, "disallowed_pct_of_known": 12.5, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.7, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4528, "material_floor": 100, "DISALLOWED": 4177, "NOT_DISALLOWED_BY_TESTED_RULE": 65152, "UNKNOWN": 30671, "known_denominator": 69329, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.7, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-28", "collection_batch_sha256": "1b8c267a1b5a1887434b094b080e0e16b5701e5c399942a4c2b23175e8f1f71a", "collection_batch_parts": [ { "run_id": "2026-07-28:2026-07-28T10:08:43.458044+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "1b8c267a1b5a1887434b094b080e0e16b5701e5c399942a4c2b23175e8f1f71a", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "b92dd5e821467d8177a2172b3a76bf3bb8eaef6447a88d770e2b54d9fba6f110", "panel_denominator": 100000, "status_classes": { "status_2xx": 62434, "status_4xx": 12868, "status_5xx": 492, "transport_error": 24054, "other_status": 152 }, "unknown_reasons": { "transport_error": 24054, "server_error": 492, "redirect_or_other_status": 152, "missing_body": 734, "truncated_body": 388, "html_soft_200": 4468, "invalid_utf8": 229 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10856, "material_floor": 100, "DISALLOWED": 10208, "NOT_DISALLOWED_BY_TESTED_RULE": 59275, "UNKNOWN": 30517, "known_denominator": 69483, "panel_denominator": 100000, "disallowed_pct_of_known": 14.7, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.5, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9579, "material_floor": 100, "DISALLOWED": 9323, "NOT_DISALLOWED_BY_TESTED_RULE": 60160, "UNKNOWN": 30517, "known_denominator": 69483, "panel_denominator": 100000, "disallowed_pct_of_known": 13.4, "disallowed_pct_of_panel": 9.3, "unknown_pct_of_panel": 30.5, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9359, "material_floor": 100, "DISALLOWED": 10099, "NOT_DISALLOWED_BY_TESTED_RULE": 59384, "UNKNOWN": 30517, "known_denominator": 69483, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.5, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8729, "material_floor": 100, "DISALLOWED": 8692, "NOT_DISALLOWED_BY_TESTED_RULE": 60791, "UNKNOWN": 30517, "known_denominator": 69483, "panel_denominator": 100000, "disallowed_pct_of_known": 12.5, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.5, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4538, "material_floor": 100, "DISALLOWED": 4188, "NOT_DISALLOWED_BY_TESTED_RULE": 65295, "UNKNOWN": 30517, "known_denominator": 69483, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.5, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-29", "collection_batch_sha256": "96bb8d479500236c234397e4a88a30453a2bcae70a9aab849ad5afc1bae6f8f2", "collection_batch_parts": [ { "run_id": "2026-07-29:2026-07-29T10:00:46.779870+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "96bb8d479500236c234397e4a88a30453a2bcae70a9aab849ad5afc1bae6f8f2", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "7ba1696f5f5087f51685074669af92ec9f82578497debdba5a36b30117877c48", "panel_denominator": 100000, "status_classes": { "status_2xx": 62354, "status_4xx": 12822, "status_5xx": 494, "transport_error": 24184, "other_status": 146 }, "unknown_reasons": { "transport_error": 24184, "server_error": 494, "redirect_or_other_status": 146, "missing_body": 720, "truncated_body": 389, "html_soft_200": 4463, "invalid_utf8": 229 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10864, "material_floor": 100, "DISALLOWED": 10211, "NOT_DISALLOWED_BY_TESTED_RULE": 59164, "UNKNOWN": 30625, "known_denominator": 69375, "panel_denominator": 100000, "disallowed_pct_of_known": 14.7, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.6, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9584, "material_floor": 100, "DISALLOWED": 9326, "NOT_DISALLOWED_BY_TESTED_RULE": 60049, "UNKNOWN": 30625, "known_denominator": 69375, "panel_denominator": 100000, "disallowed_pct_of_known": 13.4, "disallowed_pct_of_panel": 9.3, "unknown_pct_of_panel": 30.6, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9363, "material_floor": 100, "DISALLOWED": 10111, "NOT_DISALLOWED_BY_TESTED_RULE": 59264, "UNKNOWN": 30625, "known_denominator": 69375, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.6, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8736, "material_floor": 100, "DISALLOWED": 8700, "NOT_DISALLOWED_BY_TESTED_RULE": 60675, "UNKNOWN": 30625, "known_denominator": 69375, "panel_denominator": 100000, "disallowed_pct_of_known": 12.5, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.6, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4547, "material_floor": 100, "DISALLOWED": 4192, "NOT_DISALLOWED_BY_TESTED_RULE": 65183, "UNKNOWN": 30625, "known_denominator": 69375, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.6, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-30", "collection_batch_sha256": "fd39902aad32a001bbd0c0cd01fa525f80625acf958c61a0c458cb06511ef877", "collection_batch_parts": [ { "run_id": "2026-07-30:2026-07-30T10:04:25.973704+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "fd39902aad32a001bbd0c0cd01fa525f80625acf958c61a0c458cb06511ef877", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "662e25d50a14f81b296901b056002fc4337477ba02871fb96dd6e01a64e39907", "panel_denominator": 100000, "status_classes": { "status_2xx": 62317, "status_4xx": 12806, "status_5xx": 488, "transport_error": 24239, "other_status": 150 }, "unknown_reasons": { "transport_error": 24239, "server_error": 488, "redirect_or_other_status": 150, "missing_body": 721, "truncated_body": 385, "html_soft_200": 4458, "invalid_utf8": 235 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10872, "material_floor": 100, "DISALLOWED": 10208, "NOT_DISALLOWED_BY_TESTED_RULE": 59116, "UNKNOWN": 30676, "known_denominator": 69324, "panel_denominator": 100000, "disallowed_pct_of_known": 14.7, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.7, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9596, "material_floor": 100, "DISALLOWED": 9324, "NOT_DISALLOWED_BY_TESTED_RULE": 60000, "UNKNOWN": 30676, "known_denominator": 69324, "panel_denominator": 100000, "disallowed_pct_of_known": 13.4, "disallowed_pct_of_panel": 9.3, "unknown_pct_of_panel": 30.7, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9359, "material_floor": 100, "DISALLOWED": 10101, "NOT_DISALLOWED_BY_TESTED_RULE": 59223, "UNKNOWN": 30676, "known_denominator": 69324, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.7, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8742, "material_floor": 100, "DISALLOWED": 8700, "NOT_DISALLOWED_BY_TESTED_RULE": 60624, "UNKNOWN": 30676, "known_denominator": 69324, "panel_denominator": 100000, "disallowed_pct_of_known": 12.5, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.7, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4563, "material_floor": 100, "DISALLOWED": 4202, "NOT_DISALLOWED_BY_TESTED_RULE": 65122, "UNKNOWN": 30676, "known_denominator": 69324, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.7, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-07-31", "collection_batch_sha256": "15e86f2e37124655b3d874e5b409ef7cd753cffa0622180b357d4c2a955e2fec", "collection_batch_parts": [ { "run_id": "2026-07-31:2026-07-31T10:07:16.229471+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "15e86f2e37124655b3d874e5b409ef7cd753cffa0622180b357d4c2a955e2fec", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "d8d79123657c3012afa2b3a407863542111bfb6615ac9a6217072fe049626f2e", "panel_denominator": 100000, "status_classes": { "status_2xx": 62468, "status_4xx": 12849, "status_5xx": 492, "transport_error": 24042, "other_status": 149 }, "unknown_reasons": { "transport_error": 24042, "server_error": 492, "redirect_or_other_status": 149, "missing_body": 723, "truncated_body": 397, "html_soft_200": 4473, "invalid_utf8": 236 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10890, "material_floor": 100, "DISALLOWED": 10251, "NOT_DISALLOWED_BY_TESTED_RULE": 59237, "UNKNOWN": 30512, "known_denominator": 69488, "panel_denominator": 100000, "disallowed_pct_of_known": 14.8, "disallowed_pct_of_panel": 10.3, "unknown_pct_of_panel": 30.5, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9610, "material_floor": 100, "DISALLOWED": 9368, "NOT_DISALLOWED_BY_TESTED_RULE": 60120, "UNKNOWN": 30512, "known_denominator": 69488, "panel_denominator": 100000, "disallowed_pct_of_known": 13.5, "disallowed_pct_of_panel": 9.4, "unknown_pct_of_panel": 30.5, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9367, "material_floor": 100, "DISALLOWED": 10146, "NOT_DISALLOWED_BY_TESTED_RULE": 59342, "UNKNOWN": 30512, "known_denominator": 69488, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.5, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8766, "material_floor": 100, "DISALLOWED": 8741, "NOT_DISALLOWED_BY_TESTED_RULE": 60747, "UNKNOWN": 30512, "known_denominator": 69488, "panel_denominator": 100000, "disallowed_pct_of_known": 12.6, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.5, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4591, "material_floor": 100, "DISALLOWED": 4206, "NOT_DISALLOWED_BY_TESTED_RULE": 65282, "UNKNOWN": 30512, "known_denominator": 69488, "panel_denominator": 100000, "disallowed_pct_of_known": 6.1, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.5, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-08-01", "collection_batch_sha256": "89ddf055cc75be9751ab8e2f08ce7fc91b9c5746263ce1917c0b1cfd0549acc4", "collection_batch_parts": [ { "run_id": "2026-08-01:2026-08-01T10:07:34.229994+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "89ddf055cc75be9751ab8e2f08ce7fc91b9c5746263ce1917c0b1cfd0549acc4", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "0ee07b7e8471c4f88cce749c3bfd8f4e3ead1686808db8bc6d4bbedd5ea168f1", "panel_denominator": 100000, "status_classes": { "status_2xx": 62183, "status_4xx": 12824, "status_5xx": 491, "transport_error": 24357, "other_status": 145 }, "unknown_reasons": { "transport_error": 24357, "server_error": 491, "redirect_or_other_status": 145, "missing_body": 708, "truncated_body": 393, "html_soft_200": 4440, "invalid_utf8": 234 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10837, "material_floor": 100, "DISALLOWED": 10180, "NOT_DISALLOWED_BY_TESTED_RULE": 59052, "UNKNOWN": 30768, "known_denominator": 69232, "panel_denominator": 100000, "disallowed_pct_of_known": 14.7, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.8, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9567, "material_floor": 100, "DISALLOWED": 9292, "NOT_DISALLOWED_BY_TESTED_RULE": 59940, "UNKNOWN": 30768, "known_denominator": 69232, "panel_denominator": 100000, "disallowed_pct_of_known": 13.4, "disallowed_pct_of_panel": 9.3, "unknown_pct_of_panel": 30.8, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9324, "material_floor": 100, "DISALLOWED": 10068, "NOT_DISALLOWED_BY_TESTED_RULE": 59164, "UNKNOWN": 30768, "known_denominator": 69232, "panel_denominator": 100000, "disallowed_pct_of_known": 14.5, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.8, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8717, "material_floor": 100, "DISALLOWED": 8664, "NOT_DISALLOWED_BY_TESTED_RULE": 60568, "UNKNOWN": 30768, "known_denominator": 69232, "panel_denominator": 100000, "disallowed_pct_of_known": 12.5, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.8, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4562, "material_floor": 100, "DISALLOWED": 4159, "NOT_DISALLOWED_BY_TESTED_RULE": 65073, "UNKNOWN": 30768, "known_denominator": 69232, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.8, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-08-02", "collection_batch_sha256": "768dedbc309405bed4458c826270e6bab9dca8f41afecfb28843eaf7f759dbb8", "collection_batch_parts": [ { "run_id": "2026-08-02:2026-08-02T10:01:47.090442+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "768dedbc309405bed4458c826270e6bab9dca8f41afecfb28843eaf7f759dbb8", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "10f7f5ba1730cc5b04dea24410ac0a7d761a7354003efcd8a706d9ddd0c5ca42", "panel_denominator": 100000, "status_classes": { "status_2xx": 62437, "status_4xx": 12866, "status_5xx": 478, "transport_error": 24072, "other_status": 147 }, "unknown_reasons": { "transport_error": 24072, "server_error": 478, "redirect_or_other_status": 147, "missing_body": 728, "truncated_body": 387, "html_soft_200": 4475, "invalid_utf8": 227 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10882, "material_floor": 100, "DISALLOWED": 10247, "NOT_DISALLOWED_BY_TESTED_RULE": 59239, "UNKNOWN": 30514, "known_denominator": 69486, "panel_denominator": 100000, "disallowed_pct_of_known": 14.7, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.5, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9606, "material_floor": 100, "DISALLOWED": 9356, "NOT_DISALLOWED_BY_TESTED_RULE": 60130, "UNKNOWN": 30514, "known_denominator": 69486, "panel_denominator": 100000, "disallowed_pct_of_known": 13.5, "disallowed_pct_of_panel": 9.4, "unknown_pct_of_panel": 30.5, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9355, "material_floor": 100, "DISALLOWED": 10136, "NOT_DISALLOWED_BY_TESTED_RULE": 59350, "UNKNOWN": 30514, "known_denominator": 69486, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.5, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8755, "material_floor": 100, "DISALLOWED": 8731, "NOT_DISALLOWED_BY_TESTED_RULE": 60755, "UNKNOWN": 30514, "known_denominator": 69486, "panel_denominator": 100000, "disallowed_pct_of_known": 12.6, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.5, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4580, "material_floor": 100, "DISALLOWED": 4200, "NOT_DISALLOWED_BY_TESTED_RULE": 65286, "UNKNOWN": 30514, "known_denominator": 69486, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.5, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-08-03", "collection_batch_sha256": "db731e07879a33d07ed86c89f951c8c45998687b330dc6b5e73850d1f8e1caed", "collection_batch_parts": [ { "run_id": "2026-08-03:2026-08-03T10:09:58.724214+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "db731e07879a33d07ed86c89f951c8c45998687b330dc6b5e73850d1f8e1caed", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "4c25ef8ac62f58e334ab442758fe4298a07b53f297c7a2a21f481dd81e1ceb1e", "panel_denominator": 100000, "status_classes": { "status_2xx": 62337, "status_4xx": 12873, "status_5xx": 487, "transport_error": 24157, "other_status": 146 }, "unknown_reasons": { "transport_error": 24157, "server_error": 487, "redirect_or_other_status": 146, "missing_body": 727, "truncated_body": 387, "html_soft_200": 4387, "invalid_utf8": 231 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10896, "material_floor": 100, "DISALLOWED": 10246, "NOT_DISALLOWED_BY_TESTED_RULE": 59232, "UNKNOWN": 30522, "known_denominator": 69478, "panel_denominator": 100000, "disallowed_pct_of_known": 14.7, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.5, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9622, "material_floor": 100, "DISALLOWED": 9364, "NOT_DISALLOWED_BY_TESTED_RULE": 60114, "UNKNOWN": 30522, "known_denominator": 69478, "panel_denominator": 100000, "disallowed_pct_of_known": 13.5, "disallowed_pct_of_panel": 9.4, "unknown_pct_of_panel": 30.5, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9375, "material_floor": 100, "DISALLOWED": 10148, "NOT_DISALLOWED_BY_TESTED_RULE": 59330, "UNKNOWN": 30522, "known_denominator": 69478, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.5, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8771, "material_floor": 100, "DISALLOWED": 8739, "NOT_DISALLOWED_BY_TESTED_RULE": 60739, "UNKNOWN": 30522, "known_denominator": 69478, "panel_denominator": 100000, "disallowed_pct_of_known": 12.6, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.5, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4587, "material_floor": 100, "DISALLOWED": 4187, "NOT_DISALLOWED_BY_TESTED_RULE": 65291, "UNKNOWN": 30522, "known_denominator": 69478, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.5, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-08-04", "collection_batch_sha256": "96ee475f43bc310458de9cd1db0cd67629c88b2cb47c73f037dff3f44754df23", "collection_batch_parts": [ { "run_id": "2026-08-04:2026-08-04T10:03:26.225339+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "96ee475f43bc310458de9cd1db0cd67629c88b2cb47c73f037dff3f44754df23", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "d60a43a97e65b98da64ad7cb22374fa551a4b603052fdf195835e5ab8e128b6b", "panel_denominator": 100000, "status_classes": { "status_2xx": 62352, "status_4xx": 12899, "status_5xx": 478, "transport_error": 24128, "other_status": 143 }, "unknown_reasons": { "transport_error": 24128, "server_error": 478, "redirect_or_other_status": 143, "missing_body": 724, "truncated_body": 392, "html_soft_200": 4314, "invalid_utf8": 238 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10902, "material_floor": 100, "DISALLOWED": 10258, "NOT_DISALLOWED_BY_TESTED_RULE": 59325, "UNKNOWN": 30417, "known_denominator": 69583, "panel_denominator": 100000, "disallowed_pct_of_known": 14.7, "disallowed_pct_of_panel": 10.3, "unknown_pct_of_panel": 30.4, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9635, "material_floor": 100, "DISALLOWED": 9374, "NOT_DISALLOWED_BY_TESTED_RULE": 60209, "UNKNOWN": 30417, "known_denominator": 69583, "panel_denominator": 100000, "disallowed_pct_of_known": 13.5, "disallowed_pct_of_panel": 9.4, "unknown_pct_of_panel": 30.4, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9374, "material_floor": 100, "DISALLOWED": 10160, "NOT_DISALLOWED_BY_TESTED_RULE": 59423, "UNKNOWN": 30417, "known_denominator": 69583, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.4, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8774, "material_floor": 100, "DISALLOWED": 8746, "NOT_DISALLOWED_BY_TESTED_RULE": 60837, "UNKNOWN": 30417, "known_denominator": 69583, "panel_denominator": 100000, "disallowed_pct_of_known": 12.6, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.4, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4593, "material_floor": 100, "DISALLOWED": 4194, "NOT_DISALLOWED_BY_TESTED_RULE": 65389, "UNKNOWN": 30417, "known_denominator": 69583, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.4, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-08-05", "collection_batch_sha256": "71695428201ec82a77101e6902163bdc07ae24f2c16660f999d455161854ec30", "collection_batch_parts": [ { "run_id": "2026-08-05:2026-08-05T10:01:39.511349+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "71695428201ec82a77101e6902163bdc07ae24f2c16660f999d455161854ec30", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "dab281489a8f0730e2bda14a658e323df92b90b46ebd9bf41aa99204ae0216e6", "panel_denominator": 100000, "status_classes": { "status_2xx": 62258, "status_4xx": 12910, "status_5xx": 478, "transport_error": 24206, "other_status": 148 }, "unknown_reasons": { "transport_error": 24206, "server_error": 478, "redirect_or_other_status": 148, "missing_body": 722, "truncated_body": 400, "html_soft_200": 4313, "invalid_utf8": 238 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10913, "material_floor": 100, "DISALLOWED": 10244, "NOT_DISALLOWED_BY_TESTED_RULE": 59251, "UNKNOWN": 30505, "known_denominator": 69495, "panel_denominator": 100000, "disallowed_pct_of_known": 14.7, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.5, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9646, "material_floor": 100, "DISALLOWED": 9364, "NOT_DISALLOWED_BY_TESTED_RULE": 60131, "UNKNOWN": 30505, "known_denominator": 69495, "panel_denominator": 100000, "disallowed_pct_of_known": 13.5, "disallowed_pct_of_panel": 9.4, "unknown_pct_of_panel": 30.5, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9387, "material_floor": 100, "DISALLOWED": 10157, "NOT_DISALLOWED_BY_TESTED_RULE": 59338, "UNKNOWN": 30505, "known_denominator": 69495, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.5, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8793, "material_floor": 100, "DISALLOWED": 8740, "NOT_DISALLOWED_BY_TESTED_RULE": 60755, "UNKNOWN": 30505, "known_denominator": 69495, "panel_denominator": 100000, "disallowed_pct_of_known": 12.6, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.5, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4598, "material_floor": 100, "DISALLOWED": 4178, "NOT_DISALLOWED_BY_TESTED_RULE": 65317, "UNKNOWN": 30505, "known_denominator": 69495, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.5, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-08-06", "collection_batch_sha256": "2e10ea54b0cbc2813c93c3192b0219a8e5d12787d03f803eb78ee429fe49fafa", "collection_batch_parts": [ { "run_id": "2026-08-06:2026-08-06T10:00:36.999015+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "2e10ea54b0cbc2813c93c3192b0219a8e5d12787d03f803eb78ee429fe49fafa", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "2a8d2036ed28cd0b41b693aca49abf7165def9fb7becfd06f0b777b7a346cbfd", "panel_denominator": 100000, "status_classes": { "status_2xx": 62144, "status_4xx": 12970, "status_5xx": 473, "transport_error": 24265, "other_status": 148 }, "unknown_reasons": { "transport_error": 24265, "server_error": 473, "redirect_or_other_status": 148, "missing_body": 714, "truncated_body": 392, "html_soft_200": 4312, "invalid_utf8": 233 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10891, "material_floor": 100, "DISALLOWED": 10222, "NOT_DISALLOWED_BY_TESTED_RULE": 59241, "UNKNOWN": 30537, "known_denominator": 69463, "panel_denominator": 100000, "disallowed_pct_of_known": 14.7, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.5, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9623, "material_floor": 100, "DISALLOWED": 9342, "NOT_DISALLOWED_BY_TESTED_RULE": 60121, "UNKNOWN": 30537, "known_denominator": 69463, "panel_denominator": 100000, "disallowed_pct_of_known": 13.4, "disallowed_pct_of_panel": 9.3, "unknown_pct_of_panel": 30.5, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9359, "material_floor": 100, "DISALLOWED": 10127, "NOT_DISALLOWED_BY_TESTED_RULE": 59336, "UNKNOWN": 30537, "known_denominator": 69463, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.5, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8790, "material_floor": 100, "DISALLOWED": 8734, "NOT_DISALLOWED_BY_TESTED_RULE": 60729, "UNKNOWN": 30537, "known_denominator": 69463, "panel_denominator": 100000, "disallowed_pct_of_known": 12.6, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.5, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4595, "material_floor": 100, "DISALLOWED": 4168, "NOT_DISALLOWED_BY_TESTED_RULE": 65295, "UNKNOWN": 30537, "known_denominator": 69463, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.5, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-08-07", "collection_batch_sha256": "75fb56e1003565618421147859ed2ce12c98feba98ce92ea75a82bf837fb63a9", "collection_batch_parts": [ { "run_id": "2026-08-07:2026-08-07T10:03:40.499900+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "75fb56e1003565618421147859ed2ce12c98feba98ce92ea75a82bf837fb63a9", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "1a9d75f96f8bcacb84dac4557a2c2da5bd58f1b831848768abd5037625987d0a", "panel_denominator": 100000, "status_classes": { "status_2xx": 62288, "status_4xx": 12962, "status_5xx": 474, "transport_error": 24130, "other_status": 146 }, "unknown_reasons": { "transport_error": 24130, "server_error": 474, "redirect_or_other_status": 146, "missing_body": 726, "truncated_body": 399, "html_soft_200": 4323, "invalid_utf8": 231 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10944, "material_floor": 100, "DISALLOWED": 10264, "NOT_DISALLOWED_BY_TESTED_RULE": 59307, "UNKNOWN": 30429, "known_denominator": 69571, "panel_denominator": 100000, "disallowed_pct_of_known": 14.8, "disallowed_pct_of_panel": 10.3, "unknown_pct_of_panel": 30.4, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9675, "material_floor": 100, "DISALLOWED": 9384, "NOT_DISALLOWED_BY_TESTED_RULE": 60187, "UNKNOWN": 30429, "known_denominator": 69571, "panel_denominator": 100000, "disallowed_pct_of_known": 13.5, "disallowed_pct_of_panel": 9.4, "unknown_pct_of_panel": 30.4, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9411, "material_floor": 100, "DISALLOWED": 10175, "NOT_DISALLOWED_BY_TESTED_RULE": 59396, "UNKNOWN": 30429, "known_denominator": 69571, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.4, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8847, "material_floor": 100, "DISALLOWED": 8778, "NOT_DISALLOWED_BY_TESTED_RULE": 60793, "UNKNOWN": 30429, "known_denominator": 69571, "panel_denominator": 100000, "disallowed_pct_of_known": 12.6, "disallowed_pct_of_panel": 8.8, "unknown_pct_of_panel": 30.4, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4606, "material_floor": 100, "DISALLOWED": 4165, "NOT_DISALLOWED_BY_TESTED_RULE": 65406, "UNKNOWN": 30429, "known_denominator": 69571, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.2, "unknown_pct_of_panel": 30.4, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-08-09", "collection_batch_sha256": "207cd58a0a8ae0e8acc466157be8875ce7692735c54e3be3cab41e3349d65538", "collection_batch_parts": [ { "run_id": "2026-08-09:2026-08-09T10:00:32.628430+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "207cd58a0a8ae0e8acc466157be8875ce7692735c54e3be3cab41e3349d65538", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "f40a23683a532366f157415bec72b81ff49d891a2757fb640cd758fe0c666dba", "panel_denominator": 100000, "status_classes": { "status_2xx": 62007, "status_4xx": 12959, "status_5xx": 479, "transport_error": 24414, "other_status": 141 }, "unknown_reasons": { "transport_error": 24414, "server_error": 479, "redirect_or_other_status": 141, "missing_body": 704, "truncated_body": 395, "html_soft_200": 4299, "invalid_utf8": 225 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10911, "material_floor": 100, "DISALLOWED": 10211, "NOT_DISALLOWED_BY_TESTED_RULE": 59132, "UNKNOWN": 30657, "known_denominator": 69343, "panel_denominator": 100000, "disallowed_pct_of_known": 14.7, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.7, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9655, "material_floor": 100, "DISALLOWED": 9341, "NOT_DISALLOWED_BY_TESTED_RULE": 60002, "UNKNOWN": 30657, "known_denominator": 69343, "panel_denominator": 100000, "disallowed_pct_of_known": 13.5, "disallowed_pct_of_panel": 9.3, "unknown_pct_of_panel": 30.7, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9380, "material_floor": 100, "DISALLOWED": 10120, "NOT_DISALLOWED_BY_TESTED_RULE": 59223, "UNKNOWN": 30657, "known_denominator": 69343, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.7, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8818, "material_floor": 100, "DISALLOWED": 8732, "NOT_DISALLOWED_BY_TESTED_RULE": 60611, "UNKNOWN": 30657, "known_denominator": 69343, "panel_denominator": 100000, "disallowed_pct_of_known": 12.6, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.7, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4607, "material_floor": 100, "DISALLOWED": 4148, "NOT_DISALLOWED_BY_TESTED_RULE": 65195, "UNKNOWN": 30657, "known_denominator": 69343, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.1, "unknown_pct_of_panel": 30.7, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-08-10", "collection_batch_sha256": "4958345b1ab4a4753d2a171341bb01744a158061d65203610a4e50a529ff2bde", "collection_batch_parts": [ { "run_id": "2026-08-10:2026-08-10T10:02:06.229256+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "4958345b1ab4a4753d2a171341bb01744a158061d65203610a4e50a529ff2bde", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "d80486431daea65a1a446f88c0a7c1a7f57f598330f0affcd256350a50408038", "panel_denominator": 100000, "status_classes": { "status_2xx": 62078, "status_4xx": 12824, "status_5xx": 470, "transport_error": 24485, "other_status": 143 }, "unknown_reasons": { "transport_error": 24485, "server_error": 470, "redirect_or_other_status": 143, "missing_body": 703, "truncated_body": 394, "html_soft_200": 4388, "invalid_utf8": 225 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10926, "material_floor": 100, "DISALLOWED": 10213, "NOT_DISALLOWED_BY_TESTED_RULE": 58979, "UNKNOWN": 30808, "known_denominator": 69192, "panel_denominator": 100000, "disallowed_pct_of_known": 14.8, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.8, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9675, "material_floor": 100, "DISALLOWED": 9345, "NOT_DISALLOWED_BY_TESTED_RULE": 59847, "UNKNOWN": 30808, "known_denominator": 69192, "panel_denominator": 100000, "disallowed_pct_of_known": 13.5, "disallowed_pct_of_panel": 9.3, "unknown_pct_of_panel": 30.8, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9401, "material_floor": 100, "DISALLOWED": 10126, "NOT_DISALLOWED_BY_TESTED_RULE": 59066, "UNKNOWN": 30808, "known_denominator": 69192, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.8, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8834, "material_floor": 100, "DISALLOWED": 8733, "NOT_DISALLOWED_BY_TESTED_RULE": 60459, "UNKNOWN": 30808, "known_denominator": 69192, "panel_denominator": 100000, "disallowed_pct_of_known": 12.6, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.8, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4613, "material_floor": 100, "DISALLOWED": 4141, "NOT_DISALLOWED_BY_TESTED_RULE": 65051, "UNKNOWN": 30808, "known_denominator": 69192, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.1, "unknown_pct_of_panel": 30.8, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } }, { "snapshot_date": "2026-08-11", "collection_batch_sha256": "81b6ee750b6b1d913a77d3a57fed364d5215eeae857fac04f8e0ee00fa475887", "collection_batch_parts": [ { "run_id": "2026-08-11:2026-08-11T10:01:42.131256+00:00", "universe_version": "tranco_L8QG4_top100000.txt:8778484edc1487aa", "row_count": 400000, "batch_sha256": "81b6ee750b6b1d913a77d3a57fed364d5215eeae857fac04f8e0ee00fa475887", "collector_root_status": "VERIFIED" } ], "collection_manifest_status": "COLLECTOR_ROOTS_VERIFIED", "unmanifested_row_count": 0, "robots_rows_sha256": "718cca7fa413510cfff33147c93f56982d677d0164cafda77527bf5ba3fa68e7", "panel_denominator": 100000, "status_classes": { "status_2xx": 62158, "status_4xx": 12863, "status_5xx": 464, "transport_error": 24376, "other_status": 139 }, "unknown_reasons": { "transport_error": 24376, "server_error": 464, "redirect_or_other_status": 139, "missing_body": 701, "truncated_body": 400, "html_soft_200": 4443, "invalid_utf8": 235 }, "crawlers": { "GPTBot": { "token": "GPTBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 10926, "material_floor": 100, "DISALLOWED": 10214, "NOT_DISALLOWED_BY_TESTED_RULE": 59028, "UNKNOWN": 30758, "known_denominator": 69242, "panel_denominator": 100000, "disallowed_pct_of_known": 14.8, "disallowed_pct_of_panel": 10.2, "unknown_pct_of_panel": 30.8, "operator": "OpenAI" }, "ClaudeBot": { "token": "ClaudeBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9675, "material_floor": 100, "DISALLOWED": 9342, "NOT_DISALLOWED_BY_TESTED_RULE": 59900, "UNKNOWN": 30758, "known_denominator": 69242, "panel_denominator": 100000, "disallowed_pct_of_known": 13.5, "disallowed_pct_of_panel": 9.3, "unknown_pct_of_panel": 30.8, "operator": "Anthropic" }, "CCBot": { "token": "CCBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 9400, "material_floor": 100, "DISALLOWED": 10127, "NOT_DISALLOWED_BY_TESTED_RULE": 59115, "UNKNOWN": 30758, "known_denominator": 69242, "panel_denominator": 100000, "disallowed_pct_of_known": 14.6, "disallowed_pct_of_panel": 10.1, "unknown_pct_of_panel": 30.8, "operator": "Common Crawl" }, "Google-Extended": { "token": "Google-Extended", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 8833, "material_floor": 100, "DISALLOWED": 8729, "NOT_DISALLOWED_BY_TESTED_RULE": 60513, "UNKNOWN": 30758, "known_denominator": 69242, "panel_denominator": 100000, "disallowed_pct_of_known": 12.6, "disallowed_pct_of_panel": 8.7, "unknown_pct_of_panel": 30.8, "operator": "Google" }, "PerplexityBot": { "token": "PerplexityBot", "evidence_status": "OBSERVED", "reason": null, "exact_token_policy_count": 4615, "material_floor": 100, "DISALLOWED": 4140, "NOT_DISALLOWED_BY_TESTED_RULE": 65102, "UNKNOWN": 30758, "known_denominator": 69242, "panel_denominator": 100000, "disallowed_pct_of_known": 6, "disallowed_pct_of_panel": 4.1, "unknown_pct_of_panel": 30.8, "operator": "Perplexity" } }, "absent_control": { "token": "USTAAbsentControlBotZeta", "evidence_status": "UNKNOWN", "reason": "exact token appears in 0 policies; material floor is 100", "exact_token_policy_count": 0, "material_floor": 100, "DISALLOWED": null, "NOT_DISALLOWED_BY_TESTED_RULE": null, "UNKNOWN": null, "known_denominator": null, "panel_denominator": 100000, "disallowed_pct_of_known": null, "disallowed_pct_of_panel": null, "unknown_pct_of_panel": null } } ], "facts": { "honestyStatement": "Every count is re-derived from the append-only seal. Nothing is estimated, modeled, smoothed, interpolated, or extrapolated.", "privacyStatement": "Public artifacts contain aggregates and hashes only; stored bodies, headers, observed-site identities, and per-site verdicts are not serialized.", "denominatorStatement": "Known-policy rates divide DISALLOWED by DISALLOWED plus NOT_DISALLOWED_BY_TESTED_RULE. Panel shares use the full frozen panel. UNKNOWN counts and shares remain visible.", "comparisonStatement": "The ranking uses the latest known-policy disallow rate, with token name as a deterministic tie-breaker. It is descriptive, not a quality or compliance score." }, "ranking": [ { "token": "GPTBot", "known_policy_disallow_rate": 14.8 }, { "token": "CCBot", "known_policy_disallow_rate": 14.6 }, { "token": "ClaudeBot", "known_policy_disallow_rate": 13.5 }, { "token": "Google-Extended", "known_policy_disallow_rate": 12.6 }, { "token": "PerplexityBot", "known_policy_disallow_rate": 6 } ], "displays": [ { "key": "coverage.completeDates", "display": "58", "value": 58, "kind": "count" }, { "key": "coverage.panel", "display": "100,000", "value": 100000, "kind": "count" }, { "key": "coverage.firstDate", "display": "June 9, 2026", "value": null, "kind": "window" }, { "key": "coverage.latestDate", "display": "August 11, 2026", "value": null, "kind": "window" }, { "key": "coverage.gapDates", "display": "6", "value": 6, "kind": "count" }, { "key": "latest.GPTBot.exactTokenPolicies", "display": "10,926", "value": 10926, "kind": "count" }, { "key": "latest.GPTBot.DISALLOWED", "display": "10,214", "value": 10214, "kind": "count" }, { "key": "latest.GPTBot.NOT_DISALLOWED_BY_TESTED_RULE", "display": "59,028", "value": 59028, "kind": "count" }, { "key": "latest.GPTBot.UNKNOWN", "display": "30,758", "value": 30758, "kind": "count" }, { "key": "latest.GPTBot.known_denominator", "display": "69,242", "value": 69242, "kind": "count" }, { "key": "latest.GPTBot.panel_denominator", "display": "100,000", "value": 100000, "kind": "count" }, { "key": "latest.GPTBot.disallowed_pct_of_known", "display": "14.8%", "value": 14.8, "kind": "rate" }, { "key": "latest.GPTBot.disallowed_pct_of_panel", "display": "10.2%", "value": 10.2, "kind": "rate" }, { "key": "latest.GPTBot.unknown_pct_of_panel", "display": "30.8%", "value": 30.8, "kind": "rate" }, { "key": "trend.GPTBot.firstKnownRate", "display": "14.2%", "value": 14.2, "kind": "rate" }, { "key": "trend.GPTBot.latestKnownRate", "display": "14.8%", "value": 14.8, "kind": "rate" }, { "key": "trend.GPTBot.delta", "display": "+0.6", "value": 0.6, "kind": "delta" }, { "key": "latest.ClaudeBot.exactTokenPolicies", "display": "9,675", "value": 9675, "kind": "count" }, { "key": "latest.ClaudeBot.DISALLOWED", "display": "9,342", "value": 9342, "kind": "count" }, { "key": "latest.ClaudeBot.NOT_DISALLOWED_BY_TESTED_RULE", "display": "59,900", "value": 59900, "kind": "count" }, { "key": "latest.ClaudeBot.UNKNOWN", "display": "30,758", "value": 30758, "kind": "count" }, { "key": "latest.ClaudeBot.known_denominator", "display": "69,242", "value": 69242, "kind": "count" }, { "key": "latest.ClaudeBot.panel_denominator", "display": "100,000", "value": 100000, "kind": "count" }, { "key": "latest.ClaudeBot.disallowed_pct_of_known", "display": "13.5%", "value": 13.5, "kind": "rate" }, { "key": "latest.ClaudeBot.disallowed_pct_of_panel", "display": "9.3%", "value": 9.3, "kind": "rate" }, { "key": "latest.ClaudeBot.unknown_pct_of_panel", "display": "30.8%", "value": 30.8, "kind": "rate" }, { "key": "trend.ClaudeBot.firstKnownRate", "display": "12.8%", "value": 12.8, "kind": "rate" }, { "key": "trend.ClaudeBot.latestKnownRate", "display": "13.5%", "value": 13.5, "kind": "rate" }, { "key": "trend.ClaudeBot.delta", "display": "+0.7", "value": 0.7, "kind": "delta" }, { "key": "latest.CCBot.exactTokenPolicies", "display": "9,400", "value": 9400, "kind": "count" }, { "key": "latest.CCBot.DISALLOWED", "display": "10,127", "value": 10127, "kind": "count" }, { "key": "latest.CCBot.NOT_DISALLOWED_BY_TESTED_RULE", "display": "59,115", "value": 59115, "kind": "count" }, { "key": "latest.CCBot.UNKNOWN", "display": "30,758", "value": 30758, "kind": "count" }, { "key": "latest.CCBot.known_denominator", "display": "69,242", "value": 69242, "kind": "count" }, { "key": "latest.CCBot.panel_denominator", "display": "100,000", "value": 100000, "kind": "count" }, { "key": "latest.CCBot.disallowed_pct_of_known", "display": "14.6%", "value": 14.6, "kind": "rate" }, { "key": "latest.CCBot.disallowed_pct_of_panel", "display": "10.1%", "value": 10.1, "kind": "rate" }, { "key": "latest.CCBot.unknown_pct_of_panel", "display": "30.8%", "value": 30.8, "kind": "rate" }, { "key": "trend.CCBot.firstKnownRate", "display": "14%", "value": 14, "kind": "rate" }, { "key": "trend.CCBot.latestKnownRate", "display": "14.6%", "value": 14.6, "kind": "rate" }, { "key": "trend.CCBot.delta", "display": "+0.6", "value": 0.6, "kind": "delta" }, { "key": "latest.Google-Extended.exactTokenPolicies", "display": "8,833", "value": 8833, "kind": "count" }, { "key": "latest.Google-Extended.DISALLOWED", "display": "8,729", "value": 8729, "kind": "count" }, { "key": "latest.Google-Extended.NOT_DISALLOWED_BY_TESTED_RULE", "display": "60,513", "value": 60513, "kind": "count" }, { "key": "latest.Google-Extended.UNKNOWN", "display": "30,758", "value": 30758, "kind": "count" }, { "key": "latest.Google-Extended.known_denominator", "display": "69,242", "value": 69242, "kind": "count" }, { "key": "latest.Google-Extended.panel_denominator", "display": "100,000", "value": 100000, "kind": "count" }, { "key": "latest.Google-Extended.disallowed_pct_of_known", "display": "12.6%", "value": 12.6, "kind": "rate" }, { "key": "latest.Google-Extended.disallowed_pct_of_panel", "display": "8.7%", "value": 8.7, "kind": "rate" }, { "key": "latest.Google-Extended.unknown_pct_of_panel", "display": "30.8%", "value": 30.8, "kind": "rate" }, { "key": "trend.Google-Extended.firstKnownRate", "display": "12%", "value": 12, "kind": "rate" }, { "key": "trend.Google-Extended.latestKnownRate", "display": "12.6%", "value": 12.6, "kind": "rate" }, { "key": "trend.Google-Extended.delta", "display": "+0.6", "value": 0.6, "kind": "delta" }, { "key": "latest.PerplexityBot.exactTokenPolicies", "display": "4,615", "value": 4615, "kind": "count" }, { "key": "latest.PerplexityBot.DISALLOWED", "display": "4,140", "value": 4140, "kind": "count" }, { "key": "latest.PerplexityBot.NOT_DISALLOWED_BY_TESTED_RULE", "display": "65,102", "value": 65102, "kind": "count" }, { "key": "latest.PerplexityBot.UNKNOWN", "display": "30,758", "value": 30758, "kind": "count" }, { "key": "latest.PerplexityBot.known_denominator", "display": "69,242", "value": 69242, "kind": "count" }, { "key": "latest.PerplexityBot.panel_denominator", "display": "100,000", "value": 100000, "kind": "count" }, { "key": "latest.PerplexityBot.disallowed_pct_of_known", "display": "6%", "value": 6, "kind": "rate" }, { "key": "latest.PerplexityBot.disallowed_pct_of_panel", "display": "4.1%", "value": 4.1, "kind": "rate" }, { "key": "latest.PerplexityBot.unknown_pct_of_panel", "display": "30.8%", "value": 30.8, "kind": "rate" }, { "key": "trend.PerplexityBot.firstKnownRate", "display": "6.1%", "value": 6.1, "kind": "rate" }, { "key": "trend.PerplexityBot.latestKnownRate", "display": "6%", "value": 6, "kind": "rate" }, { "key": "trend.PerplexityBot.delta", "display": "-0.1", "value": -0.1, "kind": "delta" } ], "sha256": "305ffc125e3fa9f1f114e8a5fc64284525e3e52b79dadc56d8c9e38d84aad8bd" }