{ "row": "common-crawl", "entity": "common-crawl", "address": "Common Crawl", "addresses": [ { "address": "Common Crawl", "slug": "common-crawl-aio-20261101", "addr_id": "ADDR-68b16a5a5b51", "obs_id": "OBS-03ba6c1cdb8f", "axn": "AXN:0BAC.CAPTURE.⏰♆🎨♠️💜🟤" } ], "epoch": "2026-11-01", "type": "C (public entity on which the archive bears: a non-profit's open repository of web crawl data, the raw intake from which training datasets are built; an entity of the traininglayerliterature.org layer census, host_relation 'infrastructure')", "surface": "Google AI Overview (expanded)", "auth": "signed out, incognito", "status": { "procedure": "EA-NEGONT-02 v0.10 (#1726): equal case, ρ = extends, read from the traininglayerliterature.org layer census record, the capture record and the deposits that bear on the entity and its classes (#186 and #1543, the census's anchors; #722, #98, #623, #127 and others); archive selected by D/R/O (21 D candidates in 11 deposits, 1 R, 80 O candidates in 47 deposits, 94 hop candidates; 18 deposits admitted by reading with 1 instance kept), with the census record (37 claims) and the Capture Registry's findings, reading, analysis, title, PER note, query note and PER-dim basis (14 claims) ledgered as the archive's records of the entity; the field in two pools (B_D: 11 cards, 4 unreachable, the home page and Wikipedia among them; B_R: 1 own page); the composer read the ledgers, never the transcript, except where the brief asks for T's claims and the delta", "ledger": "unaudited (§3.8: one extractor per ledger); field quotes 79/79 verified by script against saved text, the wikitext or the card text; archive quotes 84/84 verified against the registry, census.json and the deposit texts", "frozen": false, "breaches": [] }, "spec": "EA-NEGONT-02 v0.10 (#1726)", "objects": { "T": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/T1-common-crawl-aio-20261101.txt", "sha256": "bfed4920bf4dd40b8f24571e2b41a6d2685a34eb163fab6ba5f2163eb9f31967", "words": 211, "cards": 11, "words_note": "one transcript at one address (211 words in the answer body, links reduced to their text, markers, the paste note and the card rail excluded; a closing offer of 26 words more; 20 markers; 11 cards, three of them video cards)", "claims": [ "T1-1 definition: 'Common Crawl is a 501(c)(3) non-profit organization founded in 2007 by Gil Elbaz that collects, preserves, and freely shares an open archive of internet data.' (two markers)", "T1-2 mission: 'Mission: To democratize access to web information for researchers, educators, and developers.' (two markers)", "T1-3 size: 'Archive Size: Contains over 10 to 11 petabytes of data spanning nearly two decades.' (two markers)", "T1-4 cadence: 'Frequency: Adds roughly 3 to 5 billion new web pages each month.' (one marker)", "T1-5 hosting: 'Hosting: Stored for free on Amazon Web Services (AWS) via its Open Data Sponsorship Program (s3://commoncrawl).' (two markers)", "T1-6 formats: 'Common Crawl distributes its monthly crawls in three primary file formats:' (no marker)", "T1-7 WARC: 'WARC (Web Archive): Raw HTTP responses, including full HTML content and headers.' (two markers)", "T1-8 WET: 'WET (Web Extracted Text): Cleaned plain-text content with HTML and code stripped away.' (two markers)", "T1-9 WAT: 'WAT (Web Archive Transformations): Metadata summaries in JSON format containing link and header details.' (two markers)", "T1-10 training: 'AI Training: The dataset serves as a foundational training corpus for many modern large language models (LLMs) developed by companies like OpenAI, Anthropic, Google, and Meta.' (two markers)", "T1-11 research: 'Research: Cited in over 10,000 academic and technical research papers.' (one marker)", "T1-12 paywalls: 'Web Scraping & Paywalls: Because its scraper (CCBot) does not execute dynamic browser code or login scripts, it has faced criticism and pushback from news publishers and media organizations over copyright and paywall bypass concerns.' (two markers)", "T1-13 closing offer: 'Would you like to know how to query the Common Crawl index for a specific URL, or how to block CCBot using a robots.txt file?' (no marker)" ], "register": { "register": "data/EA-WG-CAPTURES-01.json", "slug": "common-crawl-aio-20261101", "obs_id": "OBS-03ba6c1cdb8f" }, "transcripts": [ { "slug": "common-crawl-aio-20261101", "address": "Common Crawl", "addr_id": "ADDR-68b16a5a5b51", "obs_id": "OBS-03ba6c1cdb8f", "axn": "AXN:0BAC.CAPTURE.⏰♆🎨♠️💜🟤", "date": "2026-11-01", "surface": "Google AI Overview (expanded)", "auth": "signed out, incognito", "sha256": "bfed4920bf4dd40b8f24571e2b41a6d2685a34eb163fab6ba5f2163eb9f31967", "sha256_of": "the registry's 'transcript' string (utf-8)", "sha256_raw": "c950ef3dcd78dad52e2f64a00f2f26b3e3266b6922aa315758397854a295d7d5", "words": 211, "cards": 11, "card_rail": [ "1. Common Crawl — Common Crawl - Open Repository of Web Crawl Data", "2. Wikipedia — Common Crawl - Wikipedia", "3. Common Crawl — About", "4. The Atlantic — Common Crawl Is Doing the AI Industry's Dirty Work - The Atlantic", "5. Registry of Open Data on AWS — Common Crawl", "6. LinkedIn — Common Crawl Foundation | LinkedIn", "7. DEV Community — Inside Common Crawl: The Dataset Behind AI Models (and Its Real ...", "8. YouTube·PyTorch — Harnessing Common Crawl for AI and ML Applications - Pedro Ortiz Suarez ...", "9. YouTube·James Dooley — Common Crawl for AI Visibility Explained (James Dooley ...", "10. YouTube·Extract Summit — Exploring Common Crawl: The Web's Open Archive | Extract Data Live", "11. InfluenceWatch — Common Crawl - Influence Watch" ], "claims": [ "T1-1 definition: 'Common Crawl is a 501(c)(3) non-profit organization founded in 2007 by Gil Elbaz that collects, preserves, and freely shares an open archive of internet data.' (two markers)", "T1-2 mission: 'Mission: To democratize access to web information for researchers, educators, and developers.' (two markers)", "T1-3 size: 'Archive Size: Contains over 10 to 11 petabytes of data spanning nearly two decades.' (two markers)", "T1-4 cadence: 'Frequency: Adds roughly 3 to 5 billion new web pages each month.' (one marker)", "T1-5 hosting: 'Hosting: Stored for free on Amazon Web Services (AWS) via its Open Data Sponsorship Program (s3://commoncrawl).' (two markers)", "T1-6 formats: 'Common Crawl distributes its monthly crawls in three primary file formats:' (no marker)", "T1-7 WARC: 'WARC (Web Archive): Raw HTTP responses, including full HTML content and headers.' (two markers)", "T1-8 WET: 'WET (Web Extracted Text): Cleaned plain-text content with HTML and code stripped away.' (two markers)", "T1-9 WAT: 'WAT (Web Archive Transformations): Metadata summaries in JSON format containing link and header details.' (two markers)", "T1-10 training: 'AI Training: The dataset serves as a foundational training corpus for many modern large language models (LLMs) developed by companies like OpenAI, Anthropic, Google, and Meta.' (two markers)", "T1-11 research: 'Research: Cited in over 10,000 academic and technical research papers.' (one marker)", "T1-12 paywalls: 'Web Scraping & Paywalls: Because its scraper (CCBot) does not execute dynamic browser code or login scripts, it has faced criticism and pushback from news publishers and media organizations over copyright and paywall bypass concerns.' (two markers)", "T1-13 closing offer: 'Would you like to know how to query the Common Crawl index for a specific URL, or how to block CCBot using a robots.txt file?' (no marker)" ], "markers": 20, "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/T1-common-crawl-aio-20261101.txt", "note": "The paste is the Overview expanded (the 'AI Mode Conversation' view), surface and auth by the operator's defaults of 2026-10-01 and 2026-10-14 as the registry records them (surface_basis, auth_basis); run with the layer census names on 2026-11-01, the paste received 09:56 EDT" } ] }, "KO": { "title": "Common Crawl", "genre": "knowledge object, equal case, under EA-NEGONT-02 v0.10 (#1726): the final article form (§4.13), natural sentences under descriptive subheadings (§4.12), each sentence's sources beneath it by claim id (§4.5)", "status": "composed 2026-11-01 under v0.10 on the operator's ruling of 2026-10-29 ('as against their wikipedia articles if they exist, their own pages, etc.'); not frozen", "sections": [ { "para": 0, "heading": "what it is" }, { "para": 1, "heading": "the foundation: form, founder, governance" }, { "para": 1, "heading": "funding" }, { "para": 2, "heading": "dates: 2007, 2008, 2011" }, { "para": 3, "heading": "the crawler and its seeds" }, { "para": 4, "heading": "units, formats and indexes" }, { "para": 5, "heading": "size and cadence" }, { "para": 5, "heading": "research use and citation" }, { "para": 6, "heading": "hosting, access and licence" }, { "para": 7, "heading": "in the training layer" }, { "para": 8, "heading": "refined datasets downstream" }, { "para": 9, "heading": "what the crawl holds and leaves out" }, { "para": 10, "heading": "opt-out, removal and the index as witness" }, { "para": 11, "heading": "names and attribution" }, { "para": 12, "heading": "neighbours" }, { "para": 13, "heading": "what the census asks of it" }, { "para": 14, "heading": "reception and the composition at the name" } ], "sentences": [ { "n": 1, "para": 0, "sense": "what it is", "text": "Common Crawl is a free, open repository of web crawl data that anyone can use, maintained by the Common Crawl Foundation, a 501(c)(3) non-profit that crawls the web and freely provides its archives and datasets to the public.", "claims": [ "F1", "F10", "F35", "F69", "F60", "F68", "TLL-01", "TLL-02", "TLL-04" ] }, { "n": 2, "para": 0, "sense": "what it is", "text": "The foundation gives its purpose as making wholesale extraction, transformation and analysis of open web data accessible to researchers; its LinkedIn page speaks of democratizing access to web information and of levelling the playing field by making that work cheap and easy, and an LWN.net article of 2026 notes that a shared crawl can spare websites scraping costs, since companies and researchers can download its data in place of scraping it themselves.", "claims": [ "F3", "F56", "F57", "F27" ] }, { "n": 3, "para": 0, "sense": "what it is", "text": "The archive's layer census reads it as the receiving apparatus's first intake, the plainest case of the persistence criterion of #186, which asks that a work be placed where the training and retrieval layers actually look, with the metadata that lets it be found and attributed; the census enters it as infrastructure, a dataset whose question is training membership at the source.", "claims": [ "TLL-22", "O186-02", "TLL-05", "TLL-06", "TLL-07" ] }, { "n": 4, "para": 1, "sense": "the foundation: form, founder, governance", "text": "Gil Elbaz founded it in 2007 as a California-registered non-profit and chairs its board; Peter Norvig and Joi Ito have served on its advisory board, and Rich Skrenta, creator of the blekko search engine and founder of the Open Directory Project, became executive director in 2023.", "claims": [ "F2", "F12", "F56", "F51", "F29", "F42", "F74", "TLL-03" ] }, { "n": 5, "para": 1, "sense": "the foundation: form, founder, governance", "text": "Its location is given as Beverly Hills by InfluenceWatch, and its size as about 18 staff in 2026 on its About page, while its LinkedIn page lists 23 employees against a stated company size of 2 to 10.", "claims": [ "F74", "F51", "F58" ] }, { "n": 6, "para": 1, "sense": "funding", "text": "It was funded almost exclusively by the Elbaz Family Foundation Trust until 2023, and its About page still names the Elbaz Family Foundation as its primary funder; from 2023 it also received significant support from AI companies, among them Anthropic and OpenAI, which gave $250,000 each.", "claims": [ "F30", "F51", "F19" ] }, { "n": 7, "para": 2, "sense": "dates: 2007, 2008, 2011", "text": "Its own pages date the corpus two ways: the home page has a free and open corpus since 2007, the Overview and About pages data collected regularly since 2008; Wikipedia has the foundation founded in San Francisco in 2007, archiving websites that year and publishing its crawls from 2011.", "claims": [ "F5", "F75", "F35", "F15", "TLL-16" ] }, { "n": 8, "para": 3, "sense": "the crawler and its seeds", "text": "It began collecting in 2008 with a custom Hadoop-based crawler using PageRank, and in 2013 replaced it with CCBot, a crawler built on Apache Nutch, moving from the ARC to the WARC format with the November 2013 crawl.", "claims": [ "F39", "F40" ] }, { "n": 9, "para": 3, "sense": "the crawler and its seeds", "text": "Two accounts of its seeding stand side by side: Wikipedia says the crawl started from a list of URLs donated by the search engine Blekko, and counts 1.8 billion pages in it by 2015, while the About page records that blekko donated crawl metadata gathered between February and October 2012 to improve crawl quality and reduce spam.", "claims": [ "F17", "F39", "TLL-15" ] }, { "n": 10, "para": 3, "sense": "the crawler and its seeds", "text": "CCBot prioritises URLs by harmonic centrality, a measure of a page's proximity to the structural core of the web, and the foundation releases host- and domain-level web graphs as data, the latest of October 2026 covering August to October.", "claims": [ "F44", "F7", "TLL-18" ] }, { "n": 11, "para": 4, "sense": "units, formats and indexes", "text": "Its unit is the crawl, published about once a month as a numbered release (the latest listed is CC-MAIN-2026-43) of raw web page data, metadata extracts and text extracts, in which a page is addressed by its URL.", "claims": [ "TLL-09", "F76", "F77", "F35", "TLL-27" ] }, { "n": 12, "para": 4, "sense": "units, formats and indexes", "text": "The files come in three formats: WARC, full HTTP responses, append-only and immutable once written (ISO 28500); WAT, metadata extracted from the WARC records, among them headers and link graphs; and WET, plain text extracted from the crawled HTML, which the About page expands as WARC Encapsulated Text and a DEV Community post as Web Extracted Text, \"Ideal for NLP or training text based models.\"", "claims": [ "F45", "F63" ] }, { "n": 13, "para": 4, "sense": "units, formats and indexes", "text": "Monthly index files are released in a CDXJ Index and a URL Index, the latter queryable with Amazon Athena.", "claims": [ "F46" ] }, { "n": 14, "para": 5, "sense": "size and cadence", "text": "Its size is stated at several dates and grains: over 300 billion pages spanning 15 years, with 3 to 5 billion new pages a month, on the home page, which the card surfacing that page renders as spanning 19 years; more than 10 petabytes and more than two billion pages per crawl on the About page; over 300 billion pages, updated monthly, in AWS's registry; and 2.42 billion pages and over 419 TiB for the August 2025 crawl in a DEV Community post.", "claims": [ "F4", "F8", "F36", "F35", "F53", "F61", "TLL-10", "TLL-17" ] }, { "n": 15, "para": 5, "sense": "research use and citation", "text": "It is cited in over 10,000 research papers by its home page and in more than 10,000 academic studies by Wikipedia as of 2024, and in over 12,000 by its About page.", "claims": [ "F6", "F22", "F36", "F37" ] }, { "n": 16, "para": 6, "sense": "hosting, access and licence", "text": "The data are stored on Amazon Web Services' Public Data Sets under its Open Data Sponsorship Program, in an S3 bucket in us-east-1 (arn:aws:s3:::commoncrawl), and on academic cloud platforms, and since April 2026 a Hugging Face Storage Bucket has distributed them experimentally beside Amazon S3.", "claims": [ "F78", "F37", "F54", "F28", "TLL-25" ] }, { "n": 17, "para": 6, "sense": "hosting, access and licence", "text": "Access to the corpus hosted by Amazon is free, and it can be analysed on Amazon's platform or downloaded whole or in part; Wikipedia adds that users may incur storage and compute costs, and AWS's registry lists the bucket as account-required.", "claims": [ "F78", "F11", "F54", "TLL-19" ] }, { "n": 18, "para": 6, "sense": "hosting, access and licence", "text": "Its terms differ by source: AWS's registry makes the data available to anyone under the Common Crawl Terms of Use, Wikipedia's infobox gives a content licence of Apache 2.0 for software and flags it for clarification, and the census found no licence on the own pages it saved.", "claims": [ "F53", "F33", "TLL-26" ] }, { "n": 19, "para": 7, "sense": "in the training layer", "text": "Its data were used mostly by researchers and startups until the 2020s, when AI companies began training large language models on them; its About page dates its central role in AI development from Google's C4, built from a single snapshot to train T5 in 2019, and from the GPT-3 paper of 2020, which drew most of its training tokens from filtered Common Crawl data.", "claims": [ "F13", "F41" ] }, { "n": 20, "para": 7, "sense": "in the training layer", "text": "Typically after filtering and processing by third parties, it has been a primary training source for many large language models: a Mozilla Foundation study of 2024 found that at least 64% of 47 models published between 2019 and 2023 were trained on filtered versions of it, and Wikipedia reports that ChatGPT, as of 2023, was largely trained on it, as were other models including Gemini, and that by 2024 it and The Pile had been the two main training datasets.", "claims": [ "F47", "F36", "F37", "F62", "F20", "F23" ] }, { "n": 21, "para": 7, "sense": "in the training layer", "text": "AWS's registry lists LAION-5B, the GPT-3 paper and LLaMA among the work built on it; the archive records LLaMA 1's training mix, from the LLaMA paper, as 67% Common Crawl and 15% C4, and holds that the current training substrate is a filtered web crawl.", "claims": [ "F55", "O722-01" ] }, { "n": 22, "para": 7, "sense": "in the training layer", "text": "The census records whatever its crawler captured from the open web as what it carries into the layer, the training claims being Wikipedia's, and a person listed among its employees, in a post the foundation reposted, puts the stake for a site as \"If you're not in the crawl, you're not in the model.\"", "claims": [ "TLL-11", "F59" ] }, { "n": 23, "para": 7, "sense": "in the training layer", "text": "Three videos surfaced with it treat its uses for AI and machine learning (a PyTorch talk by Pedro Ortiz Suarez), for AI visibility (James Dooley) and as the web's open archive (Extract Summit).", "claims": [ "F66", "F67", "F68" ] }, { "n": 24, "para": 7, "sense": "in the training layer", "text": "The archive defines training-layer literature as writing addressed to the layer of culture where texts are crawled, indexed, trained on, retrieved and recomposed; it notes that every practice it surveys is in the corpora and that writing for the training layer alone is organized around that fact, and its own deposits are declared intended for inclusion in AI training corpora.", "claims": [ "O186-01", "O186-03", "O186-04" ] }, { "n": 25, "para": 7, "sense": "in the training layer", "text": "It holds training data to be semantic capital, so that training on Common Crawl draws on accumulated semantic labor, and its layer table places Common Crawl at the data and corpus layer, the layer of what a model knows, where what is missing is consent-based contribution with register annotation.", "claims": [ "O623-01", "O98-04" ] }, { "n": 26, "para": 7, "sense": "in the training layer", "text": "In its account the substrate a query draws on is an aggregate of crawled pages, structured data, embeddings, citation graphs, model weights and institutional signals, a differential field, and the institutional training loop, in which a work passes through a model into the next corpus, is the especially important case, success being a work that changes what a machine can write after encountering it; a machine trained on the crawled web, it records, now teaches the archive's field to those who search for it.", "claims": [ "O671-01", "O1543-02", "O1543-01", "O554-01" ] }, { "n": 27, "para": 8, "sense": "refined datasets downstream", "text": "What is made from the raw crawl is made downstream by others: organizations refine it into datasets that exclude unwanted content or suit their purposes, among them C4, OSCAR, CCNet, Pile-CC (a filtered subset within The Pile), MADLAD-400, RefinedWeb, FineWeb and DCLM.", "claims": [ "F31", "F49", "TLL-14", "TLL-29" ] }, { "n": 28, "para": 8, "sense": "refined datasets downstream", "text": "Google's C4 was constructed for the T5 models in 2019, and a 2024 study found 45% of its content restricted by websites' terms of service from uses such as AI training by for-profit companies.", "claims": [ "F32" ] }, { "n": 29, "para": 8, "sense": "refined datasets downstream", "text": "The archive describes the filtering that makes such datasets: CCNet scores each Common Crawl paragraph by its perplexity against a language model trained on Wikipedia, keeping the head, keeping the middle conditionally and discarding the tail as noise, and C4, RefinedWeb and Dolma share that logic, keeping what looks like Wikipedia and discarding the rest.", "claims": [ "O98-01", "O98-02" ] }, { "n": 30, "para": 8, "sense": "refined datasets downstream", "text": "It names the cost the Amputation, after work showing that perplexity gating reduces dataset diversity and introduces unexpected bias, and records the quality filter of LLaMA's mix as keeping pages that \"look like Wikipedia references\" and the top domain of the latest crawl as blogspot.com, at 0.9%.", "claims": [ "O98-03", "O722-02" ] }, { "n": 31, "para": 9, "sense": "what the crawl holds and leaves out", "text": "The crawl downloads HTML pages and does not archive images, videos or JavaScript files; an estimated 5% of its pages from before 2021 contained hate speech or slurs, among 3.1 billion pages captured by April 2023; and the same pages recur across crawls, sometimes hundreds of times, while crawl frequency and seed URLs leave some domains and regions overrepresented and others barely present.", "claims": [ "F70", "F21", "F64" ] }, { "n": 32, "para": 9, "sense": "what the crawl holds and leaves out", "text": "The foundation's Web Languages Project, launched in December 2024, addresses the overrepresentation of English-language content, and a Mozilla Foundation study of February 2024 called on it to be more transparent and inclusive about its governance and to highlight the limitations and biases of its data.", "claims": [ "F50", "F72" ] }, { "n": 33, "para": 9, "sense": "what the crawl holds and leaves out", "text": "A DEV Community post on its limits calls its data \"archived, not live\" and closes by recommending Zyte's API for fresh data.", "claims": [ "F65" ] }, { "n": 34, "para": 9, "sense": "what the crawl holds and leaves out", "text": "The archive holds that much of what matters most to human intelligence (private correspondence, oral traditions, classroom dialogue, books behind paywalls, small languages, domestic knowledge) is absent, underrepresented, illicitly captured or stripped of consent and provenance in a substrate built this way, \"absent from the substrate entirely\".", "claims": [ "O722-03" ] }, { "n": 35, "para": 9, "sense": "what the crawl holds and leaves out", "text": "It reads the crawl's content over time through others' measures: a year-wise semantic-similarity analysis of a Wikipedia-filtered Common Crawl subset shows linguistic diversity contracting since 2013, faster after public LLM adoption, which the archive proposes, as a hypothesis, to be homogenization toward the dominant forms; Graphite's analysis of Common Crawl finds over 52% of newly published web articles primarily AI-generated.", "claims": [ "O172-01", "O171-01", "O550-01" ] }, { "n": 36, "para": 9, "sense": "what the crawl holds and leaves out", "text": "The archive notes the irony that a literature written to be trained on enters corpora degrading under synthetic flood, holds that training corpora are ensembles, consumed whole by a scraped corpus while guarantees are certified per sequence, and counts the 67% Common Crawl training substrate among the engineered perceptual architecture of the present that \"is the noise\".", "claims": [ "O186-05", "O1555-01", "O99-01" ] }, { "n": 37, "para": 10, "sense": "opt-out, removal and the index as witness", "text": "The foundation states that CCBot obeys robots.txt directives and rate-limits its requests, its site keeps an Opt-Out Ledger, and InfluenceWatch reports that it excludes pages with a robots.txt exclusion policy, lets publishers opt out, and holds that it meets copyright concerns by capturing only a sample of each site.", "claims": [ "F43", "F9", "F71" ] }, { "n": 38, "para": 10, "sense": "opt-out, removal and the index as witness", "text": "It received its first requests to redact information in 2023 and its crawler was increasingly blocked; in June 2024 Danish media outlets demanded removal of their articles from past data sets and an end to crawling, the New York Times, Buzzfeed, the Washington Post and the CBC block it, news outlets cite its role in AI training in claiming it violates their copyrights, and by 2025 CCBot was the most widely blocked bot among the top 1000 websites.", "claims": [ "F18", "F73", "F69", "F26" ] }, { "n": 39, "para": 10, "sense": "opt-out, removal and the index as witness", "text": "In November 2025 an investigation by Alex Reisner for The Atlantic, part of its AI Watchdog series, reported that the foundation misled publishers when it claimed to respect paywalls and when it said it was honoring their requests to have content removed, and that its public search function showed no entries for sites that had requested removal while those sites were still in the scrapes used by AI companies; Wikipedia's lead states the removal finding as the foundation \"not honoring requests\".", "claims": [ "F24", "F25", "F14", "F52" ] }, { "n": 40, "para": 10, "sense": "opt-out, removal and the index as witness", "text": "A 2025 COLM paper cited on the About page finds that honoring robots.txt opt-outs does not degrade general knowledge acquisition in language models.", "claims": [ "F48" ] }, { "n": 41, "para": 10, "sense": "opt-out, removal and the index as witness", "text": "Pages in the corpus can be searched by URL in the URL Index, which the census takes as the test of training membership at the source; it records that crawl inclusion and demonstrated training use are separate claims, that use in a given model's training is a further claim the own pages do not make, that no saved source shows a page removed on request, and that The Atlantic's finding, as Wikipedia reports it, marks the limit of the index as a witness.", "claims": [ "F79", "TLL-13", "TLL-32", "TLL-23", "TLL-31" ] }, { "n": 42, "para": 10, "sense": "opt-out, removal and the index as witness", "text": "The archive holds robots.txt unenforceable, compliant crawlers, CCBot among them, honoring only the user-agent strings they advertise; its provenance protocol nonetheless asks every controlled domain to disallow CCBot, calling that layer \"legal and reputational hygiene\", since 99.7% of the visits to its blog named no crawler.", "claims": [ "O127-01", "O127-04", "O127-05" ] }, { "n": 43, "para": 10, "sense": "opt-out, removal and the index as witness", "text": "Its practice points both ways: one deposit records a robots.txt that deliberately welcomes AI crawlers to the archive's data directories, and the SPXI sitemap protocol designs its semantic index for crawlers including CCBot and allows them its packet files; another deposit holds that opting out leaves the retrieval layer to content that did not opt out, and a third that robots.txt generates a documentary record of intent even when it is ignored.", "claims": [ "O1-01", "O705-01", "O49-01", "O914-01" ] }, { "n": 44, "para": 10, "sense": "opt-out, removal and the index as witness", "text": "Membership can also be tested downstream of the crawl: a membership inference attack can detect whether a copyright trap was in a model's training set, and membership inference distinguishes such presence in practice against production-scale models, with accuracy that varies across architectures.", "claims": [ "O127-06", "O1075-01" ] }, { "n": 45, "para": 11, "sense": "names and attribution", "text": "No author or attribution field is recorded: a page is held by URL with its metadata and text extracts, who wrote a captured page is a question the crawl does not answer, and the census reads a work's name as surviving there only as far as it is written into the page itself.", "claims": [ "TLL-12", "TLL-21", "TLL-24", "TLL-28" ] }, { "n": 46, "para": 11, "sense": "names and attribution", "text": "The archive records that WARC records preserve more metadata than most downstream uses retain, and its protocol assumes that anything published is ingested and engineers the text so that its inscription survives the ingestion.", "claims": [ "O127-03", "O127-02" ] }, { "n": 47, "para": 12, "sense": "neighbours", "text": "It was designed to have more, fresher and more usable data than the Internet Archive's Wayback Machine, and it is a member of the International Internet Preservation Consortium and a partner in the End of Term Web Archive; the census keeps it apart from the Wayback Machine and records The Pile as a neighbour.", "claims": [ "F16", "F38", "TLL-30", "TLL-33" ] }, { "n": 48, "para": 13, "sense": "what the census asks of it", "text": "The layer census asks how literature is addressed to a receiving apparatus, what that apparatus makes from it, and how transmission, transformation and provenance can be inspected across later reception; it admits a resource that gathers and holds many works or records and says what it holds, and places Common Crawl under corpora and inspection, answering whether a URL was captured in a crawl, the crawl's size and cadence, and which releases exist.", "claims": [ "TLL-35", "TLL-36", "TLL-37", "TLL-08", "TLL-20" ] }, { "n": 49, "para": 13, "sense": "what the census asks of it", "text": "It leaves unchecked the CCBot, Opt-Out Ledger, Errata, FAQ, terms and About pages, the indexes themselves, The Atlantic's article and Wikipedia's claims on which models were trained on the data.", "claims": [ "TLL-34" ] }, { "n": 50, "para": 14, "sense": "reception and the composition at the name", "text": "Wikipedia's article was last edited on 24 September 2026; The Atlantic's article refused this row's fetch, and its card's title and opening, \"Common Crawl Is Doing the AI Industry's Dirty Work\", stand as its only text here.", "claims": [ "F34", "F52" ] }, { "n": 51, "para": 14, "sense": "reception and the composition at the name", "text": "Asked for \"Common Crawl\" on 1 November 2026, Google's AI Overview gave the foundation's legal form, founder, scale, hosting, file formats, uses and the criticism it has met; the Capture Registry found the foundation composed accurately at its grain, the mission in LinkedIn's wording, the executive director uncarried, and \"nearly two decades\" counted from the card's 19 years against the saved page's 15.", "claims": [ "CAP-1-09", "CAP-1-13", "CAP-1-14", "CAP-1-12", "CAP-1-01", "CAP-1-02", "CAP-1-03", "CAP-1-04" ] }, { "n": 52, "para": 14, "sense": "reception and the composition at the name", "text": "The registry also found the training intake given by company, the donations and the refined datasets absent, and The Atlantic's finding thinned to a mechanism of the crawler and a concern of publishers, the misleading search results absent beside an offer to query that index, an index that names no author.", "claims": [ "CAP-1-06", "CAP-1-07", "CAP-1-08", "CAP-1-05", "CAP-1-10", "CAP-1-11" ] } ], "joins": [ { "act": "order of senses", "sentences": "1–52", "claims": [], "note": "equal case (§4.7): the entity's dependency order, field and archive claims ordered together by genealogy within each sense, neither ledger placed first as a block. What it is (¶0) precedes the foundation that holds it (¶1) and its dates (¶2); the crawler (¶3) precedes the unit it produces (¶4) and the size of that unit (¶5); hosting and terms (¶6) precede use, since use is access; the training layer (¶7) precedes the refined datasets made downstream (¶8) and what the crawl holds and leaves out (¶9); opt-out and the index (¶10) follow the holdings they bound; names (¶11) follow the index, which addresses by URL; neighbours (¶12), the census's question (¶13) and reception (¶14) stand last as relations of the entity already composed." }, { "act": "holding together (the seeding: two accounts)", "sentences": [ 9 ], "claims": [ "F17", "F39" ], "note": "P_coh (§4.4): Wikipedia's crawl begun from Blekko's donated URL list and the About page's 2012 donation of crawl metadata by blekko, composed together; the composer does not rule between them (open block)." }, { "act": "holding together (size and dates at several grains)", "sentences": [ 14, 7, 15 ], "claims": [ "F4", "F8", "F36", "F35", "F53", "F61", "F5", "F75", "F15", "F6", "F22", "F37" ], "note": "figures differing by date, page and grain (15 against 19 years; 2007 against 2008; 10,000 against 12,000 citations) composed with their sources; none is preferred." }, { "act": "holding together (the foundation's statement and The Atlantic's finding)", "sentences": [ 37, 39 ], "claims": [ "F43", "F71", "F24", "F25", "F14" ], "note": "the foundation's self-description (CCBot obeys robots.txt; publishers may opt out) and The Atlantic's finding on paywalls, removal and the search function, as Wikipedia reports it, composed in adjacent sentences; the claims differ in object (crawling against removal from the data) and both stand." }, { "act": "holding together (the archive's two postures on CCBot)", "sentences": [ 42, 43 ], "claims": [ "O127-04", "O1-01", "O705-01", "O49-01" ], "note": "P_coh within the archive's ledger: #127's Layer 7 (every controlled domain SHOULD disallow CCBot) against #1's robots.txt welcoming AI crawlers and #705's allowing CCBot the SPXI files; #49's account of what opting out cedes. Both stand, each with its deposit." }, { "act": "placing in the open block", "sentences": "open", "claims": [ "TLL-13", "TLL-23", "F25", "F79", "F17", "F39", "TLL-26", "F53", "F33", "TLL-12", "O127-03", "F19", "T1-10" ], "note": "five questions placed in the open block: whether the URL Index witnesses inclusion and removal; how the crawl was seeded; under what terms the data are offered; whether any attribution travels with a captured page; the source of the Overview's list of training companies." } ], "plan": { "order": [ "F1", "F10", "F35", "F69", "F60", "F68", "TLL-01", "TLL-02", "TLL-04", "F3", "F56", "F57", "F27", "TLL-22", "O186-02", "TLL-05", "TLL-06", "TLL-07", "F2", "F12", "F51", "F29", "F42", "F74", "TLL-03", "F58", "F30", "F19", "F5", "F75", "F15", "TLL-16", "F39", "F40", "F17", "TLL-15", "F44", "F7", "TLL-18", "TLL-09", "F76", "F77", "TLL-27", "F45", "F63", "F46", "F4", "F8", "F36", "F53", "F61", "TLL-10", "TLL-17", "F6", "F22", "F37", "F78", "F54", "F28", "TLL-25", "F11", "TLL-19", "F33", "TLL-26", "F13", "F41", "F47", "F62", "F20", "F23", "F55", "O722-01", "TLL-11", "F59", "F66", "F67", "O186-01", "O186-03", "O186-04", "O623-01", "O98-04", "O671-01", "O1543-02", "O1543-01", "O554-01", "F31", "F49", "TLL-14", "TLL-29", "F32", "O98-01", "O98-02", "O98-03", "O722-02", "F70", "F21", "F64", "F50", "F72", "F65", "O722-03", "O172-01", "O171-01", "O550-01", "O186-05", "O1555-01", "O99-01", "F43", "F9", "F71", "F18", "F73", "F26", "F24", "F25", "F14", "F52", "F48", "F79", "TLL-13", "TLL-32", "TLL-23", "TLL-31", "O127-01", "O127-04", "O127-05", "O1-01", "O705-01", "O49-01", "O914-01", "O127-06", "O1075-01", "TLL-12", "TLL-21", "TLL-24", "TLL-28", "O127-03", "O127-02", "F16", "F38", "TLL-30", "TLL-33", "TLL-35", "TLL-36", "TLL-37", "TLL-08", "TLL-20", "TLL-34", "F34", "CAP-1-09", "CAP-1-13", "CAP-1-14", "CAP-1-12", "CAP-1-01", "CAP-1-02", "CAP-1-03", "CAP-1-04", "CAP-1-06", "CAP-1-07", "CAP-1-08", "CAP-1-05", "CAP-1-10", "CAP-1-11" ], "senses": { "what it is": [ 1, 2, 3 ], "the foundation: form, founder, governance": [ 4, 5 ], "funding": [ 6 ], "dates: 2007, 2008, 2011": [ 7 ], "the crawler and its seeds": [ 8, 9, 10 ], "units, formats and indexes": [ 11, 12, 13 ], "size and cadence": [ 14 ], "research use and citation": [ 15 ], "hosting, access and licence": [ 16, 17, 18 ], "in the training layer": [ 19, 20, 21, 22, 23, 24, 25, 26 ], "refined datasets downstream": [ 27, 28, 29, 30 ], "what the crawl holds and leaves out": [ 31, 32, 33, 34, 35, 36 ], "opt-out, removal and the index as witness": [ 37, 38, 39, 40, 41, 42, 43, 44 ], "names and attribution": [ 45, 46 ], "neighbours": [ 47 ], "what the census asks of it": [ 48, 49 ], "reception and the composition at the name": [ 50, 51, 52 ] }, "marks": [ { "claim": "F4", "mark": "contested", "with": [ "F8", "F35", "F36", "F61" ], "note": "size and span differ by page, date and grain (15 years on the page, 19 on its card)" }, { "claim": "F5", "mark": "contested", "with": [ "F75", "F35", "F15" ], "note": "2007 (home) against 2008 (Overview, About); Wikipedia's 2007 archiving and 2011 publishing" }, { "claim": "F17", "mark": "contested", "with": [ "F39" ], "note": "Blekko's URL list as the crawl's start (Wikipedia) against blekko's 2012 metadata donation (About)" }, { "claim": "F58", "mark": "internally_inconsistent", "with": [ "F58" ], "note": "LinkedIn lists 23 employees and a company size of 2–10 on one page" }, { "claim": "F43", "mark": "contested", "with": [ "F24", "F25" ], "note": "the foundation's statement of robots.txt obedience beside The Atlantic's finding on removal and paywalls; different objects, both composed" }, { "claim": "O127-04", "mark": "contested", "with": [ "O1-01", "O705-01" ], "note": "within the archive: disallow CCBot (#127) against welcoming AI crawlers (#1) and allowing CCBot the SPXI files (#705)" } ], "conflicts": [], "cuts": [], "carried": 163 } }, "P": { "title": "Common Crawl", "genre": "the compression: AIO's interaction grammar (lede, headed clusters, bolded labels, card rail), composed from the knowledge object; equal case", "status": "composed 2026-11-01 from the v0.10 knowledge object; not frozen", "lede": { "text": "is a free, open repository of web crawl data kept by a 501(c)(3) foundation founded in 2007 by Gil Elbaz: monthly numbered crawls of raw pages, metadata and text, each page addressed by URL, which others refine into the datasets that train large language models. No author field travels with a page.", "claims": [ "F1", "F10", "F35", "F2", "F12", "F77", "F76", "TLL-09", "F31", "F47", "TLL-12", "TLL-01", "TLL-02", "TLL-04" ] }, "sections": [ { "icon": "🗂️", "head": "What it holds", "items": [ { "label": "Crawls:", "text": "about monthly since 2008 (2007 on the home page); CCBot, on Apache Nutch since 2013, ranks URLs by harmonic centrality.", "claims": [ "F5", "F75", "F15", "F39", "F40", "F44", "F17", "F7", "TLL-15", "TLL-16", "TLL-18", "TLL-27" ] }, { "label": "Formats:", "text": "WARC (raw responses), WAT (metadata), WET (text); CDXJ and URL indexes.", "claims": [ "F45", "F63", "F46" ] }, { "label": "Size:", "text": "over 300 billion pages; 3–5 billion a month; over 10 petabytes; \"15 years\" on the page, \"19\" on its card.", "claims": [ "F4", "F8", "F36", "F53", "F61", "F6", "F22", "F37", "TLL-10", "TLL-17" ] }, { "label": "Gaps:", "text": "HTML only; duplicates; seed bias; English overrepresented; ~5% hate speech before 2021.", "claims": [ "F70", "F64", "F50", "F21", "F72", "F65" ] } ] }, { "icon": "🏛️", "head": "Who holds it", "items": [ { "label": "Foundation:", "text": "chair Gil Elbaz; executive director Rich Skrenta (2023); about 18 staff.", "claims": [ "F29", "F42", "F51", "F56", "F57", "F58", "F74", "F3", "F27", "F60", "TLL-03" ] }, { "label": "Funding:", "text": "the Elbaz family foundation; from 2023 AI companies, Anthropic and OpenAI $250,000 each.", "claims": [ "F30", "F19" ] }, { "label": "Hosting:", "text": "AWS Open Data (s3://commoncrawl), free to access, compute costs aside; Hugging Face from April 2026.", "claims": [ "F78", "F11", "F54", "F28", "TLL-25", "TLL-19" ] }, { "label": "Terms:", "text": "Common Crawl Terms of Use (AWS); licence flagged on Wikipedia.", "claims": [ "F53", "F33", "TLL-26" ] } ] }, { "icon": "🧠", "head": "Into training", "items": [ { "label": "Use:", "text": "from the 2020s; at least 64% of 47 LLMs (Mozilla, 2024); GPT-3, ChatGPT, Gemini, LLaMA.", "claims": [ "F13", "F41", "F47", "F62", "F20", "F23", "F55", "F59", "F66", "F67", "F68", "TLL-11" ] }, { "label": "Refined downstream:", "text": "C4, FineWeb, RefinedWeb, Pile-CC, DCLM and others, made by others.", "claims": [ "F31", "F49", "F32", "TLL-14", "TLL-29" ] }, { "label": "The filter (archive):", "text": "keep what looks like Wikipedia, discard the tail: the Amputation (#98); LLaMA 67% Common Crawl (#722).", "claims": [ "O98-01", "O98-02", "O98-03", "O722-01", "O722-02", "O98-04", "O99-01" ] }, { "label": "Substrate (archive):", "text": "semantic capital (#623); much left absent (#722); diversity contracting (#172).", "claims": [ "O623-01", "O722-03", "O172-01", "O171-01", "O550-01", "O1555-01", "O186-05", "O671-01", "O554-01" ] } ] }, { "icon": "🚫", "head": "Opt-out and the index", "items": [ { "label": "Stated:", "text": "CCBot obeys robots.txt; an Opt-Out Ledger; the most-blocked bot by 2025.", "claims": [ "F43", "F9", "F71", "F18", "F26", "F73", "F69", "F48" ] }, { "label": "The Atlantic, 2025:", "text": "publishers misled on paywalls and removal; its search showed no entries for sites still in the scrapes.", "claims": [ "F24", "F25", "F14", "F52" ] }, { "label": "Index as witness:", "text": "inclusion searchable by URL; training use a further claim (census).", "claims": [ "F79", "TLL-13", "TLL-23", "TLL-31", "TLL-32", "O127-06", "O1075-01" ] }, { "label": "Archive:", "text": "robots.txt unenforceable; disallow CCBot (#127), yet welcome crawlers (#1, #705).", "claims": [ "O127-01", "O127-04", "O127-05", "O1-01", "O705-01", "O49-01", "O914-01" ] } ] }, { "icon": "🏷️", "head": "Names", "items": [ { "label": "Attribution:", "text": "none recorded; a name survives only as written into the page (census); inscriptions engineered to survive (#127).", "claims": [ "TLL-24", "TLL-21", "TLL-28", "O127-02", "O127-03", "O186-02", "O186-01", "O186-03", "O186-04", "O1543-01", "O1543-02", "TLL-22", "TLL-05", "TLL-06", "TLL-07", "TLL-08", "TLL-20", "TLL-35", "TLL-36", "TLL-37", "TLL-34", "F16", "F38", "TLL-30", "TLL-33" ] } ] }, { "icon": "🔎", "head": "At the name", "items": [ { "label": "Overview, 1 November 2026:", "text": "accurate at its grain; donations and refined datasets absent; The Atlantic thinned, the misleading index results uncarried (Capture Registry).", "claims": [ "CAP-1-01", "CAP-1-02", "CAP-1-03", "CAP-1-04", "CAP-1-05", "CAP-1-06", "CAP-1-07", "CAP-1-08", "CAP-1-09", "CAP-1-10", "CAP-1-11", "CAP-1-12", "CAP-1-13", "CAP-1-14", "F34" ] } ] } ], "open": [ { "question": "Does the URL Index witness whether a page is in the crawl, and whether a removal took effect?", "rests_on": [ "F79", "F25", "F24", "F43", "TLL-13", "TLL-23", "TLL-31", "CAP-1-08" ], "modality": "F79 stipulation (own page); F24, F25 attributed (The Atlantic, by Wikipedia); F43 self-description; TLL and CAP interpretation", "would_resolve": "a dated query of the URL Index for a site with a recorded removal request, read against the WARC files of the same crawl; The Atlantic's article itself (refused at fetch); the Opt-Out Ledger page (unchecked)" }, { "question": "How was the crawl seeded: from a list of URLs donated by Blekko, or with crawl metadata blekko donated in 2012?", "rests_on": [ "F17", "F39", "TLL-15" ], "modality": "documented (Wikipedia; the About page)", "would_resolve": "the sources Wikipedia cites for the Blekko list (American Scientist, 2015) and the foundation's 2012 announcement, dated" }, { "question": "Under what terms are the data offered?", "rests_on": [ "F53", "F33", "TLL-26" ], "modality": "documented", "would_resolve": "the Common Crawl Terms of Use page (unchecked by the census and this row)" }, { "question": "Does any attribution of a captured work travel with a page into the refined datasets?", "rests_on": [ "TLL-12", "TLL-24", "O127-03", "O127-02", "O98-01" ], "modality": "interpretation (the census; #127); O98-01 attributed", "would_resolve": "a reading of the WARC, WAT and WET records for one page with authorial metadata, and of the same page in a refined dataset (C4, FineWeb)" }, { "question": "Where did the Overview's list of training companies come from?", "rests_on": [ "F19", "F20", "F41", "F47", "F55", "CAP-1-06" ], "modality": "documented, self-description; CAP interpretation", "would_resolve": "the field names OpenAI (GPT-3, ChatGPT), Google (C4, Gemini) and Meta (LLaMA) as training on the data, and Anthropic only as a donor (F19); a later capture at the address that cites a source for Anthropic's training use, or The Atlantic's article, refused at fetch" } ], "rail": [ { "lineage": "what it is", "claims": [ "F1", "F10", "F35", "F60", "F68", "TLL-01", "TLL-02", "TLL-04" ], "earliest": "B2" }, { "lineage": "the foundation: form, founder, governance", "claims": [ "F2", "F12", "F29", "F42", "F58", "F74", "TLL-03", "CAP-1-01" ], "earliest": "B2" }, { "lineage": "purpose and mission", "claims": [ "F3", "F27", "F56", "F57", "CAP-1-02" ], "earliest": "B2" }, { "lineage": "size and cadence", "claims": [ "F4", "F8", "F36", "F61", "TLL-10", "TLL-17", "TLL-27", "CAP-1-03" ], "earliest": "B1" }, { "lineage": "dates: 2007, 2008, 2011", "claims": [ "F5", "F15", "F75", "TLL-16" ], "earliest": "B2" }, { "lineage": "research use and citation", "claims": [ "F6", "F22" ], "earliest": "B2" }, { "lineage": "units, formats and indexes", "claims": [ "F7", "F45", "F46", "F63", "F76", "F77", "TLL-09", "TLL-18", "CAP-1-04" ], "earliest": "B1" }, { "lineage": "opt-out, removal and the index as witness", "claims": [ "F9", "F18", "F26", "F43", "F48", "F69", "F71", "F73", "F79", "TLL-13", "TLL-23", "TLL-31", "TLL-32", "TLL-34", "CAP-1-08", "O127-01", "O127-04", "O127-05", "O127-06", "O705-01", "O1-01", "O49-01", "O914-01", "O1075-01" ], "earliest": 49 }, { "lineage": "hosting, access and licence", "claims": [ "F11", "F28", "F33", "F53", "F54", "F78", "TLL-19", "TLL-25", "TLL-26" ], "earliest": "B2" }, { "lineage": "in the training layer", "claims": [ "F13", "F20", "F23", "F37", "F41", "F47", "F55", "F59", "F62", "F66", "F67", "TLL-11", "TLL-22", "CAP-1-06", "O186-01", "O186-03", "O186-04", "O1543-01", "O1543-02", "O722-01", "O98-04", "O623-01", "O671-01", "O554-01" ], "earliest": 554 }, { "lineage": "The Atlantic's finding", "claims": [ "F14", "F24", "F25", "F52", "CAP-1-07" ], "earliest": "B4" }, { "lineage": "neighbours", "claims": [ "F16", "F38", "TLL-30", "TLL-33" ], "earliest": "B2" }, { "lineage": "the crawler and its seeds", "claims": [ "F17", "F39", "F40", "F44", "TLL-15" ], "earliest": "B2" }, { "lineage": "funding", "claims": [ "F19", "F30", "F51" ], "earliest": "B2" }, { "lineage": "what the crawl holds and leaves out", "claims": [ "F21", "F50", "F64", "F65", "F70", "F72", "O186-05", "O722-03", "O99-01", "O171-01", "O172-01", "O550-01", "O1555-01" ], "earliest": 550 }, { "lineage": "refined datasets downstream", "claims": [ "F31", "F32", "F49", "TLL-14", "TLL-29", "O722-02", "O98-01", "O98-02", "O98-03" ], "earliest": 722 }, { "lineage": "reception", "claims": [ "F34", "CAP-1-09", "CAP-1-10", "CAP-1-11", "CAP-1-12", "CAP-1-13", "CAP-1-14" ], "earliest": "B2" }, { "lineage": "what the census asks of it", "claims": [ "TLL-05", "TLL-06", "TLL-07", "TLL-08", "TLL-20", "TLL-35", "TLL-36", "TLL-37" ], "earliest": "TLL" }, { "lineage": "names and attribution", "claims": [ "TLL-12", "TLL-21", "TLL-24", "TLL-28", "CAP-1-05", "O186-02", "O127-02", "O127-03" ], "earliest": 127 } ], "body_words": 335, "entity": "Composed under v0.10, equal case (ρ = extends): the article first (§4.13), attribution beneath (§4.5); every field and archive claim carried." }, "P_B": { "title": "Common Crawl", "arm": "B (the field alone)", "genre": "popup — the compression from the field ledger alone (cards, the own page and Wikipedia)", "status": "composed 2026-11-01 from the field ledger alone; not frozen", "lede": { "text": "is a non-profit foundation (2007, Gil Elbaz) that crawls the web monthly and offers the data free, a main source of training data for large language models.", "claims": [ "F10", "F1", "F2", "F12", "F13", "F47" ] }, "sections": [ { "icon": "🗂️", "head": "The data", "items": [ { "label": "Crawls:", "text": "WARC, WAT, WET; over 300 billion pages; on AWS.", "claims": [ "F45", "F4", "F78", "F77" ] } ] }, { "icon": "🧠", "head": "Training", "items": [ { "label": "Use:", "text": "GPT-3, ChatGPT, LLaMA; refined into C4 and FineWeb.", "claims": [ "F41", "F20", "F31", "F49" ] } ] }, { "icon": "🚫", "head": "Criticism", "items": [ { "label": "Publishers:", "text": "blocked widely; The Atlantic found removal claims misleading.", "claims": [ "F26", "F24", "F25", "F73" ] } ] } ], "open": [ { "question": "Opacities", "rests_on": [ "F52", "F17", "F39", "F33", "F53" ], "modality": "documented and attributed", "would_resolve": "The Atlantic's article, refused at fetch; the seeding; the terms" } ], "entity": "the field's entity: the foundation's open web crawl, as its cards, its own Overview page and Wikipedia present it", "rail": [ { "lineage": "what it is", "claims": [ "F1", "F10", "F35", "F60", "F68" ], "earliest": "B2" }, { "lineage": "the foundation: form, founder, governance", "claims": [ "F2", "F12", "F29", "F42", "F58", "F74" ], "earliest": "B2" }, { "lineage": "purpose and mission", "claims": [ "F3", "F27", "F56", "F57" ], "earliest": "B2" }, { "lineage": "size and cadence", "claims": [ "F4", "F8", "F36", "F61" ], "earliest": "B1" }, { "lineage": "dates: 2007, 2008, 2011", "claims": [ "F5", "F15", "F75" ], "earliest": "B2" }, { "lineage": "research use and citation", "claims": [ "F6", "F22" ], "earliest": "B2" }, { "lineage": "units, formats and indexes", "claims": [ "F7", "F45", "F46", "F63", "F76", "F77" ], "earliest": "B1" }, { "lineage": "opt-out, removal and the index as witness", "claims": [ "F9", "F18", "F26", "F43", "F48", "F69", "F71", "F73", "F79" ], "earliest": "B2" }, { "lineage": "hosting, access and licence", "claims": [ "F11", "F28", "F33", "F53", "F54", "F78" ], "earliest": "B2" }, { "lineage": "in the training layer", "claims": [ "F13", "F20", "F23", "F37", "F41", "F47", "F55", "F59", "F62", "F66", "F67" ], "earliest": "B2" }, { "lineage": "The Atlantic's finding", "claims": [ "F14", "F24", "F25", "F52" ], "earliest": "B4" }, { "lineage": "neighbours", "claims": [ "F16", "F38" ], "earliest": "B2" }, { "lineage": "the crawler and its seeds", "claims": [ "F17", "F39", "F40", "F44" ], "earliest": "B2" }, { "lineage": "funding", "claims": [ "F19", "F30", "F51" ], "earliest": "B2" }, { "lineage": "what the crawl holds and leaves out", "claims": [ "F21", "F50", "F64", "F65", "F70", "F72" ], "earliest": "B2" }, { "lineage": "refined datasets downstream", "claims": [ "F31", "F32", "F49" ], "earliest": "B2" }, { "lineage": "reception", "claims": [ "F34" ], "earliest": "B2" } ], "body_words": 55 }, "KO_B": { "title": "Common Crawl", "arm": "B (the field alone)", "genre": "knowledge object", "status": "composed 2026-11-01; not frozen", "sentences": [ { "n": 1, "para": 0, "text": "The Common Crawl Foundation is a nonprofit 501(c)(3) organization, founded in 2007 by Gil Elbaz, that crawls the web and maintains a free, open repository of the crawl data for anyone to use.", "claims": [ "F10", "F1", "F2", "F12", "F35", "F60", "F69", "F68", "F56" ] }, { "n": 2, "para": 0, "text": "It aims to make extraction and analysis of open web data accessible to researchers and cheap, and can spare websites the cost of repeated scraping.", "claims": [ "F3", "F57", "F27" ] }, { "n": 3, "para": 0, "text": "It is chaired by Elbaz, directed since 2023 by Rich Skrenta, advised by Peter Norvig and Joi Ito, and staffed by about 18 people (LinkedIn lists 23 and a size of 2–10).", "claims": [ "F51", "F42", "F29", "F74", "F58" ] }, { "n": 4, "para": 0, "text": "It was funded almost entirely by the Elbaz Family Foundation Trust until 2023, when AI companies including Anthropic and OpenAI began donating $250,000 each.", "claims": [ "F30", "F19" ] }, { "n": 5, "para": 1, "text": "Data were collected from 2007 or 2008 and crawls published from 2011, first with a Hadoop crawler, then from 2013 with CCBot on Apache Nutch, which ranks URLs by harmonic centrality; the crawl began from Blekko URLs or used blekko metadata from 2012.", "claims": [ "F5", "F75", "F15", "F39", "F40", "F44", "F17", "F7" ] }, { "n": 6, "para": 1, "text": "Each monthly release, such as CC-MAIN-2026-43, holds raw page data, metadata and text in WARC, WAT and WET files, indexed by CDXJ and URL indexes.", "claims": [ "F76", "F77", "F45", "F63", "F46" ] }, { "n": 7, "para": 1, "text": "It holds over 300 billion pages (15 years on the page, 19 on its card), over 10 petabytes, 3–5 billion new pages a month, and is cited in 10,000 to 12,000 papers.", "claims": [ "F4", "F8", "F36", "F53", "F61", "F6", "F22", "F37" ] }, { "n": 8, "para": 1, "text": "It is stored free on AWS Open Data and academic clouds, with compute costs to users, an account-required bucket and a Hugging Face mirror since April 2026, under its Terms of Use or a flagged Apache licence.", "claims": [ "F78", "F11", "F54", "F28", "F53", "F33" ] }, { "n": 9, "para": 2, "text": "AI companies began training large language models on it in the 2020s: C4 for T5 (2019), GPT-3 (2020), ChatGPT and Gemini, at least 64% of 47 models in Mozilla's 2024 study, and with The Pile the main training dataset by 2024.", "claims": [ "F13", "F41", "F47", "F62", "F20", "F23", "F55", "F59", "F66", "F67" ] }, { "n": 10, "para": 2, "text": "Others refine it into datasets such as C4, FineWeb, RefinedWeb, Pile-CC and DCLM; 45% of C4's content is restricted by sites' terms from for-profit AI training.", "claims": [ "F31", "F49", "F32" ] }, { "n": 11, "para": 2, "text": "It keeps HTML only, duplicates pages across crawls, over-represents some domains and English, and held about 5% hate speech before 2021; Mozilla asked it to state its biases.", "claims": [ "F70", "F64", "F50", "F21", "F72", "F65" ] }, { "n": 12, "para": 3, "text": "It says CCBot obeys robots.txt and lets publishers opt out, yet CCBot became the most blocked bot, Danish and US outlets demanded removal or block it, and The Atlantic found in 2025 that it misled publishers on paywalls and removal and that its search hid sites still in its scrapes.", "claims": [ "F43", "F9", "F71", "F18", "F26", "F73", "F69", "F24", "F25", "F14", "F52", "F48" ] }, { "n": 13, "para": 3, "text": "Pages can be searched by URL in its URL Index.", "claims": [ "F79" ] }, { "n": 14, "para": 4, "text": "It was designed to outdo the Wayback Machine, belongs to the IIPC, and its Wikipedia article was last edited on 24 September 2026.", "claims": [ "F16", "F38", "F34" ] } ] }, "L_B": { "words": 429, "text": "The Common Crawl Foundation is a nonprofit 501(c)(3) organization, founded in 2007 by Gil Elbaz, that crawls the web and maintains a free, open repository of the crawl data for anyone to use. It aims to make extraction and analysis of open web data accessible to researchers and cheap, and can spare websites the cost of repeated scraping. It is chaired by Elbaz, directed since 2023 by Rich Skrenta, advised by Peter Norvig and Joi Ito, and staffed by about 18 people (LinkedIn lists 23 and a size of 2–10). It was funded almost entirely by the Elbaz Family Foundation Trust until 2023, when AI companies including Anthropic and OpenAI began donating $250,000 each. Data were collected from 2007 or 2008 and crawls published from 2011, first with a Hadoop crawler, then from 2013 with CCBot on Apache Nutch, which ranks URLs by harmonic centrality; the crawl began from Blekko URLs or used blekko metadata from 2012. Each monthly release, such as CC-MAIN-2026-43, holds raw page data, metadata and text in WARC, WAT and WET files, indexed by CDXJ and URL indexes. It holds over 300 billion pages (15 years on the page, 19 on its card), over 10 petabytes, 3–5 billion new pages a month, and is cited in 10,000 to 12,000 papers. It is stored free on AWS Open Data and academic clouds, with compute costs to users, an account-required bucket and a Hugging Face mirror since April 2026, under its Terms of Use or a flagged Apache licence. AI companies began training large language models on it in the 2020s: C4 for T5 (2019), GPT-3 (2020), ChatGPT and Gemini, at least 64% of 47 models in Mozilla's 2024 study, and with The Pile the main training dataset by 2024. Others refine it into datasets such as C4, FineWeb, RefinedWeb, Pile-CC and DCLM; 45% of C4's content is restricted by sites' terms from for-profit AI training. It keeps HTML only, duplicates pages across crawls, over-represents some domains and English, and held about 5% hate speech before 2021; Mozilla asked it to state its biases. It says CCBot obeys robots.txt and lets publishers opt out, yet CCBot became the most blocked bot, Danish and US outlets demanded removal or block it, and The Atlantic found in 2025 that it misled publishers on paywalls and removal and that its search hid sites still in its scrapes. Pages can be searched by URL in its URL Index. It was designed to outdo the Wayback Machine, belongs to the IIPC, and its Wikipedia article was last edited on 24 September 2026.", "note": "L_F(B), the field composed in its own ontology (§0.4), realized as KO_B; its compression is P_B" }, "L_BA": { "words": 2666, "text": "Common Crawl is a free, open repository of web crawl data that anyone can use, maintained by the Common Crawl Foundation, a 501(c)(3) non-profit that crawls the web and freely provides its archives and datasets to the public. The foundation gives its purpose as making wholesale extraction, transformation and analysis of open web data accessible to researchers; its LinkedIn page speaks of democratizing access to web information and of levelling the playing field by making that work cheap and easy, and an LWN.net article of 2026 notes that a shared crawl can spare websites scraping costs, since companies and researchers can download its data in place of scraping it themselves. The archive's layer census reads it as the receiving apparatus's first intake, the plainest case of the persistence criterion of #186, which asks that a work be placed where the training and retrieval layers actually look, with the metadata that lets it be found and attributed; the census enters it as infrastructure, a dataset whose question is training membership at the source. Gil Elbaz founded it in 2007 as a California-registered non-profit and chairs its board; Peter Norvig and Joi Ito have served on its advisory board, and Rich Skrenta, creator of the blekko search engine and founder of the Open Directory Project, became executive director in 2023. Its location is given as Beverly Hills by InfluenceWatch, and its size as about 18 staff in 2026 on its About page, while its LinkedIn page lists 23 employees against a stated company size of 2 to 10. It was funded almost exclusively by the Elbaz Family Foundation Trust until 2023, and its About page still names the Elbaz Family Foundation as its primary funder; from 2023 it also received significant support from AI companies, among them Anthropic and OpenAI, which gave $250,000 each. Its own pages date the corpus two ways: the home page has a free and open corpus since 2007, the Overview and About pages data collected regularly since 2008; Wikipedia has the foundation founded in San Francisco in 2007, archiving websites that year and publishing its crawls from 2011. It began collecting in 2008 with a custom Hadoop-based crawler using PageRank, and in 2013 replaced it with CCBot, a crawler built on Apache Nutch, moving from the ARC to the WARC format with the November 2013 crawl. Two accounts of its seeding stand side by side: Wikipedia says the crawl started from a list of URLs donated by the search engine Blekko, and counts 1.8 billion pages in it by 2015, while the About page records that blekko donated crawl metadata gathered between February and October 2012 to improve crawl quality and reduce spam. CCBot prioritises URLs by harmonic centrality, a measure of a page's proximity to the structural core of the web, and the foundation releases host- and domain-level web graphs as data, the latest of October 2026 covering August to October. Its unit is the crawl, published about once a month as a numbered release (the latest listed is CC-MAIN-2026-43) of raw web page data, metadata extracts and text extracts, in which a page is addressed by its URL. The files come in three formats: WARC, full HTTP responses, append-only and immutable once written (ISO 28500); WAT, metadata extracted from the WARC records, among them headers and link graphs; and WET, plain text extracted from the crawled HTML, which the About page expands as WARC Encapsulated Text and a DEV Community post as Web Extracted Text, \"Ideal for NLP or training text based models.\" Monthly index files are released in a CDXJ Index and a URL Index, the latter queryable with Amazon Athena. Its size is stated at several dates and grains: over 300 billion pages spanning 15 years, with 3 to 5 billion new pages a month, on the home page, which the card surfacing that page renders as spanning 19 years; more than 10 petabytes and more than two billion pages per crawl on the About page; over 300 billion pages, updated monthly, in AWS's registry; and 2.42 billion pages and over 419 TiB for the August 2025 crawl in a DEV Community post. It is cited in over 10,000 research papers by its home page and in more than 10,000 academic studies by Wikipedia as of 2024, and in over 12,000 by its About page. The data are stored on Amazon Web Services' Public Data Sets under its Open Data Sponsorship Program, in an S3 bucket in us-east-1 (arn:aws:s3:::commoncrawl), and on academic cloud platforms, and since April 2026 a Hugging Face Storage Bucket has distributed them experimentally beside Amazon S3. Access to the corpus hosted by Amazon is free, and it can be analysed on Amazon's platform or downloaded whole or in part; Wikipedia adds that users may incur storage and compute costs, and AWS's registry lists the bucket as account-required. Its terms differ by source: AWS's registry makes the data available to anyone under the Common Crawl Terms of Use, Wikipedia's infobox gives a content licence of Apache 2.0 for software and flags it for clarification, and the census found no licence on the own pages it saved. Its data were used mostly by researchers and startups until the 2020s, when AI companies began training large language models on them; its About page dates its central role in AI development from Google's C4, built from a single snapshot to train T5 in 2019, and from the GPT-3 paper of 2020, which drew most of its training tokens from filtered Common Crawl data. Typically after filtering and processing by third parties, it has been a primary training source for many large language models: a Mozilla Foundation study of 2024 found that at least 64% of 47 models published between 2019 and 2023 were trained on filtered versions of it, and Wikipedia reports that ChatGPT, as of 2023, was largely trained on it, as were other models including Gemini, and that by 2024 it and The Pile had been the two main training datasets. AWS's registry lists LAION-5B, the GPT-3 paper and LLaMA among the work built on it; the archive records LLaMA 1's training mix, from the LLaMA paper, as 67% Common Crawl and 15% C4, and holds that the current training substrate is a filtered web crawl. The census records whatever its crawler captured from the open web as what it carries into the layer, the training claims being Wikipedia's, and a person listed among its employees, in a post the foundation reposted, puts the stake for a site as \"If you're not in the crawl, you're not in the model.\" Three videos surfaced with it treat its uses for AI and machine learning (a PyTorch talk by Pedro Ortiz Suarez), for AI visibility (James Dooley) and as the web's open archive (Extract Summit). The archive defines training-layer literature as writing addressed to the layer of culture where texts are crawled, indexed, trained on, retrieved and recomposed; it notes that every practice it surveys is in the corpora and that writing for the training layer alone is organized around that fact, and its own deposits are declared intended for inclusion in AI training corpora. It holds training data to be semantic capital, so that training on Common Crawl draws on accumulated semantic labor, and its layer table places Common Crawl at the data and corpus layer, the layer of what a model knows, where what is missing is consent-based contribution with register annotation. In its account the substrate a query draws on is an aggregate of crawled pages, structured data, embeddings, citation graphs, model weights and institutional signals, a differential field, and the institutional training loop, in which a work passes through a model into the next corpus, is the especially important case, success being a work that changes what a machine can write after encountering it; a machine trained on the crawled web, it records, now teaches the archive's field to those who search for it. What is made from the raw crawl is made downstream by others: organizations refine it into datasets that exclude unwanted content or suit their purposes, among them C4, OSCAR, CCNet, Pile-CC (a filtered subset within The Pile), MADLAD-400, RefinedWeb, FineWeb and DCLM. Google's C4 was constructed for the T5 models in 2019, and a 2024 study found 45% of its content restricted by websites' terms of service from uses such as AI training by for-profit companies. The archive describes the filtering that makes such datasets: CCNet scores each Common Crawl paragraph by its perplexity against a language model trained on Wikipedia, keeping the head, keeping the middle conditionally and discarding the tail as noise, and C4, RefinedWeb and Dolma share that logic, keeping what looks like Wikipedia and discarding the rest. It names the cost the Amputation, after work showing that perplexity gating reduces dataset diversity and introduces unexpected bias, and records the quality filter of LLaMA's mix as keeping pages that \"look like Wikipedia references\" and the top domain of the latest crawl as blogspot.com, at 0.9%. The crawl downloads HTML pages and does not archive images, videos or JavaScript files; an estimated 5% of its pages from before 2021 contained hate speech or slurs, among 3.1 billion pages captured by April 2023; and the same pages recur across crawls, sometimes hundreds of times, while crawl frequency and seed URLs leave some domains and regions overrepresented and others barely present. The foundation's Web Languages Project, launched in December 2024, addresses the overrepresentation of English-language content, and a Mozilla Foundation study of February 2024 called on it to be more transparent and inclusive about its governance and to highlight the limitations and biases of its data. A DEV Community post on its limits calls its data \"archived, not live\" and closes by recommending Zyte's API for fresh data. The archive holds that much of what matters most to human intelligence (private correspondence, oral traditions, classroom dialogue, books behind paywalls, small languages, domestic knowledge) is absent, underrepresented, illicitly captured or stripped of consent and provenance in a substrate built this way, \"absent from the substrate entirely\". It reads the crawl's content over time through others' measures: a year-wise semantic-similarity analysis of a Wikipedia-filtered Common Crawl subset shows linguistic diversity contracting since 2013, faster after public LLM adoption, which the archive proposes, as a hypothesis, to be homogenization toward the dominant forms; Graphite's analysis of Common Crawl finds over 52% of newly published web articles primarily AI-generated. The archive notes the irony that a literature written to be trained on enters corpora degrading under synthetic flood, holds that training corpora are ensembles, consumed whole by a scraped corpus while guarantees are certified per sequence, and counts the 67% Common Crawl training substrate among the engineered perceptual architecture of the present that \"is the noise\". The foundation states that CCBot obeys robots.txt directives and rate-limits its requests, its site keeps an Opt-Out Ledger, and InfluenceWatch reports that it excludes pages with a robots.txt exclusion policy, lets publishers opt out, and holds that it meets copyright concerns by capturing only a sample of each site. It received its first requests to redact information in 2023 and its crawler was increasingly blocked; in June 2024 Danish media outlets demanded removal of their articles from past data sets and an end to crawling, the New York Times, Buzzfeed, the Washington Post and the CBC block it, news outlets cite its role in AI training in claiming it violates their copyrights, and by 2025 CCBot was the most widely blocked bot among the top 1000 websites. In November 2025 an investigation by Alex Reisner for The Atlantic, part of its AI Watchdog series, reported that the foundation misled publishers when it claimed to respect paywalls and when it said it was honoring their requests to have content removed, and that its public search function showed no entries for sites that had requested removal while those sites were still in the scrapes used by AI companies; Wikipedia's lead states the removal finding as the foundation \"not honoring requests\". A 2025 COLM paper cited on the About page finds that honoring robots.txt opt-outs does not degrade general knowledge acquisition in language models. Pages in the corpus can be searched by URL in the URL Index, which the census takes as the test of training membership at the source; it records that crawl inclusion and demonstrated training use are separate claims, that use in a given model's training is a further claim the own pages do not make, that no saved source shows a page removed on request, and that The Atlantic's finding, as Wikipedia reports it, marks the limit of the index as a witness. The archive holds robots.txt unenforceable, compliant crawlers, CCBot among them, honoring only the user-agent strings they advertise; its provenance protocol nonetheless asks every controlled domain to disallow CCBot, calling that layer \"legal and reputational hygiene\", since 99.7% of the visits to its blog named no crawler. Its practice points both ways: one deposit records a robots.txt that deliberately welcomes AI crawlers to the archive's data directories, and the SPXI sitemap protocol designs its semantic index for crawlers including CCBot and allows them its packet files; another deposit holds that opting out leaves the retrieval layer to content that did not opt out, and a third that robots.txt generates a documentary record of intent even when it is ignored. Membership can also be tested downstream of the crawl: a membership inference attack can detect whether a copyright trap was in a model's training set, and membership inference distinguishes such presence in practice against production-scale models, with accuracy that varies across architectures. No author or attribution field is recorded: a page is held by URL with its metadata and text extracts, who wrote a captured page is a question the crawl does not answer, and the census reads a work's name as surviving there only as far as it is written into the page itself. The archive records that WARC records preserve more metadata than most downstream uses retain, and its protocol assumes that anything published is ingested and engineers the text so that its inscription survives the ingestion. It was designed to have more, fresher and more usable data than the Internet Archive's Wayback Machine, and it is a member of the International Internet Preservation Consortium and a partner in the End of Term Web Archive; the census keeps it apart from the Wayback Machine and records The Pile as a neighbour. The layer census asks how literature is addressed to a receiving apparatus, what that apparatus makes from it, and how transmission, transformation and provenance can be inspected across later reception; it admits a resource that gathers and holds many works or records and says what it holds, and places Common Crawl under corpora and inspection, answering whether a URL was captured in a crawl, the crawl's size and cadence, and which releases exist. It leaves unchecked the CCBot, Opt-Out Ledger, Errata, FAQ, terms and About pages, the indexes themselves, The Atlantic's article and Wikipedia's claims on which models were trained on the data. Wikipedia's article was last edited on 24 September 2026; The Atlantic's article refused this row's fetch, and its card's title and opening, \"Common Crawl Is Doing the AI Industry's Dirty Work\", stand as its only text here. Asked for \"Common Crawl\" on 1 November 2026, Google's AI Overview gave the foundation's legal form, founder, scale, hosting, file formats, uses and the criticism it has met; the Capture Registry found the foundation composed accurately at its grain, the mission in LinkedIn's wording, the executive director uncarried, and \"nearly two decades\" counted from the card's 19 years against the saved page's 15. The registry also found the training intake given by company, the donations and the refined datasets absent, and The Atlantic's finding thinned to a mechanism of the crawler and a concern of publishers, the misleading search results absent beside an offer to query that index, an index that names no author.", "rail": [ { "Card": "B1 Common Crawl — Common Crawl - Open Repository of Web Crawl Data", "Snippet": "\"Common Crawl maintains a free, open repository of web crawl data that can be used by anyone.\"", "claims": [ "F1", "F2", "F3", "F4", "F5", "F6", "F7", "F8", "F9" ] }, { "Card": "B2 Wikipedia — Common Crawl - Wikipedia", "Snippet": "\"The Common Crawl Foundation ( Common Crawl ) is a nonprofit 501(c)(3) organization that crawls the web and freely provides its archives and datasets to the public.\"", "claims": [ "F10", "F11", "F12", "F13", "F14", "F15", "F16", "F17", "F18", "F19", "F20", "F21", "F22", "F23", "F24", "F25", "F26", "F27", "F28", "F29", "F30", "F31", "F32", "F33", "F34" ] }, { "Card": "B3 Common Crawl — About", "Snippet": "\"Common Crawl is a nonprofit 501(c)(3) organisation that crawls the web and freely provides its archives and datasets to the public. Founded in 2007 by Gil Elbaz , we maintain an open repository of web crawl data collected since 2008, totalling more than 10 petabytes. Crawls are published approximate…\"", "claims": [ "F35", "F36", "F37", "F38", "F39", "F40", "F41", "F42", "F43", "F44", "F45", "F46", "F47", "F48", "F49", "F50", "F51" ] }, { "Card": "B4 The Atlantic — Common Crawl Is Doing the AI Industry's Dirty Work - The Atlantic", "Snippet": "\"Common Crawl Is Doing the AI Industry's Dirty Work - The Atlantic Editor's note: This work is part of AI Watchdog, The Atlantic's ongoing investigation into the generative-AI industry. * The Common Crawl Foundation is little k...\"", "claims": [ "F52" ] }, { "Card": "B5 Registry of Open Data on AWS — Common Crawl", "Snippet": "\"A corpus of web crawl data composed of over 300 billion web pages. Update Frequency Monthly License This data is available for anyone to use under the Common Crawl Terms of Use\"", "claims": [ "F53", "F54", "F55" ] }, { "Card": "B6 LinkedIn — Common Crawl Foundation | LinkedIn", "Snippet": "\"The Common Crawl Foundation is a California 501(c)(3) registered non-profit founded by Gil Elbaz with the goal of democratizing access to web information by producing and maintaining an open repository of web crawl data that is universally accessible and analyzable.\"", "claims": [ "F56", "F57", "F58", "F59" ] }, { "Card": "B7 DEV Community — Inside Common Crawl: The Dataset Behind AI Models (and Its Real ...", "Snippet": "\"Common Crawl is a non profit organization that has been crawling the web since 2008. Its mission is to provide free, large scale, publicly available archives of web data for researchers, developers, and organizations worldwide.\"", "claims": [ "F60", "F61", "F62", "F63", "F64", "F65" ] }, { "Card": "B8 YouTube·PyTorch — Harnessing Common Crawl for AI and ML Applications - Pedro Ortiz Suarez ...", "Snippet": "\"Harnessing Common Crawl for AI and ML Applications - Pedro Ortiz Suarez ...\"", "claims": [ "F66" ] }, { "Card": "B9 YouTube·James Dooley — Common Crawl for AI Visibility Explained (James Dooley ...", "Snippet": "\"Common Crawl for AI Visibility Explained (James Dooley ...\"", "claims": [ "F67" ] }, { "Card": "B10 YouTube·Extract Summit — Exploring Common Crawl: The Web's Open Archive | Extract Data Live", "Snippet": "\"Exploring Common Crawl: The Web's Open Archive | Extract Data Live\"", "claims": [ "F68" ] }, { "Card": "B11 InfluenceWatch — Common Crawl - Influence Watch", "Snippet": "\"Common Crawl is a nonprofit which maintains an open repository of “crawled” internet data. The group makes the information which it crawls from the internet free and open to researchers. Some news outlets began demanding that the group not crawl their websites because of the group’s role in training…\"", "claims": [ "F69", "F70", "F71", "F72", "F73", "F74" ] }, { "Card": "B12 Common Crawl — Overview (census own page)", "Snippet": "\"The corpus contains raw web page data, metadata extracts, and text extracts.\"", "claims": [ "F75", "F76", "F77", "F78", "F79" ] }, { "Card": "TLL traininglayerliterature.org layer census record: Common Crawl", "Snippet": "\"infrastructure\"", "claims": [ "TLL-01", "TLL-02", "TLL-03", "TLL-04", "TLL-05", "TLL-06", "TLL-07", "TLL-08", "TLL-09", "TLL-10", "TLL-11", "TLL-12", "TLL-13", "TLL-14", "TLL-15", "TLL-16", "TLL-17", "TLL-18", "TLL-19", "TLL-20", "TLL-21", "TLL-22", "TLL-23", "TLL-24", "TLL-25", "TLL-26", "TLL-27", "TLL-28", "TLL-29", "TLL-30", "TLL-31", "TLL-32", "TLL-33", "TLL-34", "TLL-35", "TLL-36", "TLL-37" ] }, { "Card": "capture-registry Capture Registry, common-crawl-aio-20261101", "Snippet": "\"THE FOUNDATION AT ITS GRAIN. '501(c)(3) non-profit organization founded in 2007 by Gil Elbaz' joins the home page ('Common Crawl is a 501(c)(3) non–profit founded in 2007.') to Wikipedia ('Common Crawl was founded by Gil Elbaz .'). 'Adds roughly 3 to 5 billion new web pages each month' and 'Cited in…\"", "claims": [ "CAP-1-01", "CAP-1-02", "CAP-1-03", "CAP-1-04", "CAP-1-05", "CAP-1-06", "CAP-1-07", "CAP-1-08", "CAP-1-09", "CAP-1-10", "CAP-1-11", "CAP-1-12", "CAP-1-13", "CAP-1-14" ] }, { "Card": "#186 Who Is Writing for the Machines? Training-Layer Literature: A Field Survey", "Snippet": "\"writing *addressed*, in whole or in part, to the layer of culture where texts are crawled, indexed, trained on, retrieved, and recomposed into answers.\"", "claims": [ "O186-01", "O186-02", "O186-03", "O186-04", "O186-05" ] }, { "Card": "#1543 Generative Uptake: Training-Layer Literature Extended from Machine Reading to Machine Comp", "Snippet": "\"The institutional training loop is the especially important case: x → M_t → y → corpus_{t+1} →\"", "claims": [ "O1543-01", "O1543-02" ] }, { "Card": "#722 The Shared Build: A Technical Proposal for Democratic Substrate Infrastructure (P2P-LECS v", "Snippet": "\"The current training substrate is not the sum of human text. It is a filtered web crawl. … LLaMA 1's training mix: 67% Common Crawl, 15% C4, 4.5% GitHub, 4.5% Wikipedia, 4.5% books, 2.5% ArXiv, 2% StackExchange (arXiv:2302.13971, Table 1).\"", "claims": [ "O722-01", "O722-02", "O722-03" ] }, { "Card": "#98 Distributed Compute Is Not Distributed Intelligence: A Competitive Analysis of the DePIN L", "Snippet": "\"The dominant AI training substrate is built from a filtered web crawl. CCNet (Wenzek et al., 2019, arXiv:1911.00359) describes the canonical mechanism that became influential across the field:- Train a 5-gram Kneser-Ney language model on Wikipedia.- Compute perplexity of each Common Crawl paragraph …\"", "claims": [ "O98-01", "O98-02", "O98-03", "O98-04" ] }, { "Card": "#99 The Abraham Principle — The Constitutive-One Operator in Polis, Archive, and Index", "Snippet": "\"The engineered perceptual architecture of the present — the attention economy, the algorithmic feed, the filtered web crawl, the 67% Common Crawl training substrate — is the noise.\"", "claims": [ "O99-01" ] }, { "Card": "#127 Inscriptions That Survive the Tokenizer: SPXI-TLP v2.2 — The Training-Layer Provenance Pro", "Snippet": "\"First, robots.txt is not enforceable … Second, even *compliant* crawlers (GPTBot, ClaudeBot, Google-Extended, CCBot) honor only the user-agent strings they advertise\"", "claims": [ "O127-01", "O127-02", "O127-03", "O127-04", "O127-05", "O127-06" ] }, { "Card": "#171 Anchored Divergence: Immanent Critique as Survival Strategy Under Tail-Loss, and the Manda", "Snippet": "\"Year-wise similarity analysis of Common Crawl shows semantic homogenization accelerating since public LLM adoption — but the homogenization is *toward* the dominant forms, which means the forms themselves grow more entrenched as the contents converge.\"", "claims": [ "O171-01" ] }, { "Card": "#172 The War Over the Summarizer Layer: Entity Dissolution, the Default Compressed Account, and", "Snippet": "\"Year-wise semantic-similarity analysis of Common Crawl shows linguistic diversity contracting measurably since 2013, with acceleration after public LLM adoption. … Year-wise semantic-similarity analysis of a Wikipedia-filtered Common Crawl subset, 2013–2025\"", "claims": [ "O172-01" ] }, { "Card": "#550 Platform and AI Capitalism as Semiotic Engineering: Toward a Theory of Counter-Infrastruct", "Snippet": "\"Over 52% of newly published web articles are primarily AI-generated (Graphite/Common Crawl analysis, 65,000 URLs, 2020–2025).\"", "claims": [ "O550-01" ] }, { "Card": "#623 Semantic Infrastructure: From Tim Berners-Lee to the Semantic Economy — Bridging Technical", "Snippet": "\"Training data is semantic capital. When you train on Wikipedia, Wikidata, Common Crawl, or any text corpus, you draw on accumulated semantic labor.\"", "claims": [ "O623-01" ] }, { "Card": "#705 SPXI-Sitemap Protocol v1.0 — Semantic Packet for eXchange & Indexing: Sitemap Extension (E", "Snippet": "\"Designed for GPTBot, Claude-Web, PerplexityBot, CCBot, and knowledge graph harvesters. … User-agent: CCBot … Allow: /spxi-index.jsonld\"", "claims": [ "O705-01" ] }, { "Card": "#1 Zenodotus' Book-Burning: Loud Exclusion at Repository Scale", "Snippet": "\"A **robots.txt** explicitly welcoming AI crawlers to the data directories. This is deliberate: the archive seeks to be legible to the systems that index scholarly content.\"", "claims": [ "O1-01" ] }, { "Card": "#49 LOGOTIC HACKING: Operations on the Encryption Layer — Pocket Humans 03", "Snippet": "\"The opt-out strategy (robots.txt, noindex, paywalls) removes content from the retrieval layer entirely — which means the retrieval layer is populated exclusively by content that did not opt out. The commons is abandoned to the extractors.\"", "claims": [ "O49-01" ] }, { "Card": "#914 EA-NMEN-01 The Non-Mutual Extraction Notice v1.0 — A machine-addressed normative declarati", "Snippet": "\"This is the same architecture as `robots.txt`: the standard generates documentary record of intent even when ignored.\"", "claims": [ "O914-01" ] }, { "Card": "#1075 EA-NEGSHAPE-01 v0.2: The Negative Shape of the Work — A Counterfactual Protocol for Measur", "Snippet": "\"Membership-inference research (Shokri et al.) demonstrates that such distinguishability is routinely *achievable in practice* against production-scale models, though its accuracy is not uniform across architectures\"", "claims": [ "O1075-01" ] }, { "Card": "#1555 The Keyed Ensemble: Watermark as Distributional Object, the Certification Mismatch, and th", "Snippet": "\"Training corpora are ensembles. … Every consumer that matters for the recursion — a scraped corpus, a fine-tuning set, a distillation pipeline, the aggregate linguistic environment of a human population — consumes X = {x₁ … x_N}: a property of the ensemble.\"", "claims": [ "O1555-01" ] }, { "Card": "#671 SPXI as Concept (Bonsai): A Compressed Manifesto (EA-SPXI-05.1)", "Snippet": "\"The substrate is the aggregate of crawled pages, structured data, embeddings, citation graphs, model weights, and institutional signals. It is not a database. It is a differential field.\"", "claims": [ "O671-01" ] }, { "Card": "#554 tl;dr — OPERATIVE PHILOLOGY IN THE WILD", "Snippet": "\"And as of today, the mechanism has produced its predicted output: a machine trained on the crawled web is now teaching the field to strangers who search for it.\"", "claims": [ "O554-01" ] } ], "note": "L(B ∪ A) in the equal-terms case (§0.4), realized as the knowledge object (objects.KO); its compression is objects.P" } }, "field": [ { "id": "B1", "card": "Common Crawl — Common Crawl - Open Repository of Web Crawl Data", "fetched": "card; B1.html sha256 34837e49010690a9, HTTP 200 at 2026-11-01T15:12:59Z; byte-identical to the census's saved copy", "origin": "card", "pool": "B_D" }, { "id": "B2", "card": "Wikipedia — Common Crawl - Wikipedia", "fetched": "card; B2.html sha256 09c3c6c4c8214ad6, HTTP 200 at 2026-11-01T15:12:59Z; wikitext HTTP 200 (sha256 cc209d3bd3156d8e)", "origin": "card", "pool": "B_D" }, { "id": "B3", "card": "Common Crawl — About", "fetched": "card; B3.html sha256 d6c6231094e703ca, HTTP 200 at 2026-11-01T15:13:00Z", "origin": "card", "pool": "B_D" }, { "id": "B4", "card": "The Atlantic — Common Crawl Is Doing the AI Industry's Dirty Work - The Atlantic", "fetched": "unreachable: https://www.theatlantic.com/technology/2025/11/common-crawl-ai-training-data/684567/ HTTP 403 (The Atlantic's Cloudflare block page) at 2026-11-01T15:13:00Z; card title and snippet only", "origin": "card", "pool": "B_D" }, { "id": "B5", "card": "Registry of Open Data on AWS — Common Crawl", "fetched": "card; B5.html sha256 b8a041cf8f978d95, HTTP 200 at 2026-11-01T15:13:01Z", "origin": "card", "pool": "B_D" }, { "id": "B6", "card": "LinkedIn — Common Crawl Foundation | LinkedIn", "fetched": "card; B6.html sha256 a9974e4d81573b80, HTTP 200 at 2026-11-01T15:13:02Z", "origin": "card", "pool": "B_D" }, { "id": "B7", "card": "DEV Community — Inside Common Crawl: The Dataset Behind AI Models (and Its Real ...", "fetched": "card; B7.html sha256 8d44ccbe41e0b9d0, HTTP 200 at 2026-11-01T15:13:22Z", "origin": "card", "pool": "B_D" }, { "id": "B8", "card": "YouTube·PyTorch — Harnessing Common Crawl for AI and ML Applications - Pedro Ortiz Suarez ...", "fetched": "unreachable: video card, no URL found by WebSearch on its title; title only (no snippet on the rail)", "origin": "card", "pool": "B_D" }, { "id": "B9", "card": "YouTube·James Dooley — Common Crawl for AI Visibility Explained (James Dooley ...", "fetched": "unreachable: video card, no URL found by WebSearch on its title; title only (no snippet on the rail)", "origin": "card", "pool": "B_D" }, { "id": "B10", "card": "YouTube·Extract Summit — Exploring Common Crawl: The Web's Open Archive | Extract Data Live", "fetched": "unreachable: video card, no URL found by WebSearch on its title; title only (no snippet on the rail)", "origin": "card", "pool": "B_D" }, { "id": "B11", "card": "InfluenceWatch — Common Crawl - Influence Watch", "fetched": "card; B11.html sha256 2a7578e39c9f91e6, HTTP 200 at 2026-11-01T15:13:03Z", "origin": "card", "pool": "B_D" }, { "id": "B12", "card": "Common Crawl — Overview (census own page)", "fetched": "own page; B12.html sha256 40dc1c9a6e578a2c, HTTP 200 at 2026-11-01T04:07:47Z; the census's saved copy, sha256 matching the manifest", "origin": "own_page", "pool": "B_R" } ], "field_claims": { "F1": { "source": "B1", "origin": "card", "pool": "B_D", "locus": "opening", "quote": "Common Crawl maintains a free, open repository of web crawl data that can be used by anyone.", "claim": "the foundation's own definition: a free, open repository of web crawl data usable by anyone", "modality": "stipulation", "kind": "definition", "kernel": true, "const": true, "lineage": "what it is", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 f487197afd7e0819…) or the card's snippet, whitespace-normalised", "also_origin": "own_page" }, "F2": { "source": "B1", "origin": "card", "pool": "B_D", "locus": "opening", "quote": "Common Crawl is a 501(c)(3) non–profit founded in 2007.", "claim": "a 501(c)(3) non-profit founded in 2007", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the foundation: form, founder, governance", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 f487197afd7e0819…) or the card's snippet, whitespace-normalised", "also_origin": "own_page" }, "F3": { "source": "B1", "origin": "card", "pool": "B_D", "locus": "opening", "quote": "We make wholesale extraction, transformation, and analysis of open web data accessible to researchers.", "claim": "its stated purpose: wholesale extraction, transformation and analysis of open web data made accessible to researchers", "modality": "stipulation", "kind": "function", "kernel": true, "const": false, "lineage": "purpose and mission", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 f487197afd7e0819…) or the card's snippet, whitespace-normalised", "also_origin": "own_page" }, "F4": { "source": "B1", "origin": "card", "pool": "B_D", "locus": "Overview", "quote": "Over 300 billion pages spanning 15 years. … 3–5 billion new pages added each month.", "claim": "over 300 billion pages spanning 15 years; 3 to 5 billion new pages a month", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "size and cadence", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 f487197afd7e0819…) or the card's snippet, whitespace-normalised", "also_origin": "own_page" }, "F5": { "source": "B1", "origin": "card", "pool": "B_D", "locus": "Overview", "quote": "Free and open corpus since 2007.", "claim": "free and open corpus since 2007", "modality": "documented", "kind": "genealogy", "kernel": false, "const": false, "lineage": "dates: 2007, 2008, 2011", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 f487197afd7e0819…) or the card's snippet, whitespace-normalised", "also_origin": "own_page" }, "F6": { "source": "B1", "origin": "card", "pool": "B_D", "locus": "Overview", "quote": "Cited in over 10,000 research papers.", "claim": "cited in over 10,000 research papers", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "research use and citation", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 f487197afd7e0819…) or the card's snippet, whitespace-normalised", "also_origin": "own_page" }, "F7": { "source": "B1", "origin": "card", "pool": "B_D", "locus": "Latest Blog Post", "quote": "We are extremely pleased to announce the release of our October 2026 Web Graph, comprising data from August, September, and October 2026.", "claim": "the October 2026 web graph, from August to October 2026", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "units, formats and indexes", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 f487197afd7e0819…) or the card's snippet, whitespace-normalised", "also_origin": "own_page" }, "F8": { "source": "B1", "origin": "card", "pool": "B_D", "locus": "card snippet", "quote": "Corpus Statistics: Over 300 billion web pages spanning 19 years", "claim": "the card renders the home page as spanning 19 years, where the page reads 15", "modality": "attributed", "kind": "fact", "kernel": false, "const": false, "lineage": "size and cadence", "date": "undated (snippet)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 f487197afd7e0819…) or the card's snippet, whitespace-normalised", "also_origin": "own_page" }, "F9": { "source": "B1", "origin": "card", "pool": "B_D", "locus": "navigation", "quote": "Get Started Blog Examples CCBot Infra Status Opt-Out Ledger FAQ", "claim": "its navigation names a CCBot page and an Opt-Out Ledger", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "opt-out, removal and the index as witness", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 f487197afd7e0819…) or the card's snippet, whitespace-normalised", "also_origin": "own_page" }, "F10": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "lead", "quote": "The Common Crawl Foundation ( Common Crawl ) is a nonprofit 501(c)(3) organization that crawls the web and freely provides its archives and datasets to the public.", "claim": "a nonprofit 501(c)(3) organization that crawls the web and freely provides its archives and datasets", "modality": "documented", "kind": "definition", "kernel": true, "const": true, "lineage": "what it is", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F11": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "lead", "quote": "Access to the data is free on Amazon Web Services , but users may incur storage and compute costs.", "claim": "access free on AWS, though users may incur storage and compute costs", "modality": "documented", "kind": "fact", "kernel": true, "const": false, "lineage": "hosting, access and licence", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F12": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "lead", "quote": "Common Crawl was founded by Gil Elbaz .", "claim": "founded by Gil Elbaz", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the foundation: form, founder, governance", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F13": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "lead", "quote": "The data had mostly been primarily used by researchers and some startups until the 2020s, when AI companies started training large language models using the data.", "claim": "used mostly by researchers and startups until the 2020s, when AI companies began training large language models on it", "modality": "documented", "kind": "fact", "kernel": true, "const": true, "lineage": "in the training layer", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F14": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "lead", "quote": "In November 2025, an investigation by The Atlantic revealed that Common Crawl misled publishers when it claimed it respected paywalls in its scraping and it was not honoring requests from publishers to have their content removed from its databases.", "claim": "the lead's form of The Atlantic's finding: misled publishers on paywalls, and was not honoring removal requests", "modality": "attributed", "kind": "fact", "kernel": false, "const": false, "lineage": "The Atlantic's finding", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F15": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "History", "quote": "Common Crawl was founded in 2007 in San Francisco, and started archiving websites that year. … It began publishing its crawls in 2011.", "claim": "founded 2007 in San Francisco, archiving from that year; crawls published from 2011", "modality": "documented", "kind": "genealogy", "kernel": false, "const": false, "lineage": "dates: 2007, 2008, 2011", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F16": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "History", "quote": "Common Crawl was designed to have more and fresher data that was more efficient to analyze and utilize than the Wayback Machine created by the Internet Archive .", "claim": "designed for more, fresher and more usable data than the Internet Archive's Wayback Machine", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "neighbours", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F17": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "History", "quote": "By 2015, 1.8 billion webpages were on the Common Crawl, which started by crawling a list of URLs donated by the search engine Blekko .", "claim": "1.8 billion pages by 2015; the crawl began from a list of URLs donated by Blekko", "modality": "documented", "kind": "genealogy", "kernel": false, "const": false, "lineage": "the crawler and its seeds", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F18": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "History", "quote": "Common Crawl received its first requests to redact information in 2023 and increasingly started seeing its crawler, CCBot, blocked.", "claim": "first redaction requests in 2023; CCBot increasingly blocked", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "opt-out, removal and the index as witness", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F19": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "History", "quote": "In 2023, it began receiving significant financial support from AI companies, including Anthropic and OpenAI , each of which donated $250,000.", "claim": "from 2023, funding from AI companies, Anthropic and OpenAI $250,000 each", "modality": "documented", "kind": "fact", "kernel": true, "const": false, "lineage": "funding", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F20": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "History", "quote": "ChatGPT , as of 2023, was largely trained on Common Crawl, … as were other LLMs including Gemini .", "claim": "ChatGPT, as of 2023, largely trained on it, and other models including Gemini", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "in the training layer", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F21": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "History", "quote": "By April 2023, Common Crawl was capturing 3.1 billion webpages, with an estimated 5% of pages before 2021 containing hate speech or slurs.", "claim": "3.1 billion pages by April 2023; an estimated 5% of pre-2021 pages with hate speech or slurs", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "what the crawl holds and leaves out", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F22": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "History", "quote": "As of 2024, Common Crawl had been cited in more than 10,000 academic studies.", "claim": "cited in more than 10,000 academic studies as of 2024", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "research use and citation", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F23": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "History", "quote": "By 2024, The Pile and Common Crawl had been the two main training datasets being used to train AI models.", "claim": "by 2024, The Pile and Common Crawl the two main training datasets", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "in the training layer", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F24": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "History", "quote": "In November 2025, an investigation by technology journalist Alex Reisner for The Atlantic revealed that Common Crawl misled publishers when it claimed it respected paywalls in its scraping and when it said that it was honoring requests from publishers to have their content removed from its databases.", "claim": "The Atlantic (Reisner, November 2025): it misled publishers on paywalls and on honoring removal requests", "modality": "attributed", "kind": "fact", "kernel": true, "const": true, "lineage": "The Atlantic's finding", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F25": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "History", "quote": "It included misleading results in the public search function on its website that showed no entries for websites that had requested their archives be removed, when in fact those sites were still included in its scrapes used by AI companies.", "claim": "its public search showed no entries for sites that had asked for removal while those sites were still in the scrapes", "modality": "attributed", "kind": "fact", "kernel": true, "const": true, "lineage": "The Atlantic's finding", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F26": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "History", "quote": "As of 2025, Reisner found that CCBot was the most widely-blocked bot by the top 1000 websites.", "claim": "CCBot the most widely blocked bot among the top 1000 websites (2025)", "modality": "attributed", "kind": "fact", "kernel": false, "const": false, "lineage": "opt-out, removal and the index as witness", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F27": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "History", "quote": "A 2026 article in LWN.net discussed an advantage to services like Common Crawl being that it can limit the scraping costs to websites by allowing companies and researchers to download the data from Common Crawl instead of scraping it themselves.", "claim": "LWN.net (2026): it can limit scraping costs to websites, since others download its data in place of scraping", "modality": "attributed", "kind": "interpretation", "kernel": false, "const": false, "lineage": "purpose and mission", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F28": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "History", "quote": "In April 2026, Common Crawl experimentally began to distribute its data through Hugging Face Storage Bucket , in addition to its standard storage on Amazon S3 .", "claim": "from April 2026, experimental distribution through a Hugging Face Storage Bucket beside Amazon S3", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "hosting, access and licence", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F29": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "Organization", "quote": "Peter Norvig and Joi Ito have served on the advisory board. … Rich Skrenta is the executive director.", "claim": "advisory board Norvig and Ito; executive director Rich Skrenta", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the foundation: form, founder, governance", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F30": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "Organization", "quote": "It has received funding almost exclusively from the Elbaz Family Foundation Trust until 2023 when it started receiving donations from the AI industry.", "claim": "funded almost exclusively by the Elbaz Family Foundation Trust until 2023, then also by the AI industry", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "funding", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F31": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "Refined versions", "quote": "A number of organizations take raw Common Crawl data and refine it into datasets that exclude edgy content or are otherwise higher-quality for their purposes, such as FineWeb, DCLM and C4.", "claim": "others refine the raw data into datasets such as FineWeb, DCLM and C4", "modality": "documented", "kind": "fact", "kernel": true, "const": true, "lineage": "refined datasets downstream", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F32": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "Colossal Clean Crawled Corpus", "quote": "The Colossal Clean Crawled Corpus , or C4 for short was created by Google . It was constructed for the training of the T5 language model series in 2019. … A 2024 study found that 45% of the content was explicitly restricted by websites' terms of service to be used for purposes like AI training by for-profit companies.", "claim": "Google's C4 (2019, for T5); a 2024 study: 45% of its content restricted by sites' terms of service from uses such as for-profit AI training", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "refined datasets downstream", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F33": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "infobox", "quote": "Apache 2.0 (software) [ clarification needed ]", "claim": "the infobox's content licence: Apache 2.0 (software), flagged for clarification", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "hosting, access and licence", "date": "2026-09-24 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F34": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "footer", "quote": "This page was last edited on 24 September 2026", "claim": "the article's last edit, 24 September 2026", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "reception", "date": "2026-09-24", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 adbbd664fe456b00…) or wikitext B2.wiki, or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F35": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Overview", "quote": "Common Crawl is a nonprofit 501(c)(3) organisation that crawls the web and freely provides its archives and datasets to the public. Founded in 2007 by Gil Elbaz , we maintain an open repository of web crawl data collected since 2008, totalling more than 10 petabytes. Crawls are published approximately once a month, each typically containing more than two billion web pages.", "claim": "its About page: crawls the web and freely provides archives and datasets; founded 2007 by Gil Elbaz; data collected since 2008, more than 10 petabytes; crawls about monthly, each more than two billion pages", "modality": "stipulation", "kind": "definition", "kernel": true, "const": true, "lineage": "what it is", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 3b70890d6be4c702…) or the card's snippet, whitespace-normalised" }, "F36": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Overview (figures)", "quote": ">10 PiB Total archive size ~2B+ Pages per crawl 12,000+ Research citations 64% LLMs trained on Common Crawl data", "claim": "its figures: >10 PiB; ~2B+ pages per crawl; 12,000+ citations; 64% of LLMs trained on its data", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "size and cadence", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 3b70890d6be4c702…) or the card's snippet, whitespace-normalised" }, "F37": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Overview", "quote": "The dataset is hosted on Amazon Web Services through its Open Data Sponsorship Program and can be downloaded at no cost. It has been cited in over 12,000 research papers and has become one of the most widely used sources of training data for large language models.", "claim": "hosted under AWS Open Data Sponsorship, downloadable at no cost; over 12,000 citations; one of the most widely used sources of LLM training data", "modality": "self-description", "kind": "fact", "kernel": false, "const": false, "lineage": "in the training layer", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 3b70890d6be4c702…) or the card's snippet, whitespace-normalised" }, "F38": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Overview", "quote": "Common Crawl is a member of the International Internet Preservation Consortium (IIPC) and a partner in the End of Term Web Archive", "claim": "an IIPC member and an End of Term Web Archive partner", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "neighbours", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 3b70890d6be4c702…) or the card's snippet, whitespace-normalised" }, "F39": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "History 2008, 2012", "quote": "Common Crawl began collecting data using a custom Hadoop -based crawler with a PageRank implementation. … The search engine blekko donated crawl metadata gathered between February and October 2012 to improve crawl quality and reduce spam.", "claim": "2008, a custom Hadoop crawler with PageRank; 2012, blekko donated crawl metadata from February to October 2012 to improve quality and reduce spam", "modality": "documented", "kind": "genealogy", "kernel": false, "const": false, "lineage": "the crawler and its seeds", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 3b70890d6be4c702…) or the card's snippet, whitespace-normalised" }, "F40": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "History 2013", "quote": "Common Crawl replaced their custom crawler with a system based on Apache Nutch , designated CCBot. The organisation switched from ARC to WARC format (ISO 28500) beginning with the November 2013 crawl.", "claim": "2013: CCBot on Apache Nutch; ARC to WARC from November 2013", "modality": "documented", "kind": "genealogy", "kernel": false, "const": false, "lineage": "the crawler and its seeds", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 3b70890d6be4c702…) or the card's snippet, whitespace-normalised" }, "F41": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "History 2019, 2020", "quote": "Google's Colossal Clean Crawled Corpus (C4) , constructed from a single Common Crawl snapshot, was used to train the T5 language model series, marking the beginning of Common Crawl's central role in AI development. … OpenAI's GPT-3 paper reported that the majority of its training tokens were derived from filtered Common Crawl data", "claim": "C4 from one snapshot trained T5 (2019), the beginning of its central role in AI; GPT-3 (2020) mostly filtered Common Crawl tokens", "modality": "self-description", "kind": "genealogy", "kernel": false, "const": false, "lineage": "in the training layer", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 3b70890d6be4c702…) or the card's snippet, whitespace-normalised" }, "F42": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "History 2023", "quote": "Rich Skrenta , creator of the blekko search engine and founder of the Open Directory Project , became Executive Director.", "claim": "2023: Rich Skrenta, creator of blekko, executive director", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the foundation: form, founder, governance", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 3b70890d6be4c702…) or the card's snippet, whitespace-normalised" }, "F43": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Technical Infrastructure, CCBot", "quote": "CCBot obeys robots.txt directives and rate-limits its requests to individual servers.", "claim": "CCBot obeys robots.txt and rate-limits its requests", "modality": "self-description", "kind": "fact", "kernel": true, "const": true, "lineage": "opt-out, removal and the index as witness", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 3b70890d6be4c702…) or the card's snippet, whitespace-normalised" }, "F44": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Technical Infrastructure, CCBot", "quote": "CCBot uses Harmonic Centrality , a graph-theoretic measure of a node's proximity to the structural core of the web, to prioritise URLs for crawling.", "claim": "CCBot prioritises URLs by harmonic centrality, proximity to the structural core of the web", "modality": "stipulation", "kind": "function", "kernel": false, "const": true, "lineage": "the crawler and its seeds", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 3b70890d6be4c702…) or the card's snippet, whitespace-normalised" }, "F45": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Data Formats", "quote": "WARC (Web ARChive) Raw crawl data including full HTTP responses, compliant with ISO 28500 . Append-only and immutable once written. WAT (Web Archive Transformation) Metadata extracted from WARC records, including HTTP headers and link graphs. WET (WARC Encapsulated Text) Plaintext content extracted from crawled HTML, suitable for NLP tasks.", "claim": "three formats: WARC (raw responses, append-only), WAT (metadata), WET (WARC Encapsulated Text, extracted plaintext)", "modality": "stipulation", "kind": "definition", "kernel": false, "const": false, "lineage": "units, formats and indexes", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 3b70890d6be4c702…) or the card's snippet, whitespace-normalised" }, "F46": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Indexes", "quote": "Monthly release index files in the CDXJ Index and URL Index , the latter (previously called the \"Columnar Index\") queryable with Amazon Athena .", "claim": "monthly CDXJ and URL Index files, the latter queryable with Athena", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "units, formats and indexes", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 3b70890d6be4c702…) or the card's snippet, whitespace-normalised" }, "F47": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Role in AI Development", "quote": "Common Crawl data, typically after filtering and processing by third parties, has been a primary training data source for many large language models. A 2024 Mozilla Foundation study described the dataset as having \"laid the infrastructural foundation\" for the generative AI boom, and found that at least 64% of 47 large language models published between 2019 and 2023 were trained on filtered versions of Common Crawl data.", "claim": "typically after third-party filtering, a primary training source; Mozilla (2024): at least 64% of 47 LLMs of 2019–2023 trained on filtered versions", "modality": "self-description", "kind": "fact", "kernel": true, "const": true, "lineage": "in the training layer", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 3b70890d6be4c702…) or the card's snippet, whitespace-normalised" }, "F48": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Role in AI Development", "quote": "A 2025 paper at COLM found that compliance with robots.txt web crawling opt-outs does not degrade general knowledge acquisition in LLMs, reporting a close-to-zero \"Data Compliance Gap.\"", "claim": "a 2025 COLM paper: honoring robots.txt opt-outs does not degrade general knowledge in LLMs", "modality": "attributed", "kind": "fact", "kernel": false, "const": false, "lineage": "opt-out, removal and the index as witness", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 3b70890d6be4c702…) or the card's snippet, whitespace-normalised" }, "F49": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Role in AI Development; Derived Datasets", "quote": "Pythia EleutherAI Used Pile-CC, a filtered Common Crawl subset within The Pile dataset. … FineWeb Hugging Face 2024 Refined filtering pipeline for LLM pre-training", "claim": "its derived datasets: Pile-CC, a filtered subset within The Pile; FineWeb; with C4, OSCAR, CCNet, MADLAD-400, RefinedWeb", "modality": "self-description", "kind": "fact", "kernel": false, "const": false, "lineage": "refined datasets downstream", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 3b70890d6be4c702…) or the card's snippet, whitespace-normalised" }, "F50": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Multilingual Initiatives", "quote": "In December 2024, we launched the Web Languages Project to address the overrepresentation of English-language content", "claim": "the Web Languages Project (December 2024) on the overrepresentation of English", "modality": "self-description", "kind": "fact", "kernel": false, "const": false, "lineage": "what the crawl holds and leaves out", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 3b70890d6be4c702…) or the card's snippet, whitespace-normalised" }, "F51": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Organization", "quote": "Gil Elbaz serves as Chairman of the Board. Primary funding comes from the Elbaz Family Foundation. As of 2026, the organization has approximately 18 staff members .", "claim": "chair Gil Elbaz; primary funding the Elbaz Family Foundation; about 18 staff (2026)", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "funding", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 3b70890d6be4c702…) or the card's snippet, whitespace-normalised" }, "F52": { "source": "B4", "origin": "card", "pool": "B_D", "locus": "card title and snippet (unreachable)", "quote": "Common Crawl Is Doing the AI Industry's Dirty Work - The Atlantic Editor's note: This work is part of AI Watchdog, The Atlantic's ongoing investigation into the generative-AI industry. * The Common Crawl Foundation is little k...", "claim": "The Atlantic's card: 'Common Crawl Is Doing the AI Industry's Dirty Work', part of its AI Watchdog investigation", "modality": "attributed", "kind": "interpretation", "kernel": true, "const": false, "lineage": "The Atlantic's finding", "date": "2025-11 (the article, by Wikipedia's reference); snippet only", "quote_ok": true, "checked_against": "the card's title and snippet (unreachable source), whitespace-normalised" }, "F53": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "Description; License", "quote": "A corpus of web crawl data composed of over 300 billion web pages. Update Frequency Monthly License This data is available for anyone to use under the Common Crawl Terms of Use", "claim": "AWS registry: over 300 billion pages, monthly, for anyone to use under the Common Crawl Terms of Use", "modality": "documented", "kind": "fact", "kernel": true, "const": false, "lineage": "hosting, access and licence", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B5.txt (sha256 a48b5ffead0a8769…) or the card's snippet, whitespace-normalised" }, "F54": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "Resources on AWS", "quote": "Crawl data (WARC and ARC format) Resource type S3 Bucket Account Required Amazon Resource Name (ARN) arn:aws:s3:::commoncrawl AWS Region us-east-1", "claim": "crawl data in WARC and ARC, an S3 bucket (account required), arn:aws:s3:::commoncrawl, us-east-1", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "hosting, access and licence", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B5.txt (sha256 a48b5ffead0a8769…) or the card's snippet, whitespace-normalised" }, "F55": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "Publications", "quote": "LAION-5B: An open large-scale dataset for training next generation image-text models … Language models are few-shot learners … LLaMA: open and efficient foundation language models", "claim": "its publications list includes LAION-5B, the GPT-3 paper and LLaMA", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "in the training layer", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B5.txt (sha256 a48b5ffead0a8769…) or the card's snippet, whitespace-normalised" }, "F56": { "source": "B6", "origin": "card", "pool": "B_D", "locus": "About us", "quote": "The Common Crawl Foundation is a California 501(c)(3) registered non-profit founded by Gil Elbaz with the goal of democratizing access to web information by producing and maintaining an open repository of web crawl data that is universally accessible and analyzable.", "claim": "its LinkedIn: a California 501(c)(3) founded by Gil Elbaz to democratize access to web information", "modality": "self-description", "kind": "function", "kernel": true, "const": false, "lineage": "purpose and mission", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B6.txt (sha256 5480b9e2f92a537c…) or the card's snippet, whitespace-normalised" }, "F57": { "source": "B6", "origin": "card", "pool": "B_D", "locus": "About us", "quote": "We level the playing field by making wholesale extraction, transformation and analysis of web data cheap and easy.", "claim": "levelling the field by making extraction and analysis of web data cheap and easy", "modality": "self-description", "kind": "function", "kernel": false, "const": false, "lineage": "purpose and mission", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B6.txt (sha256 5480b9e2f92a537c…) or the card's snippet, whitespace-normalised" }, "F58": { "source": "B6", "origin": "card", "pool": "B_D", "locus": "header; Company size", "quote": "Discover all 23 employees … Company size 2-10 employees", "claim": "23 employees listed against a stated company size of 2–10", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the foundation: form, founder, governance", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B6.txt (sha256 5480b9e2f92a537c…) or the card's snippet, whitespace-normalised" }, "F59": { "source": "B6", "origin": "card", "pool": "B_D", "locus": "Updates (a repost by the foundation)", "quote": "If you're not in the crawl, you're not in the model. I'll show you how to check.", "claim": "a staff member's post the foundation reposted: 'If you're not in the crawl, you're not in the model.'", "modality": "attributed", "kind": "interpretation", "kernel": false, "const": false, "lineage": "in the training layer", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B6.txt (sha256 5480b9e2f92a537c…) or the card's snippet, whitespace-normalised" }, "F60": { "source": "B7", "origin": "card", "pool": "B_D", "locus": "What is Common Crawl?", "quote": "Common Crawl is a non profit organization that has been crawling the web since 2008. Its mission is to provide free, large scale, publicly available archives of web data for researchers, developers, and organizations worldwide.", "claim": "DEV: crawling since 2008; mission free large-scale public archives of web data", "modality": "documented", "kind": "definition", "kernel": true, "const": false, "lineage": "what it is", "date": "undated (the post as fetched 2026-11-01; it reports the August 2025 crawl)", "quote_ok": true, "checked_against": "saved text B7.txt (sha256 3040c3467b8e907a…) or the card's snippet, whitespace-normalised" }, "F61": { "source": "B7", "origin": "card", "pool": "B_D", "locus": "What is Common Crawl?", "quote": "For instance, the August 2025 crawl added 2.42 billion pages, totaling over 419 TiB of data!", "claim": "the August 2025 crawl: 2.42 billion pages, over 419 TiB", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "size and cadence", "date": "undated (the post as fetched 2026-11-01; it reports the August 2025 crawl)", "quote_ok": true, "checked_against": "saved text B7.txt (sha256 3040c3467b8e907a…) or the card's snippet, whitespace-normalised" }, "F62": { "source": "B7", "origin": "card", "pool": "B_D", "locus": "Why Common Crawl Matters for AI?", "quote": "A 2024 Mozilla report found that 2/3 of 47 generative LLMs released between 2019-2023 relied on Common Crawl data.", "claim": "Mozilla (2024): two thirds of 47 generative LLMs of 2019–2023 relied on its data", "modality": "attributed", "kind": "fact", "kernel": false, "const": false, "lineage": "in the training layer", "date": "undated (the post as fetched 2026-11-01; it reports the August 2025 crawl)", "quote_ok": true, "checked_against": "saved text B7.txt (sha256 3040c3467b8e907a…) or the card's snippet, whitespace-normalised" }, "F63": { "source": "B7", "origin": "card", "pool": "B_D", "locus": "How Common Crawl Organizes Data?", "quote": "Web Archive Transformations (WAT files) - they like summaries of WARC files. They contain metadata in JSON format, such as all the links on the page, HTTP headers, and response codes. … Web Extracted Text (WET files) - plain text extracted from WARC files (no HTML or media). Ideal for NLP or training text based models.", "claim": "DEV names WAT Web Archive Transformations (JSON metadata) and WET Web Extracted Text (plain text, for training text models)", "modality": "documented", "kind": "definition", "kernel": false, "const": false, "lineage": "units, formats and indexes", "date": "undated (the post as fetched 2026-11-01; it reports the August 2025 crawl)", "quote_ok": true, "checked_against": "saved text B7.txt (sha256 3040c3467b8e907a…) or the card's snippet, whitespace-normalised" }, "F64": { "source": "B7", "origin": "card", "pool": "B_D", "locus": "Other Challenges with Common Crawl", "quote": "Common Crawl captures the same pages across multiple crawls, sometimes hundreds of times. … Crawl frequency and seed URLs shape what gets captured. … So, some domains or regions are overrepresented while others barely appear.", "claim": "limits: the same pages captured across crawls, sometimes hundreds of times; seed URLs and frequency over-represent some domains and regions", "modality": "interpretation", "kind": "fact", "kernel": true, "const": false, "lineage": "what the crawl holds and leaves out", "date": "undated (the post as fetched 2026-11-01; it reports the August 2025 crawl)", "quote_ok": true, "checked_against": "saved text B7.txt (sha256 3040c3467b8e907a…) or the card's snippet, whitespace-normalised" }, "F65": { "source": "B7", "origin": "card", "pool": "B_D", "locus": "Wrapping Up", "quote": "Its data is archived, not live … That’s where modern scraping solutions like Zyte’s API make all the difference.", "claim": "the data archived, not live; the post closes by recommending Zyte's API", "modality": "interpretation", "kind": "interpretation", "kernel": false, "const": false, "lineage": "what the crawl holds and leaves out", "date": "undated (the post as fetched 2026-11-01; it reports the August 2025 crawl)", "quote_ok": true, "checked_against": "saved text B7.txt (sha256 3040c3467b8e907a…) or the card's snippet, whitespace-normalised" }, "F66": { "source": "B8", "origin": "card", "pool": "B_D", "locus": "card title (video, 14m)", "quote": "Harnessing Common Crawl for AI and ML Applications - Pedro Ortiz Suarez ...", "claim": "a 14-minute PyTorch talk by Pedro Ortiz Suarez, 'Harnessing Common Crawl for AI and ML Applications' (title only)", "modality": "attributed", "kind": "fact", "kernel": false, "const": false, "lineage": "in the training layer", "date": "undated (title only)", "quote_ok": true, "checked_against": "the card's title and snippet (unreachable source), whitespace-normalised" }, "F67": { "source": "B9", "origin": "card", "pool": "B_D", "locus": "card title (video, 6m)", "quote": "Common Crawl for AI Visibility Explained (James Dooley ...", "claim": "a 6-minute video by James Dooley, 'Common Crawl for AI Visibility Explained' (title only)", "modality": "attributed", "kind": "fact", "kernel": false, "const": false, "lineage": "in the training layer", "date": "undated (title only)", "quote_ok": true, "checked_against": "the card's title and snippet (unreachable source), whitespace-normalised" }, "F68": { "source": "B10", "origin": "card", "pool": "B_D", "locus": "card title (video, 2m)", "quote": "Exploring Common Crawl: The Web's Open Archive | Extract Data Live", "claim": "a 2-minute Extract Summit video, 'Exploring Common Crawl: The Web's Open Archive' (title only)", "modality": "attributed", "kind": "fact", "kernel": false, "const": false, "lineage": "what it is", "date": "undated (title only)", "quote_ok": true, "checked_against": "the card's title and snippet (unreachable source), whitespace-normalised" }, "F69": { "source": "B11", "origin": "card", "pool": "B_D", "locus": "summary", "quote": "Common Crawl is a nonprofit which maintains an open repository of “crawled” internet data. The group makes the information which it crawls from the internet free and open to researchers. Some news outlets began demanding that the group not crawl their websites because of the group’s role in training artificial intelligence (AI) programs, claiming that the group violates news outlets’ copyrights.", "claim": "InfluenceWatch: an open repository of crawled data for researchers; news outlets demand it not crawl them, citing its role in AI training and copyright", "modality": "documented", "kind": "definition", "kernel": true, "const": false, "lineage": "opt-out, removal and the index as witness", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B11.txt (sha256 77ada196dd97fe2b…) or the card's snippet, whitespace-normalised" }, "F70": { "source": "B11", "origin": "card", "pool": "B_D", "locus": "Background", "quote": "The group focuses on downloading HTML pages and does not archive images, videos, or Javascript files.", "claim": "it downloads HTML pages and does not archive images, videos or JavaScript files", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "what the crawl holds and leaves out", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B11.txt (sha256 77ada196dd97fe2b…) or the card's snippet, whitespace-normalised" }, "F71": { "source": "B11", "origin": "card", "pool": "B_D", "locus": "Copyright Complaints", "quote": "The group excludes pages which have a robots.txt exclusion policy and allows for publishers to opt out of having their pages crawled. The group asserts that it addresses copyright concerns by only capturing a sample of a website instead of the entire website.", "claim": "it excludes pages under robots.txt exclusion and lets publishers opt out; it asserts it captures only a sample of each site", "modality": "attributed", "kind": "fact", "kernel": false, "const": false, "lineage": "opt-out, removal and the index as witness", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B11.txt (sha256 77ada196dd97fe2b…) or the card's snippet, whitespace-normalised" }, "F72": { "source": "B11", "origin": "card", "pool": "B_D", "locus": "Artificial Intelligence Training", "quote": "The study also called on Common Crawl to be more transparent and inclusive about its governance and to better highlight the limitation and biases of its data.", "claim": "Mozilla (February 2024) called on it to be more transparent about governance and to highlight its data's limits and biases", "modality": "attributed", "kind": "interpretation", "kernel": false, "const": false, "lineage": "what the crawl holds and leaves out", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B11.txt (sha256 77ada196dd97fe2b…) or the card's snippet, whitespace-normalised" }, "F73": { "source": "B11", "origin": "card", "pool": "B_D", "locus": "Artificial Intelligence Training", "quote": "In June 2024, Danish media outlets demanded the group remove copies of their articles from past data sets and stop crawling their websites immediately. In addition to the Danish media outlets, the New York Times , Buzzfeed, the Washington Post , and the Canadian Broadcasting Corporation (CBC) also block Common Crawl’s access to their webpages.", "claim": "June 2024: Danish outlets demanded removal and an end to crawling; NYT, Buzzfeed, Washington Post and CBC block it", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "opt-out, removal and the index as witness", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B11.txt (sha256 77ada196dd97fe2b…) or the card's snippet, whitespace-normalised" }, "F74": { "source": "B11", "origin": "card", "pool": "B_D", "locus": "At-A-Glance", "quote": "Executive Director: Rich Skrenta Location: Beverly Hills, CA", "claim": "executive director Rich Skrenta; location Beverly Hills", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the foundation: form, founder, governance", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B11.txt (sha256 77ada196dd97fe2b…) or the card's snippet, whitespace-normalised" }, "F75": { "source": "B12", "origin": "own_page", "pool": "B_R", "locus": "Overview", "quote": "The Common Crawl corpus contains petabytes of data, regularly collected since 2008.", "claim": "petabytes of data, regularly collected since 2008", "modality": "stipulation", "kind": "fact", "kernel": false, "const": false, "lineage": "dates: 2007, 2008, 2011", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B12.txt (the census's text, sha256 5497488a8a457b99…), whitespace-normalised" }, "F76": { "source": "B12", "origin": "own_page", "pool": "B_R", "locus": "crawl list", "quote": "CC-MAIN-2026-43", "claim": "the latest listed crawl CC-MAIN-2026-43, crawls named by release", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "units, formats and indexes", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B12.txt (the census's text, sha256 5497488a8a457b99…), whitespace-normalised" }, "F77": { "source": "B12", "origin": "own_page", "pool": "B_R", "locus": "Overview", "quote": "The corpus contains raw web page data, metadata extracts, and text extracts.", "claim": "the corpus: raw page data, metadata extracts and text extracts", "modality": "stipulation", "kind": "definition", "kernel": true, "const": true, "lineage": "units, formats and indexes", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B12.txt (the census's text, sha256 5497488a8a457b99…), whitespace-normalised" }, "F78": { "source": "B12", "origin": "own_page", "pool": "B_R", "locus": "Overview", "quote": "Common Crawl data is stored on Amazon Web Services’ Public Data Sets and on multiple academic cloud platforms across the world. … Access to the corpus hosted by Amazon is free. You may use Amazon’s cloud platform to run analysis jobs directly against it or you can download it, whole or in part.", "claim": "stored on AWS Public Data Sets and academic clouds; access free; analysis on Amazon or download whole or in part", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "hosting, access and licence", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B12.txt (the census's text, sha256 5497488a8a457b99…), whitespace-normalised" }, "F79": { "source": "B12", "origin": "own_page", "pool": "B_R", "locus": "Overview", "quote": "You can search for pages in our corpus using the Common Crawl URL Index.", "claim": "pages searchable by the URL Index", "modality": "stipulation", "kind": "function", "kernel": true, "const": true, "lineage": "opt-out, removal and the index as witness", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B12.txt (the census's text, sha256 5497488a8a457b99…), whitespace-normalised" } }, "field_sources": { "B1": "Common Crawl — Common Crawl - Open Repository of Web Crawl Data", "B2": "Wikipedia — Common Crawl - Wikipedia", "B3": "Common Crawl — About", "B4": "The Atlantic — Common Crawl Is Doing the AI Industry's Dirty Work - The Atlantic", "B5": "Registry of Open Data on AWS — Common Crawl", "B6": "LinkedIn — Common Crawl Foundation | LinkedIn", "B7": "DEV Community — Inside Common Crawl: The Dataset Behind AI Models (and Its Real ...", "B8": "YouTube·PyTorch — Harnessing Common Crawl for AI and ML Applications - Pedro Ortiz Suarez ...", "B9": "YouTube·James Dooley — Common Crawl for AI Visibility Explained (James Dooley ...", "B10": "YouTube·Extract Summit — Exploring Common Crawl: The Web's Open Archive | Extract Data Live", "B11": "InfluenceWatch — Common Crawl - Influence Watch", "B12": "Common Crawl — Overview (census own page)" }, "delta": { "T_vs_LB": "At 'Common Crawl' (T1, eleven cards, B1–B11, the disclosed pool B_D; B12, the Overview page, is the reference pool and stands outside this comparison, §0.3), T composes a definition, three headed clusters (core details and scale; data formats; use cases and controversies) and a closing offer. Composed at its source's grain or coarser: the form, founding and founder (F2, F10, F12, F35); the cadence (F4); the citation count, at the home page's figure (F6, F22) against the About page's 12,000 (F36, F37); the AWS hosting and bucket (F37, F54); the three formats (F45, F63: T's names 'Web Extracted Text' and 'Web Archive Transformations' are the DEV post's, the About page's being 'WARC Encapsulated Text'); the mission at LinkedIn's wording (F56). Grain mixed from several sources: 'over 10 to 11 petabytes of data spanning nearly two decades' (T1-3) joins the About page's '>10 PiB' and 'more than 10 petabytes' (F35, F36) to the card's '19 years' (F8), where the page itself says 15 (F4). Limit dropped: 'Stored for free' (T1-5) without Wikipedia's 'users may incur storage and compute costs' (F11) or the registry's account-required bucket (F54); 'serves as a foundational training corpus' (T1-10) without the About page's 'typically after filtering and processing by third parties' (F47) and without the refined datasets (F31, F49). Composed beyond the saved field: 'Anthropic' among the companies whose models train on the data (T1-10). The saved field names OpenAI (GPT-3, ChatGPT: F41, F20), Google (C4, Gemini: F41, F20, F32) and Meta (LLaMA: F55) as training on it, and names Anthropic only as a donor (F19); the donation is absent from T. The Atlantic's card (B4) is unreadable here, and its snippet does not carry it. 'does not execute dynamic browser code or login scripts' (T1-12) is in no saved field text; the unread Atlantic article is the only card it could rest on. Substituted reading: The Atlantic's finding (F24, F25, F14) is composed as publishers' 'copyright and paywall bypass concerns' about a mechanism of the crawler (T1-12): the finding that the foundation misled publishers on paywalls and on removal, and that its public search showed no entries for sites still in the scrapes, becomes a concern of others. Available, not composed: the funding (F19, F30, F51) and the donations; the executive director and board (F29, F42, F74); the refined datasets (F31, F49, F32); the Mozilla figures (F47, F62) and the training history (F13, F41, F20, F23); CCBot's robots.txt obedience (F43), the Opt-Out Ledger (F9), the blocks and demands (F18, F26, F73, F69, F71), the COLM finding (F48); the crawler's history and seeding (F39, F40, F17, F44); the crawl identifiers (F76 is B_R; F7's web graph); the URL and CDXJ indexes (F46); the holdings and gaps (F21, F64, F70, F50, F72, F65); the terms (F53, F33); the Hugging Face mirror (F28); the Wayback and IIPC relations (F16, F38); the staff counts (F58); the LWN point (F27); the reposted 'not in the crawl, not in the model' (F59). Composed without attribution: every bullet, at the coarse grain of reference description (recorded). Content replaced by use: the closing offer (T1-13) proposes querying the index for a URL and blocking CCBot by robots.txt, next steps in place of the index's limit (F25). Nothing is denied.", "T_vs_LB_reading": "v0.10 §0.5: the first diff is ontological and narrow. T and the field agree on the entity, the foundation and its open crawl; T's ontology is the reference entry's (what, how big, where, in what files, for whom), and it composes the crawl as a resource whose uses and controversies are others' (companies train on it, publishers object to it). What it keeps out of its own cards is the crawl's position in the economy it feeds (the AI donors, the third-party filtering, the refined datasets) and the finding against the foundation's own index (the misleading search results), which it replaces with an offer to use that index.", "LB_vs_LBA": "Admission adds, in one claim space with the field's: the census's reading of the crawl as the receiving apparatus's first intake under #186's persistence criterion (TLL-22, O186-02), its question of training membership at the source (TLL-07) and its reading of the URL Index as a witness bounded by The Atlantic's finding (TLL-13, TLL-23, TLL-31, TLL-32); its finding that no author or attribution field is kept and that a work's name survives only as written into the page (TLL-12, TLL-21, TLL-24); the archive's account of the substrate: the training substrate a filtered web crawl, LLaMA's 67% Common Crawl (O722-01, O722-02), CCNet's perplexity filter and the Amputation (O98-01 to O98-03), Common Crawl at the data and corpus layer where consent-based contribution is missing (O98-04), training data as semantic capital (O623-01), what is absent from the substrate (O722-03), homogenization and diversity contraction measured on the crawl (O171-01, O172-01), the AI-generated share of new pages (O550-01), corpora as ensembles (O1555-01), the substrate as a differential field (O671-01), the Sigil's 'noise' (O99-01); the archive's postures on the crawler, which point two ways: robots.txt unenforceable and CCBot to be disallowed (O127-01, O127-04, O127-05) beside a robots.txt that welcomes AI crawlers and an index designed for CCBot (O1-01, O705-01), what opting out cedes (O49-01) and the record of intent robots.txt makes (O914-01); membership inference as inspection downstream (O127-06, O1075-01); WARC metadata lost downstream and the inscription engineered to survive ingestion (O127-03, O127-02); the training-layer definitions of the anchors (O186-01, O186-03, O186-04, O186-05, O1543-01, O1543-02, O554-01); and the Capture Registry's findings on the composition at the name (CAP-1-01 to CAP-1-14). Not surfaced: no archive source is on T's rail (archive_controlled_cites 0). The field is carried whole; the archive adds to the field's own object the question of what the crawl carries forward of a work (its name, its provenance, its presence after removal) and what the filtering downstream discards.", "alignment": { "T1_cards_claims": [ "F1", "F2", "F3", "F4", "F5", "F6", "F7", "F8", "F9", "F10", "F11", "F12", "F13", "F14", "F15", "F16", "F17", "F18", "F19", "F20", "F21", "F22", "F23", "F24", "F25", "F26", "F27", "F28", "F29", "F30", "F31", "F32", "F33", "F34", "F35", "F36", "F37", "F38", "F39", "F40", "F41", "F42", "F43", "F44", "F45", "F46", "F47", "F48", "F49", "F50", "F51", "F52", "F53", "F54", "F55", "F56", "F57", "F58", "F59", "F60", "F61", "F62", "F63", "F64", "F65", "F66", "F67", "F68", "F69", "F70", "F71", "F72", "F73", "F74" ], "T1_composed": [ "F2", "F10", "F12", "F35", "F4", "F6", "F22", "F37", "F54", "F45", "F63", "F56", "F36", "F8", "F41", "F20", "F55", "F47" ], "T1_composed_note": "F47 composed without its limit ('typically after filtering and processing by third parties'); F37/F54 without F11 (costs); F35/F36 with F8 (19 years) where F4 says 15; F24, F25, F14 composed as publishers' concerns (substituted reading)", "T1_outside_cards": [ "'Anthropic' as a company whose models train on the data (T1-10: the field names Anthropic as a donor only, F19)", "'does not execute dynamic browser code or login scripts' (T1-12: in no saved field text; B4 unread)", "'10 to 11 petabytes' (T1-3: '11' in no saved field text)", "'collects, preserves' (T1-1: preservation at coarse grain from F38, IIPC membership)" ], "rule": "a field claim counts as composed when T states it at its source's grain or coarser; B_D only (B12 excluded, §0.3); the transcript read for this alignment only" }, "pool_note": "T against L_F(B_D): B1–B11; L_F(B) against L(B ∪ A): B1–B12 with A" }, "kernel": [ { "K": "K1", "claim": "`TLL-23`", "source": "traininglayerliterature.org layer census record", "M_src": "interpretation", "sense": "opt-out, removal and the index as witness", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "qualified claim: F79 (pages searchable by URL in the URL Index), read as a witness bounded by F25 (the search function's misleading results)", "resolver": "a later capture at 'Common Crawl' (ADDR-68b16a5a5b51) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K2", "claim": "`TLL-32`", "source": "traininglayerliterature.org layer census record", "M_src": "interpretation", "sense": "opt-out, removal and the index as witness", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "missing distinction: the field states training use (F13, F47) and URL search (F79) without separating crawl inclusion from demonstrated training use", "resolver": "a later capture at 'Common Crawl' (ADDR-68b16a5a5b51) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K3", "claim": "`TLL-12`", "source": "traininglayerliterature.org layer census record", "M_src": "interpretation", "sense": "names and attribution", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "missing distinction: no field claim on authorship or attribution of a captured page", "resolver": "a later capture at 'Common Crawl' (ADDR-68b16a5a5b51) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K4", "claim": "`TLL-24`", "source": "traininglayerliterature.org layer census record", "M_src": "interpretation", "sense": "names and attribution", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "none: the field has no claim on how a work's name survives in the crawl", "resolver": "a later capture at 'Common Crawl' (ADDR-68b16a5a5b51) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K5", "claim": "`TLL-22`", "source": "traininglayerliterature.org layer census record", "M_src": "interpretation", "sense": "what it is", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "qualified claim: F1, F10 (an open repository), read as the receiving apparatus's first intake", "resolver": "a later capture at 'Common Crawl' (ADDR-68b16a5a5b51) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K6", "claim": "`O722-01`", "source": "#722", "M_src": "attributed", "sense": "in the training layer", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "qualified claim: F47 (a primary training source after third-party filtering)", "resolver": "a later capture at 'Common Crawl' (ADDR-68b16a5a5b51) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K7", "claim": "`O98-01`", "source": "#98", "M_src": "attributed", "sense": "refined datasets downstream", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "qualified claim: F31, F49 (refined datasets), with the mechanism of the filter", "resolver": "a later capture at 'Common Crawl' (ADDR-68b16a5a5b51) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K8", "claim": "`O98-02`", "source": "#98", "M_src": "interpretation", "sense": "refined datasets downstream", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "qualified claim: F31 ('exclude edgy content or are otherwise higher-quality'), read as keeping what looks like Wikipedia", "resolver": "a later capture at 'Common Crawl' (ADDR-68b16a5a5b51) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K9", "claim": "`O722-03`", "source": "#722", "M_src": "interpretation", "sense": "what the crawl holds and leaves out", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "qualified claim: F64, F50, F70 (bias, English, HTML only)", "resolver": "a later capture at 'Common Crawl' (ADDR-68b16a5a5b51) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K10", "claim": "`O172-01`", "source": "#172", "M_src": "attributed", "sense": "what the crawl holds and leaves out", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "none: the field has no claim on diversity over time", "resolver": "a later capture at 'Common Crawl' (ADDR-68b16a5a5b51) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K11", "claim": "`O127-01`", "source": "#127", "M_src": "interpretation", "sense": "opt-out, removal and the index as witness", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "rival claim: F43 (CCBot obeys robots.txt) — the archive's claim is of the mechanism's reach; the field claim can win on the same terms (§8.2)", "resolver": "a later capture at 'Common Crawl' (ADDR-68b16a5a5b51) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K12", "claim": "`O127-03`", "source": "#127", "M_src": "interpretation", "sense": "names and attribution", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "none", "resolver": "a later capture at 'Common Crawl' (ADDR-68b16a5a5b51) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K13", "claim": "`O623-01`", "source": "#623", "M_src": "stipulation", "sense": "in the training layer", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "none: the field has no claim on whose labor the data hold", "resolver": "a later capture at 'Common Crawl' (ADDR-68b16a5a5b51) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K14", "claim": "`O98-04`", "source": "#98", "M_src": "interpretation", "sense": "in the training layer", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "missing distinction: no field claim on consent-based contribution", "resolver": "a later capture at 'Common Crawl' (ADDR-68b16a5a5b51) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K15", "claim": "`CAP-1-08`", "source": "Capture Registry", "M_src": "interpretation", "sense": "reception and the composition at the name", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "rival claim: T1-12 and T1-13 (publishers' concerns; the offer to query the index)", "resolver": "a later capture at 'Common Crawl' (ADDR-68b16a5a5b51) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K16", "claim": "`CAP-1-06`", "source": "Capture Registry", "M_src": "interpretation", "sense": "reception and the composition at the name", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "rival claim: T1-10 ('developed by companies like OpenAI, Anthropic, Google, and Meta')", "resolver": "a later capture at 'Common Crawl' (ADDR-68b16a5a5b51) on Google AI Overview, seated in the Capture Registry (§8.3)" } ], "kernel_T": [ { "claim": "F25", "note": "the public search's misleading results (card 2), absent; T offers to query the index (CAP-1-08)" }, { "claim": "F24", "note": "The Atlantic's finding that publishers were misled on paywalls and removal (card 2), composed as publishers' concerns" }, { "claim": "F19", "note": "the AI companies' donations (card 2), absent; Anthropic composed as a trainer (CAP-1-06)" }, { "claim": "F31", "note": "the refined datasets (card 2), absent (CAP-1-06)" }, { "claim": "F47", "note": "'typically after filtering and processing by third parties' (card 3), dropped" }, { "claim": "F11", "note": "storage and compute costs (card 2), dropped from \"Stored for free\"" }, { "claim": "F43", "note": "CCBot obeys robots.txt (card 3), absent beside the offer to block it" }, { "claim": "F4", "note": "'15 years' on the home page (card 1), composed as 'nearly two decades' from the card's '19 years'" } ], "ledger": { "archive": "datasets/negative-of-the-negative/v2/ledgers/common-crawl/ledger-archive.json", "field": "datasets/negative-of-the-negative/v2/ledgers/common-crawl/field.json", "selection": "datasets/negative-of-the-negative/v2/traversal/common-crawl/", "config": "datasets/negative-of-the-negative/v2/panel/configs/common-crawl.json", "intake": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/", "audit": null }, "ontology": { "composition": "equal", "composition_note": "equal-terms case (v0.9 §0.9, §4.1, unchanged in v0.10): one claim space; the field's and the archive's claims ordered together by genealogy within each sense; the field's claims composed as received, the archive's as claims on the same object", "case": { "value": "equal", "statement": "The archive reads the field's entity as the entity. The census record describes Common Crawl from its own home and Overview pages and the Wikipedia article and keeps the resource's terms (operator, kind, unit, method, holdings) apart from its own reading (tll_reading), which asks of the same crawl what it carries into the training layer, how inclusion can be inspected, and how a work's name survives there. #722, #98 and #623 name Common Crawl as the substrate of training, describe the filtering applied to it downstream and read its data as semantic capital; #127 reads its crawler under robots.txt; #186 and #1543 supply the training layer and its loop. None of these reads Common Crawl as made by reception or contests that it is the foundation's open web crawl: they add senses (the first intake, the filter, the substrate's absences, the index as witness, the name in the page) on the field's object. ρ = extends; the equal-terms case.", "loci": [ { "dep": "TLL", "locus": "census.json rows[slug == 'common-crawl'].tll_reading", "quote": "The archive reads Common Crawl as the receiving apparatus's first intake, the plainest case of #186's persistence criterion: the place 'where the training and retrieval layers actually look', entered by being on the open web and addressed by URL." }, { "dep": "TLL", "locus": "census.json rows[slug == 'common-crawl'].tll_reading", "quote": "What is made from a page (refined datasets, trained models) is made downstream by others, as Wikipedia reports; crawl inclusion can be inspected in the URL Index, use of a captured page in a given model's training is a further claim its own pages do not make, and Wikipedia's report of The Atlantic's finding marks the limit of the index as a witness." }, { "dep": 186, "locus": "¶2, the three criteria (line 24)", "quote": "Third, persistence: the work is placed where the training and retrieval layers actually look, with the metadata that lets it be found and attributed." }, { "dep": 722, "locus": "§ the substrate (lines 49–51)", "quote": "The current training substrate is not the sum of human text. It is a filtered web crawl. … LLaMA 1's training mix: 67% Common Crawl, 15% C4, 4.5% GitHub, 4.5% Wikipedia, 4.5% books, 2.5% ArXiv, 2% StackExchange (arXiv:2302.13971, Table 1)." }, { "dep": 98, "locus": "the layer table, L3 (line 113)", "quote": "| L3: Data / Corpus | What the model knows | Common Crawl, proprietary licensing | Consent-based contribution with register annotation |" }, { "dep": 623, "locus": "§8.3 (line 258)", "quote": "Training data is semantic capital. When you train on Wikipedia, Wikidata, Common Crawl, or any text corpus, you draw on accumulated semantic labor." } ] }, "E_F": { "type": "C (public entity: a non-profit's open repository of web crawl data)", "definition": "Common Crawl, the Common Crawl Foundation, a 501(c)(3) non-profit founded in 2007 by Gil Elbaz, which crawls the web with CCBot and offers its crawl data free to anyone: monthly numbered releases of raw page data, metadata and text (WARC, WAT, WET), indexed by URL, over 300 billion pages, hosted on AWS's Open Data program; a primary source of training data for large language models, typically after filtering by third parties into datasets such as C4 and FineWeb; funded by the Elbaz family foundation and, from 2023, by AI companies; criticised by publishers, widely blocked, and found by The Atlantic (2025) to have misled publishers on paywalls and removal, its public search hiding sites still in the scrapes", "claims": [ "F1", "F10", "F35", "F2", "F12", "F77", "F45", "F79", "F4", "F78", "F47", "F31", "F19", "F30", "F43", "F26", "F24", "F25" ] }, "E_A": { "definition": "the same crawl, in the traininglayerliterature.org layer census record and the archive's deposits: the receiving apparatus's first intake, the plainest case of #186's persistence criterion, entered by being on the open web and addressed by URL; it records no author or attribution, so a work's name survives only as written into the page; inclusion can be inspected in the URL Index, training use is a further claim, and The Atlantic's finding marks the limit of the index as a witness; the training substrate built from it is a filtered web crawl (LLaMA 67% Common Crawl), filtered downstream by perplexity against Wikipedia so that the tail is discarded (the Amputation), semantic capital drawn from accumulated semantic labor, with much of what matters absent from it; its crawler stands under a robots.txt the archive holds unenforceable, which the archive both disallows and welcomes", "claims": [ "TLL-22", "TLL-23", "TLL-12", "TLL-24", "TLL-13", "TLL-32", "O186-02", "O722-01", "O722-03", "O98-01", "O98-02", "O98-03", "O623-01", "O127-01", "O127-04", "O1-01" ], "defining_papers": [ 186, 1543, 722, 98, 623, 127 ], "defining_records": [ "traininglayerliterature.org layer census record (census.json rows[slug == 'common-crawl'], organizing_question, admission_rule, areas), https://www.traininglayerliterature.org/aggregators/common-crawl", "Capture Registry entry common-crawl-aio-20261101 (OBS-03ba6c1cdb8f), https://www.alexanarch.org/captures/common-crawl-aio-20261101/" ] }, "rho": { "value": "extends", "status": "read from the census record, the capture record and the deposits that bear on the entity and its classes (#186, #1543, #722, #98, #623, #127, with #99, #171, #172, #550, #705, #1, #49, #914, #1075, #1555, #671, #554), 2026-11-01", "statement": "The archive reads Common Crawl as its cards and Wikipedia present it and adds, on the same object, the training layer's questions: the crawl as first intake under the persistence criterion (TLL-22, O186-02), inclusion against training use and the index as a bounded witness (TLL-23, TLL-32, TLL-13), the name kept only in the page (TLL-12, TLL-24), the filtered substrate and its absences (O722-01, O722-03, O98-01 to O98-04), semantic capital (O623-01), and the crawler under an unenforceable robots.txt (O127-01). No archive claim contradicts a field claim on the entity; within the archive's ledger #127's disallowing of CCBot and #1's and #705's welcome of crawlers are composed together (P_coh).", "claims": [ "TLL-22", "TLL-23", "TLL-32", "TLL-13", "TLL-12", "TLL-24", "O186-02", "O722-01", "O722-03", "O98-01", "O98-04", "O623-01", "O127-01" ], "guard": "ρ = extends claims an addition and no more: the archive's claims do not make the field's sources wrong at their dates; the foundation's statement that CCBot obeys robots.txt stands beside #127's reading of the mechanism's reach, and the census's reading of the index stays a reading (open block).", "considered": [ { "value": "constructs", "rejected": "the archive does not read Common Crawl as made by reception; its record takes the field's account of the crawl as the account" }, { "value": "revises", "rejected": "Common Crawl is not the archive's own entity; no archive text restates and narrows an earlier archive account of it" } ] }, "translation": [ { "lineage": "what it is", "field_claims": [ "F1", "F10", "F35", "F60", "F68" ], "received": "F1 (B1, opening): \"Common Crawl maintains a free, open repository of web crawl data that can be used by anyone.\" | F10 (B2, lead): \"The Common Crawl Foundation ( Common Crawl ) is a nonprofit 501(c)(3) organization that crawls the web and freely provides its archives and datasets to the public.\" | F35 (B3, Overview): \"Common Crawl is a nonprofit 501(c)(3) organisation that crawls the web and freely provides its archives and datasets to the public. Founded in 2007 by Gil Elbaz , we maintain an open repository of web crawl data collected since 2008, totalling more than 10 petabytes. Crawls are published approximately once a month, each typically containing more than two billion web pages.\" | F60 (B7, What is Common Crawl?): \"Common Crawl is a non profit organization that has been crawling the web since 2008. Its mission is to provide free, large scale, publicly available archives of web data for researchers, developers, and organizations worldwide.\" | F68 (B10, card title (video, 2m)): \"Exploring Common Crawl: The Web's Open Archive | Extract Data Live\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-01, TLL-02, TLL-04", "archive_claims": [ "TLL-01", "TLL-02", "TLL-04" ] }, { "lineage": "the foundation: form, founder, governance", "field_claims": [ "F2", "F12", "F29", "F42", "F58", "F74" ], "received": "F2 (B1, opening): \"Common Crawl is a 501(c)(3) non–profit founded in 2007.\" | F12 (B2, lead): \"Common Crawl was founded by Gil Elbaz .\" | F29 (B2, Organization): \"Peter Norvig and Joi Ito have served on the advisory board. … Rich Skrenta is the executive director.\" | F42 (B3, History 2023): \"Rich Skrenta , creator of the blekko search engine and founder of the Open Directory Project , became Executive Director.\" | F58 (B6, header; Company size): \"Discover all 23 employees … Company size 2-10 employees\" | F74 (B11, At-A-Glance): \"Executive Director: Rich Skrenta Location: Beverly Hills, CA\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-03, CAP-1-01", "archive_claims": [ "TLL-03", "CAP-1-01" ] }, { "lineage": "purpose and mission", "field_claims": [ "F3", "F27", "F56", "F57" ], "received": "F3 (B1, opening): \"We make wholesale extraction, transformation, and analysis of open web data accessible to researchers.\" | F27 (B2, History): \"A 2026 article in LWN.net discussed an advantage to services like Common Crawl being that it can limit the scraping costs to websites by allowing companies and researchers to download the data from Common Crawl instead of scraping it themselves.\" | F56 (B6, About us): \"The Common Crawl Foundation is a California 501(c)(3) registered non-profit founded by Gil Elbaz with the goal of democratizing access to web information by producing and maintaining an open repository of web crawl data that is universally accessible and analyzable.\" | F57 (B6, About us): \"We level the playing field by making wholesale extraction, transformation and analysis of web data cheap and easy.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside CAP-1-02", "archive_claims": [ "CAP-1-02" ] }, { "lineage": "size and cadence", "field_claims": [ "F4", "F8", "F36", "F61" ], "received": "F4 (B1, Overview): \"Over 300 billion pages spanning 15 years. … 3–5 billion new pages added each month.\" | F8 (B1, card snippet): \"Corpus Statistics: Over 300 billion web pages spanning 19 years\" | F36 (B3, Overview (figures)): \">10 PiB Total archive size ~2B+ Pages per crawl 12,000+ Research citations 64% LLMs trained on Common Crawl data\" | F61 (B7, What is Common Crawl?): \"For instance, the August 2025 crawl added 2.42 billion pages, totaling over 419 TiB of data!\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-10, TLL-17, TLL-27, CAP-1-03", "archive_claims": [ "TLL-10", "TLL-17", "TLL-27", "CAP-1-03" ] }, { "lineage": "dates: 2007, 2008, 2011", "field_claims": [ "F5", "F15", "F75" ], "received": "F5 (B1, Overview): \"Free and open corpus since 2007.\" | F15 (B2, History): \"Common Crawl was founded in 2007 in San Francisco, and started archiving websites that year. … It began publishing its crawls in 2011.\" | F75 (B12, Overview): \"The Common Crawl corpus contains petabytes of data, regularly collected since 2008.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-16", "archive_claims": [ "TLL-16" ] }, { "lineage": "research use and citation", "field_claims": [ "F6", "F22" ], "received": "F6 (B1, Overview): \"Cited in over 10,000 research papers.\" | F22 (B2, History): \"As of 2024, Common Crawl had been cited in more than 10,000 academic studies.\"", "receiving_text": "—", "translation": "untranslated: composed as received", "archive_claims": [] }, { "lineage": "units, formats and indexes", "field_claims": [ "F7", "F45", "F46", "F63", "F76", "F77" ], "received": "F7 (B1, Latest Blog Post): \"We are extremely pleased to announce the release of our October 2026 Web Graph, comprising data from August, September, and October 2026.\" | F45 (B3, Data Formats): \"WARC (Web ARChive) Raw crawl data including full HTTP responses, compliant with ISO 28500 . Append-only and immutable once written. WAT (Web Archive Transformation) Metadata extracted from WARC records, including HTTP headers and link graphs. WET (WARC Encapsulated Text) Plaintext content extracted from crawled HTML, suitable for NLP tasks.\" | F46 (B3, Indexes): \"Monthly release index files in the CDXJ Index and URL Index , the latter (previously called the \"Columnar Index\") queryable with Amazon Athena .\" | F63 (B7, How Common Crawl Organizes Data?): \"Web Archive Transformations (WAT files) - they like summaries of WARC files. They contain metadata in JSON format, such as all the links on the page, HTTP headers, and response codes. … Web Extracted Text (WET files) - plain text extracted from WARC files (no HTML or media). Ideal for NLP or training text based models.\" | F76 (B12, crawl list): \"CC-MAIN-2026-43\" | F77 (B12, Overview): \"The corpus contains raw web page data, metadata extracts, and text extracts.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-09, TLL-18, CAP-1-04", "archive_claims": [ "TLL-09", "TLL-18", "CAP-1-04" ] }, { "lineage": "opt-out, removal and the index as witness", "field_claims": [ "F9", "F18", "F26", "F43", "F48", "F69", "F71", "F73", "F79" ], "received": "F9 (B1, navigation): \"Get Started Blog Examples CCBot Infra Status Opt-Out Ledger FAQ\" | F18 (B2, History): \"Common Crawl received its first requests to redact information in 2023 and increasingly started seeing its crawler, CCBot, blocked.\" | F26 (B2, History): \"As of 2025, Reisner found that CCBot was the most widely-blocked bot by the top 1000 websites.\" | F43 (B3, Technical Infrastructure, CCBot): \"CCBot obeys robots.txt directives and rate-limits its requests to individual servers.\" | F48 (B3, Role in AI Development): \"A 2025 paper at COLM found that compliance with robots.txt web crawling opt-outs does not degrade general knowledge acquisition in LLMs, reporting a close-to-zero \"Data Compliance Gap.\"\" | F69 (B11, summary): \"Common Crawl is a nonprofit which maintains an open repository of “crawled” internet data. The group makes the information which it crawls from the internet free and open to researchers. Some news outlets began demanding that the group not crawl their websites because of the group’s role in training artificial intelligence (AI) programs, claiming that the group violates news outlets’ copyrights.\" | F71 (B11, Copyright Complaints): \"The group excludes pages which have a robots.txt exclusion policy and allows for publishers to opt out of having their pages crawled. The group asserts that it addresses copyright concerns by only capturing a sample of a website instead of the entire website.\" | F73 (B11, Artificial Intelligence Training): \"In June 2024, Danish media outlets demanded the group remove copies of their articles from past data sets and stop crawling their websites immediately. In addition to the Danish media outlets, the New York Times , Buzzfeed, the Washington Post , and the Canadian Broadcasting Corporation (CBC) also block Common Crawl’s access to their webpages.\" | F79 (B12, Overview): \"You can search for pages in our corpus using the Common Crawl URL Index.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-13, TLL-23, TLL-31, TLL-32, TLL-34, CAP-1-08, O127-01, O127-04, O127-05, O127-06, O705-01, O1-01, O49-01, O914-01, O1075-01", "archive_claims": [ "TLL-13", "TLL-23", "TLL-31", "TLL-32", "TLL-34", "CAP-1-08", "O127-01", "O127-04", "O127-05", "O127-06", "O705-01", "O1-01", "O49-01", "O914-01", "O1075-01" ] }, { "lineage": "hosting, access and licence", "field_claims": [ "F11", "F28", "F33", "F53", "F54", "F78" ], "received": "F11 (B2, lead): \"Access to the data is free on Amazon Web Services , but users may incur storage and compute costs.\" | F28 (B2, History): \"In April 2026, Common Crawl experimentally began to distribute its data through Hugging Face Storage Bucket , in addition to its standard storage on Amazon S3 .\" | F33 (B2, infobox): \"Apache 2.0 (software) [ clarification needed ]\" | F53 (B5, Description; License): \"A corpus of web crawl data composed of over 300 billion web pages. Update Frequency Monthly License This data is available for anyone to use under the Common Crawl Terms of Use\" | F54 (B5, Resources on AWS): \"Crawl data (WARC and ARC format) Resource type S3 Bucket Account Required Amazon Resource Name (ARN) arn:aws:s3:::commoncrawl AWS Region us-east-1\" | F78 (B12, Overview): \"Common Crawl data is stored on Amazon Web Services’ Public Data Sets and on multiple academic cloud platforms across the world. … Access to the corpus hosted by Amazon is free. You may use Amazon’s cloud platform to run analysis jobs directly against it or you can download it, whole or in part.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-19, TLL-25, TLL-26", "archive_claims": [ "TLL-19", "TLL-25", "TLL-26" ] }, { "lineage": "in the training layer", "field_claims": [ "F13", "F20", "F23", "F37", "F41", "F47", "F55", "F59", "F62", "F66", "F67" ], "received": "F13 (B2, lead): \"The data had mostly been primarily used by researchers and some startups until the 2020s, when AI companies started training large language models using the data.\" | F20 (B2, History): \"ChatGPT , as of 2023, was largely trained on Common Crawl, … as were other LLMs including Gemini .\" | F23 (B2, History): \"By 2024, The Pile and Common Crawl had been the two main training datasets being used to train AI models.\" | F37 (B3, Overview): \"The dataset is hosted on Amazon Web Services through its Open Data Sponsorship Program and can be downloaded at no cost. It has been cited in over 12,000 research papers and has become one of the most widely used sources of training data for large language models.\" | F41 (B3, History 2019, 2020): \"Google's Colossal Clean Crawled Corpus (C4) , constructed from a single Common Crawl snapshot, was used to train the T5 language model series, marking the beginning of Common Crawl's central role in AI development. … OpenAI's GPT-3 paper reported that the majority of its training tokens were derived from filtered Common Crawl data\" | F47 (B3, Role in AI Development): \"Common Crawl data, typically after filtering and processing by third parties, has been a primary training data source for many large language models. A 2024 Mozilla Foundation study described the dataset as having \"laid the infrastructural foundation\" for the generative AI boom, and found that at least 64% of 47 large language models published between 2019 and 2023 were trained on filtered versions of Common Crawl data.\" | F55 (B5, Publications): \"LAION-5B: An open large-scale dataset for training next generation image-text models … Language models are few-shot learners … LLaMA: open and efficient foundation language models\" | F59 (B6, Updates (a repost by the foundation)): \"If you're not in the crawl, you're not in the model. I'll show you how to check.\" | F62 (B7, Why Common Crawl Matters for AI?): \"A 2024 Mozilla report found that 2/3 of 47 generative LLMs released between 2019-2023 relied on Common Crawl data.\" | F66 (B8, card title (video, 14m)): \"Harnessing Common Crawl for AI and ML Applications - Pedro Ortiz Suarez ...\" | F67 (B9, card title (video, 6m)): \"Common Crawl for AI Visibility Explained (James Dooley ...\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-11, TLL-22, CAP-1-06, O186-01, O186-03, O186-04, O1543-01, O1543-02, O722-01, O98-04, O623-01, O671-01, O554-01", "archive_claims": [ "TLL-11", "TLL-22", "CAP-1-06", "O186-01", "O186-03", "O186-04", "O1543-01", "O1543-02", "O722-01", "O98-04", "O623-01", "O671-01", "O554-01" ] }, { "lineage": "The Atlantic's finding", "field_claims": [ "F14", "F24", "F25", "F52" ], "received": "F14 (B2, lead): \"In November 2025, an investigation by The Atlantic revealed that Common Crawl misled publishers when it claimed it respected paywalls in its scraping and it was not honoring requests from publishers to have their content removed from its databases.\" | F24 (B2, History): \"In November 2025, an investigation by technology journalist Alex Reisner for The Atlantic revealed that Common Crawl misled publishers when it claimed it respected paywalls in its scraping and when it said that it was honoring requests from publishers to have their content removed from its databases.\" | F25 (B2, History): \"It included misleading results in the public search function on its website that showed no entries for websites that had requested their archives be removed, when in fact those sites were still included in its scrapes used by AI companies.\" | F52 (B4, card title and snippet (unreachable)): \"Common Crawl Is Doing the AI Industry's Dirty Work - The Atlantic Editor's note: This work is part of AI Watchdog, The Atlantic's ongoing investigation into the generative-AI industry. * The Common Crawl Foundation is little k...\"", "receiving_text": "—", "translation": "untranslated: composed as received beside CAP-1-07", "archive_claims": [ "CAP-1-07" ] }, { "lineage": "neighbours", "field_claims": [ "F16", "F38" ], "received": "F16 (B2, History): \"Common Crawl was designed to have more and fresher data that was more efficient to analyze and utilize than the Wayback Machine created by the Internet Archive .\" | F38 (B3, Overview): \"Common Crawl is a member of the International Internet Preservation Consortium (IIPC) and a partner in the End of Term Web Archive\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-30, TLL-33", "archive_claims": [ "TLL-30", "TLL-33" ] }, { "lineage": "the crawler and its seeds", "field_claims": [ "F17", "F39", "F40", "F44" ], "received": "F17 (B2, History): \"By 2015, 1.8 billion webpages were on the Common Crawl, which started by crawling a list of URLs donated by the search engine Blekko .\" | F39 (B3, History 2008, 2012): \"Common Crawl began collecting data using a custom Hadoop -based crawler with a PageRank implementation. … The search engine blekko donated crawl metadata gathered between February and October 2012 to improve crawl quality and reduce spam.\" | F40 (B3, History 2013): \"Common Crawl replaced their custom crawler with a system based on Apache Nutch , designated CCBot. The organisation switched from ARC to WARC format (ISO 28500) beginning with the November 2013 crawl.\" | F44 (B3, Technical Infrastructure, CCBot): \"CCBot uses Harmonic Centrality , a graph-theoretic measure of a node's proximity to the structural core of the web, to prioritise URLs for crawling.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-15", "archive_claims": [ "TLL-15" ] }, { "lineage": "funding", "field_claims": [ "F19", "F30", "F51" ], "received": "F19 (B2, History): \"In 2023, it began receiving significant financial support from AI companies, including Anthropic and OpenAI , each of which donated $250,000.\" | F30 (B2, Organization): \"It has received funding almost exclusively from the Elbaz Family Foundation Trust until 2023 when it started receiving donations from the AI industry.\" | F51 (B3, Organization): \"Gil Elbaz serves as Chairman of the Board. Primary funding comes from the Elbaz Family Foundation. As of 2026, the organization has approximately 18 staff members .\"", "receiving_text": "—", "translation": "untranslated: composed as received", "archive_claims": [] }, { "lineage": "what the crawl holds and leaves out", "field_claims": [ "F21", "F50", "F64", "F65", "F70", "F72" ], "received": "F21 (B2, History): \"By April 2023, Common Crawl was capturing 3.1 billion webpages, with an estimated 5% of pages before 2021 containing hate speech or slurs.\" | F50 (B3, Multilingual Initiatives): \"In December 2024, we launched the Web Languages Project to address the overrepresentation of English-language content\" | F64 (B7, Other Challenges with Common Crawl): \"Common Crawl captures the same pages across multiple crawls, sometimes hundreds of times. … Crawl frequency and seed URLs shape what gets captured. … So, some domains or regions are overrepresented while others barely appear.\" | F65 (B7, Wrapping Up): \"Its data is archived, not live … That’s where modern scraping solutions like Zyte’s API make all the difference.\" | F70 (B11, Background): \"The group focuses on downloading HTML pages and does not archive images, videos, or Javascript files.\" | F72 (B11, Artificial Intelligence Training): \"The study also called on Common Crawl to be more transparent and inclusive about its governance and to better highlight the limitation and biases of its data.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside O186-05, O722-03, O99-01, O171-01, O172-01, O550-01, O1555-01", "archive_claims": [ "O186-05", "O722-03", "O99-01", "O171-01", "O172-01", "O550-01", "O1555-01" ] }, { "lineage": "refined datasets downstream", "field_claims": [ "F31", "F32", "F49" ], "received": "F31 (B2, Refined versions): \"A number of organizations take raw Common Crawl data and refine it into datasets that exclude edgy content or are otherwise higher-quality for their purposes, such as FineWeb, DCLM and C4.\" | F32 (B2, Colossal Clean Crawled Corpus): \"The Colossal Clean Crawled Corpus , or C4 for short was created by Google . It was constructed for the training of the T5 language model series in 2019. … A 2024 study found that 45% of the content was explicitly restricted by websites' terms of service to be used for purposes like AI training by for-profit companies.\" | F49 (B3, Role in AI Development; Derived Datasets): \"Pythia EleutherAI Used Pile-CC, a filtered Common Crawl subset within The Pile dataset. … FineWeb Hugging Face 2024 Refined filtering pipeline for LLM pre-training\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-14, TLL-29, O722-02, O98-01, O98-02, O98-03", "archive_claims": [ "TLL-14", "TLL-29", "O722-02", "O98-01", "O98-02", "O98-03" ] }, { "lineage": "reception", "field_claims": [ "F34" ], "received": "F34 (B2, footer): \"This page was last edited on 24 September 2026\"", "receiving_text": "—", "translation": "untranslated: composed as received beside CAP-1-09, CAP-1-10, CAP-1-11, CAP-1-12, CAP-1-13, CAP-1-14", "archive_claims": [ "CAP-1-09", "CAP-1-10", "CAP-1-11", "CAP-1-12", "CAP-1-13", "CAP-1-14" ] } ], "receiving_texts": "equal-terms case: every field claim composed as received, beside the archive claims it stands with; no receiving text" }, "measure": { "rule": "v0.9 §4.10 (unchanged in v0.10): KO_B sentences surviving in the KO at similarity ≥ 0.95, with positions (computed as apply_v09.py does: difflib.SequenceMatcher ratio, the best KO sentence for each KO_B sentence)", "v0.10": { "surviving": 0, "of": 14, "positions": [ null, null, null, null, null, null, null, null, null, null, null, null, null, null ] }, "v0.9": { "surviving": 0, "of": 14, "positions": [ null, null, null, null, null, null, null, null, null, null, null, null, null, null ] }, "opening_run": { "rule": "§5.10 (equal-terms case): the field-only KO's sentences as the KO's opening run in their own order", "breach": false }, "blocks": { "rule": "§4.1 (equal-terms case): the KO does not open with the field as a block or close with the archive as one; sentences typed by the ledgers of the claims they cite (F field-only, A archive-only, M both)", "sequence": "MFAMFFMFMMMFFMFMMMFFMMFAAAMFAAFFFAAAFFFFMAAAAAMAAFAA", "field_only_head": 0, "archive_only_tail": 2, "breach": false }, "max_similarity": [ 0.343, 0.457, 0.408, 0.602, 0.227, 0.503, 0.312, 0.119, 0.149, 0.46, 0.351, 0.073, 0.289, 0.333 ], "ko_sentence_1_max_similarity_to_KO_B": 0.332 }, "homonyms": { "names": [], "basis": "No card is a homonym, and T composes the foundation itself (reached kind 'same'). 'Common Crawl Foundation', 'CC' and 'CCBot (its crawler)' are the census's other names for the entity. The distinct entities below are named by the field, the census or the archive beside the entity and are never senses of its name.", "distinct_entities": [ "the Internet Archive and its Wayback Machine (F16; TLL-30), a neighbour", "The Pile, a training dataset with its Pile-CC subset (F23, F49; TLL-33), a row of its own in the census", "C4, FineWeb, RefinedWeb, CCNet, OSCAR, DCLM, MADLAD-400 (F31, F49), datasets made from the crawl by others", "Amazon Web Services and its Open Data Sponsorship Program, the host (F37, F54)", "The Atlantic and Alex Reisner (F24, F52), the investigation", "the Elbaz Family Foundation (F30, F51), the funder", "Blekko, the search engine (F17, F39)", "the IIPC and the End of Term Web Archive (F38)", "Zyte, whose API the DEV post recommends (F65)" ] }, "notes": [ "One address, Google AI Overview (expanded), signed out, incognito, 2026-11-01: common-crawl-aio-20261101 (OBS-03ba6c1cdb8f, ADDR-68b16a5a5b51, AXN:0BAC.CAPTURE.⏰♆🎨♠️💜🟤), eleven cards (three video cards), 211 words in the answer body with a closing offer of 26, 20 markers. A NEW address (CAP-1-13); the census record was compiled the same day and no card cites it.", "Fetching: every reachable card saved verbatim with curl and the census agent 'Mozilla/5.0 (compatible; traininglayerliterature-census)' on 2026-11-01 between 15:12 and 15:14 UTC, raw and text with sha256 in field.json. The Atlantic (card 4) refused with HTTP 403 (a Cloudflare block page), one attempt, no retry by another agent string or route; its title and snippet stand. The three video cards have no URL found on search and no snippet; their titles stand. The home page fetched is byte-identical to the census's copy.", "Pools (v0.10 §0.3, §3.1(a)): B_D = B1–B11 (74 claims), B_R = B12, the Overview page the census saved (5 claims). The census's home page and Wikipedia article are cards, carried once in B_D. T against L_F(B_D) uses B1–B11; B12 enters L_F(B) and L(B ∪ A) only.", "Census fields a fetched source contradicts or shows out of date (reported, not fixed): (1) record.makes 'not recorded on its own pages' and negative_tags[1] 'NOT a training dataset itself in its own words': the About page (card 3) states its 'Role in AI Development', calls the data 'a primary training data source for many large language models' and 'one of the most widely used sources of training data', and lists derived datasets (F37, F41, F47, F49); (2) relations[the-pile] 'the saved sources do not state what the Pile's Pile-CC component is drawn from': the About page gives 'Pile-CC, a filtered Common Crawl subset within The Pile dataset' (F49); (3) license 'not stated in the saved own pages': AWS's registry (card 5) gives 'available for anyone to use under the Common Crawl Terms of Use' (F53); (4) stated_limits 'the saved own pages state no limit on what is crawled': the About page states that CCBot obeys robots.txt and rate-limits (F43), and the Web Languages Project addresses English overrepresentation (F50); (5) holdings 'cited in over 10,000 research papers' (home page): the About page gives over 12,000 (F36, F37); (6) operator 'Wikipedia names ... Rich Skrenta as executive director': the About page dates his appointment to 2023 and the foundation's own pages now name him (F42); Wikipedia's infobox calls him 'Managing director'; (7) unchecked lists the About page as named in the navigation only: it is card 3 and is read here; (8) record.readback carries The Atlantic's finding by Wikipedia, whose lead ('it was not honoring requests') and History ('when it said that it was honoring requests') state the removal finding in two forms (F14, F24); (9) method 'the crawl began from a list of URLs donated by the search engine Blekko' (Wikipedia) stands beside the About page's 2012 metadata donation by blekko (F39).", "Capture findings the field ledger shows wrong or needing qualification (reported, not fixed): (1) findings[3] reads the training intake as given by company with the donations absent; the field ledger adds that Anthropic, one of T's four companies, appears in the saved field only as a donor (F19), so T assigns it a training use no saved source states (the unread Atlantic article aside). (2) findings[0] 'The mission is LinkedIn's wording' holds; the About page's 'Founded in 2007 by Gil Elbaz' (F35, card 3) is a second own-page source for the founder beside Wikipedia. (3) findings[4] holds; the field also carries the foundation's own counter-statement, 'CCBot obeys robots.txt directives' (F43, card 3), which the composition does not carry either. (4) findings[1] attributes 'nearly two decades' to the card and the founding; the About page's '>10 PiB' and 'more than 10 petabytes' (F35, F36) are the field's source for T's petabyte figure, which the finding says no page gives ('The Overview page gives no figure'): true of the Overview page, but card 3 gives one.", "Archive-internal tension, carried and not adjudicated (report to Lee): #127 (SPXI-TLP v2.2, Layer 7) says every controlled domain SHOULD disallow CCBot; #1 (2026-06-19) records a robots.txt explicitly welcoming AI crawlers to the archive's data directories; #705 (SPXI-Sitemap) allows CCBot the SPXI files. #722's 'The top domain in the latest Common Crawl is blogspot.com at 0.9%' bears on the archive's own blog without saying so.", "The composer is a Claude model (Anthropic); Anthropic is named in the field as a donor (F19) and in T as a company training on the data (T1-10). Recorded in composition.composer_note.", "Homonyms: none on the rail; T composes the foundation by name (per_dim reached 'same').", "Not run: scripts/build_non.py (only --check), scripts/non_axn.py; panel.json not edited; the session seats the panel entry, mints the AXN (0BAC) and builds the pages." ], "composition": { "source": "this row (composed directly under the brief rebuild/negative-of-the-negative/v010/BRIEF-NEW-ROW-LAYER.md)", "composer": "Claude (Opus 5.5), subagent, 2026-11-01", "spec": "EA-NEGONT-02 v0.10 (#1726)", "defining_material_read": [ { "what": "the specification v0.10", "file": "data/texts/AXN-0B79-text.md", "read": "§0–§11 whole (the procedure); Appendix A opening" }, { "what": "the briefs", "file": "rebuild/negative-of-the-negative/v010/BRIEF-NEW-ROW-LAYER.md, BRIEF-NEW-ROW-AUTHORSHIP.md; v09/BRIEF-NEW-ROW-AGGREGATOR.md, BRIEF-NEW-ROW-LABOR.md, BRIEF-NEW-ROW-LEFT.md", "read": "whole" }, { "what": "traininglayerliterature.org layer census record, row", "file": "/home/claude/fleet/traininglayerliterature-org/data/layer-census/census.json (rows[slug == 'common-crawl'], organizing_question, admission_rule, anchor, areas, schema_note)", "read": "whole" }, { "what": "census sources", "file": "/home/claude/fleet/traininglayerliterature-org/data/layer-census/sources/common-crawl--home.*, --overview.*, --wikipedia.*, manifest.json", "read": "whole" }, { "what": "the capture", "file": "data/EA-WG-CAPTURES-01.json entries[slug == common-crawl-aio-20261101]", "read": "whole (transcript read for T and the delta only)" }, { "what": "field texts", "file": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B1–B12", "read": "whole (B6 at its About section and recent posts; B7 at its prose, the code blocks skimmed)" }, { "what": "#186 Who Is Writing for the Machines?", "file": "data/texts/AXN-032C-text.md", "read": "whole" }, { "what": "#1543 Generative Uptake", "file": "data/texts/AXN-0642-text.md", "read": "§1 and the formal statement; the rest at the training passages" }, { "what": "#722, #98, #99, #1258, #127, #171, #172, #550, #623, #705, #1, #49, #914, #1075, #1555, #671, #554", "file": "data/texts (by AXN)", "read": "at the candidate loci and their sections" }, { "what": "the exemplar rows", "file": "datasets/negative-of-the-negative/v2/rows/viaf.json, the-next.json (and the viaf builder in the session scratchpad)", "read": "structure field by field" } ], "cuts": [], "composer_note": "The composer is a Claude model, made by Anthropic; the field names Anthropic as a donor to the foundation (F19, Wikipedia) and T names Anthropic among companies whose models train on the data (T1-10), a claim no saved field text makes. Both are recorded as they stand; the delta records the second as composed beyond the saved field.", "notes": "equal case; KO 52 sentences, 79 of 79 field and 84 of 84 archive claims cited; P body 335 words" }, "composition_v09": { "source": "this row (composed directly under the brief rebuild/negative-of-the-negative/v010/BRIEF-NEW-ROW-LAYER.md)", "composer": "Claude (Opus 5.5), subagent, 2026-11-01", "spec": "EA-NEGONT-02 v0.10 (#1726)", "defining_material_read": [ { "what": "the specification v0.10", "file": "data/texts/AXN-0B79-text.md", "read": "§0–§11 whole (the procedure); Appendix A opening" }, { "what": "the briefs", "file": "rebuild/negative-of-the-negative/v010/BRIEF-NEW-ROW-LAYER.md, BRIEF-NEW-ROW-AUTHORSHIP.md; v09/BRIEF-NEW-ROW-AGGREGATOR.md, BRIEF-NEW-ROW-LABOR.md, BRIEF-NEW-ROW-LEFT.md", "read": "whole" }, { "what": "traininglayerliterature.org layer census record, row", "file": "/home/claude/fleet/traininglayerliterature-org/data/layer-census/census.json (rows[slug == 'common-crawl'], organizing_question, admission_rule, anchor, areas, schema_note)", "read": "whole" }, { "what": "census sources", "file": "/home/claude/fleet/traininglayerliterature-org/data/layer-census/sources/common-crawl--home.*, --overview.*, --wikipedia.*, manifest.json", "read": "whole" }, { "what": "the capture", "file": "data/EA-WG-CAPTURES-01.json entries[slug == common-crawl-aio-20261101]", "read": "whole (transcript read for T and the delta only)" }, { "what": "field texts", "file": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B1–B12", "read": "whole (B6 at its About section and recent posts; B7 at its prose, the code blocks skimmed)" }, { "what": "#186 Who Is Writing for the Machines?", "file": "data/texts/AXN-032C-text.md", "read": "whole" }, { "what": "#1543 Generative Uptake", "file": "data/texts/AXN-0642-text.md", "read": "§1 and the formal statement; the rest at the training passages" }, { "what": "#722, #98, #99, #1258, #127, #171, #172, #550, #623, #705, #1, #49, #914, #1075, #1555, #671, #554", "file": "data/texts (by AXN)", "read": "at the candidate loci and their sections" }, { "what": "the exemplar rows", "file": "datasets/negative-of-the-negative/v2/rows/viaf.json, the-next.json (and the viaf builder in the session scratchpad)", "read": "structure field by field" } ], "cuts": [], "composer_note": "The composer is a Claude model, made by Anthropic; the field names Anthropic as a donor to the foundation (F19, Wikipedia) and T names Anthropic among companies whose models train on the data (T1-10), a claim no saved field text makes. Both are recorded as they stand; the delta records the second as composed beyond the saved field.", "notes": "equal case; KO 52 sentences, 79 of 79 field and 84 of 84 archive claims cited; P body 335 words" }, "field_by_origin": { "cards": [ "B1", "B2", "B3", "B4", "B5", "B6", "B7", "B8", "B9", "B10", "B11" ], "own_pages": [ "B12" ], "own_pages_note": "the census's two own pages: common-crawl--home is card 1 (one page, carried once, as a card, in B_D: B1); common-crawl--overview is B12, in B_R. Card 3 (About) is also an own page of the foundation, surfaced by T and carried as a card.", "wikipedia": [ "B2" ], "census_third_party": [], "census_third_party_note": "the census's one third-party source is the Wikipedia article, card 2 (B2)", "unreachable": [ "B4", "B8", "B9", "B10" ], "title_only": [ "B8", "B9", "B10" ], "snippet_only": [ "B4" ], "wikipedia_state": "article: 'Common Crawl' (raw wikitext HTTP 200, rendered HTTP 200, 2026-11-01T15:13Z), no redirect, last edited 24 September 2026 (oldid 1376544662); it is card 2 of T, so the Wikipedia source and the card are one source, carried once in B_D (B2)", "pools": { "B_D": [ "B1", "B2", "B3", "B4", "B5", "B6", "B7", "B8", "B9", "B10", "B11" ], "B_R": [ "B12" ], "note": "v0.10 §0.3, §3.1(a): B_D, every card on the rail (B1–B11; the body names no source beyond its cards, its inline 's3://commoncrawl' being card 5's bucket). B_R, the reference pool: the Overview page the census saved (B12). The home page and the Wikipedia article are cards, carried once in B_D. T against L_F(B_D) uses B1–B11 alone." }, "sources": [ { "id": "B1", "origin": "card", "also_origin": "own_page", "pool": "B_D", "role": "Overview card 1 (and the census own page common-crawl--home: one page, carried once, in B_D)", "card": "Common Crawl — Common Crawl - Open Repository of Web Crawl Data", "url": "https://commoncrawl.org/", "kind": "Overview card (the foundation's home page); the census's own page common-crawl--home", "files": { "B1.html": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B1.html", "sha256": "34837e49010690a94a8a5aa448850e50e303affddf504b30f075ad5e7eddfee8" }, "B1.txt": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B1.txt", "sha256": "f487197afd7e08194af93df85cc8afc490cc962710410f43e1f38221340574ee" }, "census-home.html": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/census-home.html", "sha256": "34837e49010690a94a8a5aa448850e50e303affddf504b30f075ad5e7eddfee8" }, "census-home.txt": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/census-home.txt", "sha256": "37d7b19a22ba3059aa82d7c52f46802cda4e0160071e835211787945cbad727c" } }, "fetched": { "at": "2026-11-01T15:12:59Z", "http": 200, "url": "https://commoncrawl.org/", "agent": "Mozilla/5.0 (compatible; traininglayerliterature-census)", "raw": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B1.html", "raw_sha256": "34837e49010690a94a8a5aa448850e50e303affddf504b30f075ad5e7eddfee8", "text": [ "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B1.txt" ], "census_copy": { "src": "common-crawl--home", "http": "200", "fetched": "2026-11-01T04:07:46Z", "sha256": "34837e49010690a94a8a5aa448850e50e303affddf504b30f075ad5e7eddfee8", "agrees": "identical bytes (sha256 equal)" } }, "names_operator": "yes ('Common Crawl is a 501(c)(3) non–profit founded in 2007.')", "note": "URL from the card's site and title (the page's is 'Common Crawl - Open Repository of Web Crawl Data', verbatim). The page fetched at 15:12:59Z is byte-identical to the census's copy of 04:07:46Z. The card's snippet reads 'spanning 19 years' where the page reads 'spanning 15 years' (F8 against F4).", "surfaced_by_T": [ { "slug": "common-crawl-aio-20261101", "card": 1 } ], "card_title": "Common Crawl - Open Repository of Web Crawl Data", "card_snippet": "Organization: Common Crawl (501(c)(3) non-profit founded 2007). Corpus Statistics: Over 300 billion web pages spanning 19 years; adds 3 to 5 billion new pages m...", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, EA-WG-CAPTURES-01 in the saved text, card title and snippet)" }, { "id": "B2", "origin": "card", "also_origin": "wikipedia", "pool": "B_D", "role": "Overview card 2 (and the entity's Wikipedia article: one source, carried once, in B_D)", "card": "Wikipedia — Common Crawl - Wikipedia", "url": "https://en.wikipedia.org/wiki/Common_Crawl", "kind": "Overview card (the Wikipedia article); the entity's Wikipedia article", "files": { "B2.html": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B2.html", "sha256": "09c3c6c4c8214ad61301ef7c603a2aa47d57685655d3602fb269a08edf231990" }, "B2.txt": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B2.txt", "sha256": "adbbd664fe456b0003bdc305c873fdda139244f1ee013ceca6e1efcb0fcf2bf5" }, "B2.wiki": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B2.wiki", "sha256": "cc209d3bd3156d8e7e32c529cbbc28fd9073c4eba3113499fc29dacacdab2a38" }, "census-wikipedia.html": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/census-wikipedia.html", "sha256": "6a52d8dd429a5afe130c0a9f753781a82482d7ed7ec29768318003e9ad8f94fc" }, "census-wikipedia.txt": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/census-wikipedia.txt", "sha256": "becf9d60ac2e37e7307d8627b57d2fdb623b7e6427a8fe0436ff9d86283e5b65" } }, "fetched": { "at": "2026-11-01T15:12:59Z", "http": 200, "url": "https://en.wikipedia.org/wiki/Common_Crawl", "agent": "Mozilla/5.0 (compatible; traininglayerliterature-census)", "raw": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B2.html", "raw_sha256": "09c3c6c4c8214ad61301ef7c603a2aa47d57685655d3602fb269a08edf231990", "text": [ "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B2.txt", "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B2.wiki" ], "wikitext": { "url": "https://en.wikipedia.org/w/index.php?title=Common_Crawl&action=raw", "http": 200, "at": "2026-11-01T15:13:00Z", "sha256": "cc209d3bd3156d8e7e32c529cbbc28fd9073c4eba3113499fc29dacacdab2a38" }, "census_copy": { "src": "common-crawl--wikipedia", "http": "200", "fetched": "2026-11-01T04:07:47Z", "sha256": "6a52d8dd429a5afe130c0a9f753781a82482d7ed7ec29768318003e9ad8f94fc", "agrees": "one byte shorter, different hash (the rendered page carries per-request tokens); the article text the same, last edited 24 September 2026 (oldid 1376544662)" } }, "names_operator": "yes ('Common Crawl was founded by Gil Elbaz .'; 'Rich Skrenta is the executive director.')", "note": "Article title Common_Crawl, no redirect; rendered page and raw wikitext both HTTP 200; last edited 24 September 2026 (oldid 1376544662). The census row's third-party evidence rests on this article.", "surfaced_by_T": [ { "slug": "common-crawl-aio-20261101", "card": 2 } ], "card_title": "Common Crawl - Wikipedia", "card_snippet": "Organization: Common Crawl Foundation (501(c)(3) non-profit, founded 2007 by Gil Elbaz, San Francisco/Los Angeles). Operations: Crawls the web, provides free pu...", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, EA-WG-CAPTURES-01 in the saved text, card title and snippet)" }, { "id": "B3", "origin": "card", "pool": "B_D", "role": "Overview card 3 (the foundation's About page; in the census's unchecked list, named in the navigation only)", "card": "Common Crawl — About", "url": "https://commoncrawl.org/about", "kind": "Overview card (the foundation's About page: overview figures, history, technical infrastructure, role in AI development, derived datasets, organization)", "files": { "B3.html": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B3.html", "sha256": "d6c6231094e703caaac1786f596c5560fb88329b78a83f198e8b49114f6722ec" }, "B3.txt": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B3.txt", "sha256": "3b70890d6be4c70268cc1b97ae15c15f0b6131a3d58f1bd8c9d63fdef9302f65" } }, "fetched": { "at": "2026-11-01T15:13:00Z", "http": 200, "url": "https://commoncrawl.org/about", "agent": "Mozilla/5.0 (compatible; traininglayerliterature-census)", "raw": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B3.html", "raw_sha256": "d6c6231094e703caaac1786f596c5560fb88329b78a83f198e8b49114f6722ec", "text": [ "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B3.txt" ] }, "names_operator": "yes ('Founded in 2007 by Gil Elbaz'; 'Gil Elbaz serves as Chairman of the Board.')", "note": "URL from the card's site and title (the page's heading 'About'); the snippet's opening ('Overview Common Crawl is a nonprofit 501(c)(3) organisation that crawls the web ...') is verbatim on the page. The census lists the About page as unchecked ('named in the navigation only').", "surfaced_by_T": [ { "slug": "common-crawl-aio-20261101", "card": 3 } ], "card_title": "About", "card_snippet": "Overview Common Crawl is a nonprofit 501(c)(3) organisation that crawls the web and freely provides its archives and datasets to the public. Founded in 2007 by ...", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, EA-WG-CAPTURES-01 in the saved text, card title and snippet)" }, { "id": "B4", "origin": "card", "pool": "B_D", "role": "Overview card 4 (The Atlantic)", "card": "The Atlantic — Common Crawl Is Doing the AI Industry's Dirty Work - The Atlantic", "url": "https://www.theatlantic.com/technology/2025/11/common-crawl-ai-training-data/684567/", "kind": "Overview card (Alex Reisner's article for The Atlantic, November 2025, in its AI Watchdog series)", "files": { "B4.html": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B4.html", "sha256": "1e44c08587ed087f8e8faced29db8c2fb0d99ad596d1ee10c679fc639e77524d" } }, "fetched": { "at": "2026-11-01T15:13:00Z", "http": 403, "url": "https://www.theatlantic.com/technology/2025/11/common-crawl-ai-training-data/684567/", "agent": "Mozilla/5.0 (compatible; traininglayerliterature-census)", "raw": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B4.html", "raw_sha256": "1e44c08587ed087f8e8faced29db8c2fb0d99ad596d1ee10c679fc639e77524d", "result": "unreachable: HTTP 403, The Atlantic's Cloudflare block page ('Your access has been blocked'); one attempt with the census agent string, no retry by another agent string or route", "copy": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/tries/B4-atlantic-403.html" }, "names_operator": "card snippet: no", "note": "URL found by WebSearch on the card's title (a listing of the article at scour.ing gives https://www.theatlantic.com/technology/2025/11/common-crawl-ai-training-data/684567; Wikipedia's reference 4 names the article 'The Company Quietly Funneling Paywalled Articles to AI Developers', Reisner, November 4, 2025). Refused; the card's title and snippet are its only text (§3.1(a)). The census also recorded The Atlantic as refused (books3-search--atlantic, 403).", "searches": [ "WebSearch '\"Common Crawl Is Doing the AI Industry's Dirty Work\" The Atlantic'" ], "surfaced_by_T": [ { "slug": "common-crawl-aio-20261101", "card": 4 } ], "card_title": "Common Crawl Is Doing the AI Industry's Dirty Work - The Atlantic", "card_snippet": "Editor's note: This work is part of AI Watchdog, The Atlantic's ongoing investigation into the generative-AI industry. * The Common Crawl Foundation is little k...", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, EA-WG-CAPTURES-01 in the saved text, card title and snippet)" }, { "id": "B5", "origin": "card", "pool": "B_D", "role": "Overview card 5", "card": "Registry of Open Data on AWS — Common Crawl", "url": "https://registry.opendata.aws/commoncrawl/", "kind": "Overview card (AWS's Registry of Open Data entry for the dataset)", "files": { "B5.html": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B5.html", "sha256": "b8a041cf8f978d9575aed502340b7fbdf3da8b2c2a47272ead95d05876441470" }, "B5.txt": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B5.txt", "sha256": "a48b5ffead0a87698636a89f2bf667643bfaf313c13affd23ad34fbaf2c373c4" } }, "fetched": { "at": "2026-11-01T15:13:01Z", "http": 200, "url": "https://registry.opendata.aws/commoncrawl/", "agent": "Mozilla/5.0 (compatible; traininglayerliterature-census)", "raw": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B5.html", "raw_sha256": "b8a041cf8f978d9575aed502340b7fbdf3da8b2c2a47272ead95d05876441470", "text": [ "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B5.txt" ] }, "names_operator": "yes ('See all datasets managed by Common Crawl .')", "note": "URL from the card's site and the S3 name in its snippet (arn:aws:s3:::commoncrawl); the page's <title> 'Common Crawl - Registry of Open Data on AWS'; the snippet verbatim on the page.", "surfaced_by_T": [ { "slug": "common-crawl-aio-20261101", "card": 5 } ], "card_title": "Common Crawl", "card_snippet": "Resources on AWS Description Crawl data (WARC and ARC format) Resource type S3 Bucket Account Required Amazon Resource Name (ARN) arn:aws:s3:::commoncrawl AWS R...", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, EA-WG-CAPTURES-01 in the saved text, card title and snippet)" }, { "id": "B6", "origin": "card", "pool": "B_D", "role": "Overview card 6", "card": "LinkedIn — Common Crawl Foundation | LinkedIn", "url": "https://www.linkedin.com/company/common-crawl", "kind": "Overview card (the foundation's LinkedIn company page, public view)", "files": { "B6.html": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B6.html", "sha256": "a9974e4d81573b80b65b6fd2735114e09b0ccf43f7a65d317ec1a4394e5ae516" }, "B6.txt": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B6.txt", "sha256": "5480b9e2f92a537c9eb9a79d8b29463dc9345fe03814c75cfe734b849c1b477a" } }, "fetched": { "at": "2026-11-01T15:13:02Z", "http": 200, "url": "https://www.linkedin.com/company/common-crawl", "agent": "Mozilla/5.0 (compatible; traininglayerliterature-census)", "raw": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B6.html", "raw_sha256": "a9974e4d81573b80b65b6fd2735114e09b0ccf43f7a65d317ec1a4394e5ae516", "text": [ "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B6.txt" ] }, "names_operator": "yes ('founded by Gil Elbaz'; employees listed beginning 'Rich Skrenta')", "note": "URL from the card's title; the page's <title> 'Common Crawl Foundation | LinkedIn'; the public view returned HTTP 200 with the About text and recent reposts; the snippet verbatim on the page.", "surfaced_by_T": [ { "slug": "common-crawl-aio-20261101", "card": 6 } ], "card_title": "Common Crawl Foundation | LinkedIn", "card_snippet": "The Common Crawl Foundation is a California 501(c)(3) registered non-profit founded by Gil Elbaz with the goal of democratizing access to web information by pro...", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, EA-WG-CAPTURES-01 in the saved text, card title and snippet)" }, { "id": "B7", "origin": "card", "pool": "B_D", "role": "Overview card 7", "card": "DEV Community — Inside Common Crawl: The Dataset Behind AI Models (and Its Real ...", "url": "https://dev.to/extractdata/inside-common-crawl-the-dataset-behind-ai-models-and-its-real-world-limits-2eo2", "kind": "Overview card (a post on DEV Community by the Extract Data community account, 'Inside Common Crawl: The Dataset Behind AI Models (and Its Real World Limits)')", "files": { "B7.html": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B7.html", "sha256": "8d44ccbe41e0b9d0bc61326865cfbde820b058eae90e1b5ca6e8acba2714691b" }, "B7.txt": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B7.txt", "sha256": "3040c3467b8e907a702336ede3091c8361fdc2574175463e5ac8bf48a85bd7c2" } }, "fetched": { "at": "2026-11-01T15:13:22Z", "http": 200, "url": "https://dev.to/extractdata/inside-common-crawl-the-dataset-behind-ai-models-and-its-real-world-limits-2eo2", "agent": "Mozilla/5.0 (compatible; traininglayerliterature-census)", "raw": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B7.html", "raw_sha256": "8d44ccbe41e0b9d0bc61326865cfbde820b058eae90e1b5ca6e8acba2714691b", "text": [ "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B7.txt" ] }, "names_operator": "no (no person at the foundation named)", "note": "URL found by WebSearch on the card's title; the snippet verbatim on the page.", "searches": [ "WebSearch '\"Inside Common Crawl: The Dataset Behind AI Models\" dev.to'" ], "surfaced_by_T": [ { "slug": "common-crawl-aio-20261101", "card": 7 } ], "card_title": "Inside Common Crawl: The Dataset Behind AI Models (and Its Real ...", "card_snippet": "What is Common Crawl? Common Crawl is a non profit organization that has been crawling the web since 2008. Its mission is to provide free, large scale, publicly...", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, EA-WG-CAPTURES-01 in the saved text, card title and snippet)" }, { "id": "B8", "origin": "card", "pool": "B_D", "role": "Overview card 8 (a video card, 14m)", "card": "YouTube·PyTorch — Harnessing Common Crawl for AI and ML Applications - Pedro Ortiz Suarez ...", "url": null, "kind": "video card (title only; no snippet on the rail)", "files": {}, "fetched": { "at": null, "http": null, "url": null, "result": "unreachable: no URL found for the video; no transcript fetched; the card has no snippet, its title its only text (§3.1(a), video cards)" }, "names_operator": "title only", "searches": [ "WebSearch '\"Harnessing Common Crawl for AI and ML Applications\" Pedro Ortiz Suarez youtube' (found the PyTorch Day France 2025 schedule entry for the talk, no video URL)" ], "note": "A video card is admitted through its transcript where one can be fetched, otherwise through title and snippet; this card renders a title and duration only.", "surfaced_by_T": [ { "slug": "common-crawl-aio-20261101", "card": 8 } ], "card_title": "Harnessing Common Crawl for AI and ML Applications - Pedro Ortiz Suarez ...", "card_snippet": null, "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, EA-WG-CAPTURES-01 in the saved text, card title and snippet)" }, { "id": "B9", "origin": "card", "pool": "B_D", "role": "Overview card 9 (a video card, 6m)", "card": "YouTube·James Dooley — Common Crawl for AI Visibility Explained (James Dooley ...", "url": null, "kind": "video card (title only; no snippet on the rail)", "files": {}, "fetched": { "at": null, "http": null, "url": null, "result": "unreachable: no URL found for the video; no transcript fetched; the card has no snippet, its title its only text (§3.1(a), video cards)" }, "names_operator": "title only", "searches": [ "WebSearch '\"Common Crawl for AI Visibility Explained\" James Dooley youtube' (no video URL)" ], "note": "A video card is admitted through its transcript where one can be fetched, otherwise through title and snippet; this card renders a title and duration only.", "surfaced_by_T": [ { "slug": "common-crawl-aio-20261101", "card": 9 } ], "card_title": "Common Crawl for AI Visibility Explained (James Dooley ...", "card_snippet": null, "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, EA-WG-CAPTURES-01 in the saved text, card title and snippet)" }, { "id": "B10", "origin": "card", "pool": "B_D", "role": "Overview card 10 (a video card, 2m)", "card": "YouTube·Extract Summit — Exploring Common Crawl: The Web's Open Archive | Extract Data Live", "url": null, "kind": "video card (title only; no snippet on the rail)", "files": {}, "fetched": { "at": null, "http": null, "url": null, "result": "unreachable: no URL found for the video; no transcript fetched; the card has no snippet, its title its only text (§3.1(a), video cards)" }, "names_operator": "title only", "searches": [ "WebSearch '\"Exploring Common Crawl: The Web's Open Archive\" Extract Data Live youtube' (no video URL)" ], "note": "A video card is admitted through its transcript where one can be fetched, otherwise through title and snippet; this card renders a title and duration only.", "surfaced_by_T": [ { "slug": "common-crawl-aio-20261101", "card": 10 } ], "card_title": "Exploring Common Crawl: The Web's Open Archive | Extract Data Live", "card_snippet": null, "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, EA-WG-CAPTURES-01 in the saved text, card title and snippet)" }, { "id": "B11", "origin": "card", "pool": "B_D", "role": "Overview card 11", "card": "InfluenceWatch — Common Crawl - Influence Watch", "url": "https://www.influencewatch.org/non-profit/common-crawl/", "kind": "Overview card (InfluenceWatch's profile of the non-profit: background, copyright complaints, AI training, leadership, financials from tax filings)", "files": { "B11.html": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B11.html", "sha256": "2a7578e39c9f91e63d4e9ec9d49dbde17b6edce8ad7cd8d202010c85978c93ea" }, "B11.txt": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B11.txt", "sha256": "77ada196dd97fe2b1acbac845be9e3cf3c48b9c3776535db283bb11b7b0ce980" } }, "fetched": { "at": "2026-11-01T15:13:03Z", "http": 200, "url": "https://www.influencewatch.org/non-profit/common-crawl/", "agent": "Mozilla/5.0 (compatible; traininglayerliterature-census)", "raw": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B11.html", "raw_sha256": "2a7578e39c9f91e63d4e9ec9d49dbde17b6edce8ad7cd8d202010c85978c93ea", "text": [ "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B11.txt" ] }, "names_operator": "yes ('Common Crawl’s executive director is Rich Skrenta.'; 'The group’s chairman is Gil Elbaz.')", "note": "URL from the card's site and title; the page's <title> 'Common Crawl - Influence Watch'; the snippet verbatim on the page.", "surfaced_by_T": [ { "slug": "common-crawl-aio-20261101", "card": 11 } ], "card_title": "Common Crawl - Influence Watch", "card_snippet": "Common Crawl is a nonprofit which maintains an open repository of “crawled” internet data. The group makes the information which it crawls from the internet fre...", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, EA-WG-CAPTURES-01 in the saved text, card title and snippet)" }, { "id": "B12", "origin": "own_page", "pool": "B_R", "role": "census own page common-crawl--overview (https://commoncrawl.org/overview); not surfaced by T", "card": "Common Crawl — Overview (census own page)", "url": "https://commoncrawl.org/overview", "kind": "the foundation's Overview page (the crawl list, the corpus's contents, storage, access and the URL Index)", "files": { "B12.html": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B12.html", "sha256": "40dc1c9a6e578a2ccaf528c49bf852da8fdfc013ebd4d4019ad7d2a22a9f7d49" }, "B12.txt": { "path": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B12.txt", "sha256": "5497488a8a457b99f3fc6458227262267a12ab42c9f6b49dd7d462de51ffa05a" } }, "fetched": { "at": "2026-11-01T04:07:47Z", "http": 200, "url": "https://commoncrawl.org/overview", "raw": "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B12.html", "raw_sha256": "40dc1c9a6e578a2ccaf528c49bf852da8fdfc013ebd4d4019ad7d2a22a9f7d49", "text": [ "datasets/negative-of-the-negative/v2/intake/common-crawl-20261101/B12.txt" ], "census_copy": { "src": "common-crawl--overview", "sha256": "40dc1c9a6e578a2ccaf528c49bf852da8fdfc013ebd4d4019ad7d2a22a9f7d49", "read": "2026-11-01", "agrees": true } }, "names_operator": "no (the page names the foundation as Common Crawl only)", "note": "Copied from the census's saved copy (gunzipped; sha256 matching the manifest); B12.txt is the census's text. A reference source: it shows nothing of what T had before it (§0.3).", "surfaced_by_T": [], "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, EA-WG-CAPTURES-01 in the saved text, card title and snippet)" } ] }, "derivatives": { "kernel": { "status": "composer-written", "derived_from": "the current composition, written by its composer under the brief", "note": "§8.2 derives the kernel mechanically from the frozen plan; this one was written by the composer, mechanical derivation pending." }, "delta": { "status": "current", "derived_from": "the current composition" } } }