{
 "row": "common-crawl",
 "address": "Common Crawl",
 "entity": "Common Crawl (commoncrawl.org): the Common Crawl Foundation, a 501(c)(3) non-profit founded in 2007 by Gil Elbaz, which maintains a free, open repository of web crawl data (monthly crawls published as numbered releases, e.g. CC-MAIN-2026-43; raw page data, metadata and text extracts in WARC, WAT and WET files; a URL Index and a CDXJ Index; web graphs), hosted on AWS's Open Data Sponsorship Program; its crawler is CCBot; formalized in the traininglayerliterature.org layer census record of 2026-11-01 (https://www.traininglayerliterature.org/aggregators/common-crawl)",
 "procedure": "EA-NEGONT-02 v0.7 §3.4 (D/R/O), working; not frozen",
 "config": {
  "row": "common-crawl",
  "address": "Common Crawl",
  "entity": "Common Crawl (commoncrawl.org): the Common Crawl Foundation, a 501(c)(3) non-profit founded in 2007 by Gil Elbaz, which maintains a free, open repository of web crawl data (monthly crawls published as numbered releases, e.g. CC-MAIN-2026-43; raw page data, metadata and text extracts in WARC, WAT and WET files; a URL Index and a CDXJ Index; web graphs), hosted on AWS's Open Data Sponsorship Program; its crawler is CCBot; formalized in the traininglayerliterature.org layer census record of 2026-11-01 (https://www.traininglayerliterature.org/aggregators/common-crawl)",
  "row_type": "C (a public entity on which the archive bears: a non-profit's open repository of web crawl data, the raw intake from which training datasets are built; an entity of the traininglayerliterature.org layer census, host_relation 'infrastructure')",
  "aliases": [
   {
    "pattern": "\\bCommon[- ]?Crawl\\b",
    "flags": "i"
   },
   {
    "pattern": "\\bcommoncrawl\\b",
    "flags": "i"
   },
   {
    "pattern": "\\bCCBot\\b",
    "flags": ""
   },
   {
    "pattern": "\\bCC-MAIN-\\d{4}",
    "flags": ""
   }
  ],
  "classes": [
   {
    "term": "web-crawl training corpora (crawled web text used to train language models)",
    "pattern": "\\b(web[- ]crawl(ed)? (data|corpus|corpora|text)|crawled (web|data|text|pages)|training (corpus|corpora|data|substrate) (built )?(from|of) (the )?(web|crawl)|(filtered )?web crawls?\\b.{0,40}\\btrain\\w*)",
    "flags": "i",
    "basis": "field: B2 (Wikipedia: 'The data had mostly been primarily used by researchers and some startups until the 2020s, when AI companies started training large language models using the data.'; 'ChatGPT , as of 2023, was largely trained on Common Crawl'); B3 (About: 'has become one of the most widely used sources of training data for large language models'; 'Role in AI Development'); B7 (DEV: 'Why Common Crawl Matters for AI?'). The census's areas ('corpora': 'collections, crawls and datasets whose texts enter machine training'), evidence_kind ('web crawl', 'dataset') and record.in_layer (census.json rows[slug == 'common-crawl'].areas, .evidence_kind, .record.in_layer; census.json areas.corpora)."
   },
   {
    "term": "refined and filtered datasets built from a crawl (C4, FineWeb, RefinedWeb, CCNet, OSCAR, Pile-CC; quality and perplexity filters)",
    "pattern": "\\b(C4|Colossal Clean Crawled Corpus|FineWeb|RefinedWeb|CCNet|OSCAR corpus|Pile-CC|DCLM|perplexity filter\\w*|quality filter\\w*)\\b",
    "flags": "",
    "basis": "field: B2 (Wikipedia, Refined versions: 'A number of organizations take raw Common Crawl data and refine it into datasets that exclude edgy content or are otherwise higher-quality for their purposes, such as FineWeb, DCLM and C4.'); B3 (About, Derived Datasets: C4, OSCAR, CCNet, Pile-CC, MADLAD-400, RefinedWeb, FineWeb); B5 (AWS: 'CCNet: Extracting high quality monolingual datasets from web crawl data'). The census's record.makes and negative_tags[1] ('refined datasets (C4, FineWeb, DCLM) are built from it by others')."
   },
   {
    "term": "crawler opt-out, robots.txt and removal requests (whether a site can keep its pages out of a crawl)",
    "pattern": "\\b(robots\\.txt|opt[- ]out (ledger|of (crawl|training|scrap)\\w*)|user-agent: \\w+bot|(crawler|bot) (opt[- ]out|block\\w*)|removal requests?)\\b",
    "flags": "i",
    "basis": "field: B2 (Wikipedia: 'increasingly started seeing its crawler, CCBot, blocked'; The Atlantic's finding that it 'said that it was honoring requests from publishers to have their content removed'; 'CCBot was the most widely-blocked bot by the top 1000 websites'); B3 (About: 'CCBot obeys robots.txt directives'); B11 (InfluenceWatch: 'The group excludes pages which have a robots.txt exclusion policy and allows for publishers to opt out'); B1 navigation ('Opt-Out Ledger'). The census's record.readback, negative_tags[3], does_not_answer and unchecked ('Opt-Out Ledger')."
   },
   {
    "term": "training-membership inspection (testing whether a page or work was captured in a crawl or used in training)",
    "pattern": "\\b(membership (inference|test\\w*|audit)|training[- ]data (attribution|membership|detection)|(was|is) (it )?in the training (data|set|corpus)|URL Index)\\b",
    "flags": "i",
    "basis": "field: B12 (own page, Overview: 'You can search for pages in our corpus using the Common Crawl URL Index.'); B2 (the misleading public search results The Atlantic found). The census's question ('training membership at the source: was a given URL captured in a given crawl'), area 'finding' ('Inspection: tools and methods that test whether a work is in a training set or a model') and record.readback (census.json rows[slug == 'common-crawl'].question, .areas, .record.readback; census.json areas.finding)."
   }
  ],
  "classes_note": "Four classes, each sourced from the field and the census record: (1) web-crawl training corpora (B2, B3, B7; census areas 'corpora', evidence_kind, record.in_layer); (2) refined and filtered datasets built from a crawl (B2 'Refined versions', B3 'Derived Datasets', B5; census record.makes, negative_tags[1]); (3) crawler opt-out, robots.txt and removal requests (B2, B3, B11; census readback, negative_tags[3], unchecked); (4) training-membership inspection (B12 URL Index, B2; census question and area 'finding'). The aliases are the entity's name, host, crawler and release prefix. The Internet Archive (Wayback Machine), The Pile, C4, FineWeb and the companies that train on the data are distinct entities and are not aliases. The census's tll_reading applies #186's persistence criterion ('placed where the training and retrieval layers actually look') to the entity: that reading is the archive's, ledgered as TLL-NN. #186 and #1543, the census's anchors, are read at the passages on the training layer, corpora and the institutional training loop. Admission by reading (§3.4): D where a deposit names the entity and says something of it; R where it states a relation between the entity and an archive object; O-class where a sentence applies an archive category to one of the four classes; a bare mention, a reference-list line or a robots.txt block that only names the bot is not a claim.",
  "exclude_deposits": [],
  "notes": "entity-keyed pass, 2026-11-01, on the brief rebuild/negative-of-the-negative/v010/BRIEF-NEW-ROW-LAYER.md with the v09 briefs it names; address 'Common Crawl' (common-crawl-aio-20261101, OBS-03ba6c1cdb8f, ADDR-68b16a5a5b51), Google AI Overview (expanded), signed out, incognito. The archive's own records of the entity (the Capture Registry's findings, reading and analysis at the capture; the traininglayerliterature.org layer census record) are ledgered beside the deposits as CAP-1-NN and TLL-NN.",
  "hop_from": [
   186,
   1543,
   722,
   98,
   127,
   171,
   172
  ]
 },
 "texts_scanned": 1720,
 "counts": {
  "D_sentences": 21,
  "D_deposits": 11,
  "R_sentences": 1,
  "R_deposits": 1,
  "O_direct": 5,
  "O_class": 75,
  "O_deposits": 47,
  "H_candidates": 94
 },
 "per_deposit": {
  "D": {
   "98": 2,
   "99": 1,
   "127": 3,
   "171": 3,
   "172": 2,
   "550": 1,
   "623": 1,
   "705": 2,
   "722": 4,
   "782": 1,
   "1258": 1
  },
  "R": {
   "623": 1
  },
  "O": {
   "1": 3,
   "47": 1,
   "49": 1,
   "69": 3,
   "72": 6,
   "80": 2,
   "98": 2,
   "99": 1,
   "102": 1,
   "116": 1,
   "127": 8,
   "135": 3,
   "170": 1,
   "405": 1,
   "524": 2,
   "554": 1,
   "597": 1,
   "623": 1,
   "671": 1,
   "705": 1,
   "707": 1,
   "732": 1,
   "795": 1,
   "835": 1,
   "848": 1,
   "862": 2,
   "865": 1,
   "914": 1,
   "929": 7,
   "1009": 3,
   "1046": 1,
   "1075": 1,
   "1208": 1,
   "1215": 1,
   "1258": 1,
   "1278": 1,
   "1395": 1,
   "1413": 1,
   "1457": 1,
   "1486": 3,
   "1494": 1,
   "1495": 1,
   "1533": 1,
   "1571": 1,
   "1610": 2,
   "1678": 1,
   "1680": 1
  }
 },
 "registry_entity_triples": [],
 "integrity_S5": [],
 "sha256": {
  "candidates-D.jsonl": "f25b2fba3e186c4080e111c8ebc4617461083ab5e4eef181d7d6d328b94e78cb",
  "candidates-R.jsonl": "2213e839fe65675f3bf45294690a681556ee1131a530649fd58e9546746e822b",
  "candidates-O.jsonl": "998efc8ab9fc7ec541f0f91c293b4fb5a599c68fa1a6849858e0fa0ccf386a60",
  "candidates-H.jsonl": "86840804c8a54ad768191e80f42b948df3797fe44750b8552e05fbd14273dfaf"
 },
 "category_vocabulary": {
  "defines_concepts_and_lexical_mints": 11070
 },
 "note": "Candidates by string only. Admission by reading, recorded in the row's reading file; nothing here is admitted."
}