{ "row": "the-pile", "entity": "the-pile", "address": "The Pile dataset", "addresses": [ { "address": "The Pile dataset", "slug": "the-pile-dataset-aio-20261101", "addr_id": "ADDR-c51497a0501d", "obs_id": "OBS-3bb4c361d912", "axn": "AXN:0BBA.CAPTURE.⏩🕙🏛️🧡🔅🪐" } ], "epoch": "2026-11-01", "type": "C (public entity on which the archive bears: a training dataset of English text in twenty-two components; an entity of the traininglayerliterature.org layer census, host_relation 'infrastructure')", "surface": "Google AI Overview (expanded)", "auth": "signed out, incognito", "status": { "procedure": "EA-NEGONT-02 v0.10 (#1726): equal case, ρ = extends, read from the traininglayerliterature.org layer census record, the capture record and the deposits that bear on the entity and its classes (#186 and #1543, the census's anchors; #478, #702, #264, #727, #939, #530, #753, #943, #1053, #1443, #1075, #127, #837, #102, #1701, #482, #49; #782, the one deposit that names the Pile); archive selected by D/R/O (1 D candidate, R empty, 358 O-class candidates in 232 deposits read by filter and sample, hop of 28 read; 20 deposits admitted with 4 instances kept), with the census record (40 claims) and the Capture Registry's findings, reading, analysis, title, PER note and query note (15 claims) ledgered as the archive's records of the entity; the field in two pools (B_D: 8 cards, 2 unreachable, the own home page and Wikipedia among them; B_R: 1 own page and 1 census third-party source); the composer read the ledgers, never the transcript, except where the brief asks for T's claims and the delta", "ledger": "unaudited (§3.8: one extractor per ledger); field quotes 78/78 verified by script against saved text, the wikitext, the paper's extracted text or the card text; archive quotes 86/86 verified against the registry, census.json and the deposit texts", "frozen": false, "breaches": [] }, "spec": "EA-NEGONT-02 v0.10 (#1726)", "objects": { "T": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/T1-the-pile-dataset-aio-20261101.txt", "sha256": "7cd7268e782e9ed3bdf069ac41e98fa6ea6168a63712e90b5feaad916b7a02b6", "words": 287, "cards": 8, "words_note": "one transcript at one address (287 words in the answer body, the paste note and the card rail excluded; a closing offer of 30 words more; no inline link markers; 8 cards)", "claims": [ "T1-1 definition: 'The Pile is an 825 GiB to 886 GB open-source English text dataset created by EleutherAI and released on December 31, 2020, to train large language models (LLMs).'", "T1-2 size: 'Size: ~825 GiB compressed (zstandard format) containing roughly 300 billion tokens.'", "T1-3 composition: 'Composition: Combines 22 diverse sub-datasets across academic, web, prose, and code domains.'", "T1-4 purpose: 'Purpose: Designed to improve cross-domain knowledge, reasoning, and generalization compared to raw web scrapes like Common Crawl.'", "T1-5 models: 'Notable Models Trained On It: GPT-Neo, GPT-J, and the Pythia suite by EleutherAI.'", "T1-6 the components: 'The Pile is split into 22 distinct components spanning various types of human writing and knowledge:' (twenty-two numbered, each glossed, among them '3. Books3 (Long-form literary text/books)', '11. Gutenberg (PG-19) (Public domain literature)', '16. BookCorpus2 (Additional books dataset)', '19. YouTube Subtitles (Transcribed video subtitles)')", "T1-7 copyright: 'Copyright Disputes: The dataset encountered copyright challenges regarding specific components like Books3 and subtitles, resulting in content adjustments.'", "T1-8 successor: 'The Common Pile v0.1: EleutherAI introduced ethical alternatives such as Common Pile v0.1 using licensed text.'", "T1-9 closing offer: 'Would you like to know more about how specific sub-datasets like GitHub or arXiv affected model performance, or are you looking for details on ethical alternatives like the Common Pile?'" ], "register": { "register": "data/EA-WG-CAPTURES-01.json", "slug": "the-pile-dataset-aio-20261101", "obs_id": "OBS-3bb4c361d912" }, "transcripts": [ { "slug": "the-pile-dataset-aio-20261101", "address": "The Pile dataset", "addr_id": "ADDR-c51497a0501d", "obs_id": "OBS-3bb4c361d912", "axn": "AXN:0BBA.CAPTURE.⏩🕙🏛️🧡🔅🪐", "date": "2026-11-01", "surface": "Google AI Overview (expanded)", "auth": "signed out, incognito", "sha256": "7cd7268e782e9ed3bdf069ac41e98fa6ea6168a63712e90b5feaad916b7a02b6", "sha256_of": "the registry's 'transcript' string (utf-8)", "sha256_raw": "9f8ab5b92859d59e8075fd11d067b3c1aaba1bf3e397ecc93a4321980417fac7", "words": 287, "cards": 8, "card_rail": [ "1. The Pile — The Pile", "2. Wikipedia — The Pile (dataset) - Wikipedia", "3. Hugging Face — EleutherAI/pile · Datasets at Hugging Face", "4. Reddit·r/MachineLearning — [R] The Pile: An 800GB Dataset of Diverse Text for Language Modeling - Reddit", "5. The Pile — The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "6. YouTube·The Polymath — The Pile - An 800GB Dataset for Language Models - [Paper Read #1]", "7. Hugging Face — The Common Pile v0.1 - Hugging Face", "8. Michael Brenndoerfer — The Pile: Open-Source Training Dataset for LLMs - Interactive" ], "claims": [ "T1-1 definition: 'The Pile is an 825 GiB to 886 GB open-source English text dataset created by EleutherAI and released on December 31, 2020, to train large language models (LLMs).'", "T1-2 size: 'Size: ~825 GiB compressed (zstandard format) containing roughly 300 billion tokens.'", "T1-3 composition: 'Composition: Combines 22 diverse sub-datasets across academic, web, prose, and code domains.'", "T1-4 purpose: 'Purpose: Designed to improve cross-domain knowledge, reasoning, and generalization compared to raw web scrapes like Common Crawl.'", "T1-5 models: 'Notable Models Trained On It: GPT-Neo, GPT-J, and the Pythia suite by EleutherAI.'", "T1-6 the components: 'The Pile is split into 22 distinct components spanning various types of human writing and knowledge:' (twenty-two numbered, each glossed, among them '3. Books3 (Long-form literary text/books)', '11. Gutenberg (PG-19) (Public domain literature)', '16. BookCorpus2 (Additional books dataset)', '19. YouTube Subtitles (Transcribed video subtitles)')", "T1-7 copyright: 'Copyright Disputes: The dataset encountered copyright challenges regarding specific components like Books3 and subtitles, resulting in content adjustments.'", "T1-8 successor: 'The Common Pile v0.1: EleutherAI introduced ethical alternatives such as Common Pile v0.1 using licensed text.'", "T1-9 closing offer: 'Would you like to know more about how specific sub-datasets like GitHub or arXiv affected model performance, or are you looking for details on ethical alternatives like the Common Pile?'" ], "markers": 0, "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/T1-the-pile-dataset-aio-20261101.txt", "note": "The paste is the Overview expanded (the 'AI Mode Conversation' view), surface and auth by the operator's defaults of 2026-10-01 and 2026-10-14 as the registry records them (surface_basis, auth_basis); run with the census names, the paste received 2026-11-01 09:56 EDT as one of two attached files (The Literary History of Artificial Intelligence and The Pile)" } ] }, "KO": { "title": "The Pile", "genre": "knowledge object, equal case, under EA-NEGONT-02 v0.10 (#1726): the final article form (§4.13), natural sentences under descriptive subheadings (§4.12), each sentence's sources beneath it by claim id (§4.5)", "status": "composed 2026-11-01 under v0.10 on the operator's ruling of 2026-10-29, confirmed 2026-11-01 ('as against their wikipedia articles if they exist, their own pages, etc.'); not frozen", "sentences": [ { "n": 1, "para": 0, "sense": "what it is", "text": "The Pile is an open-source dataset of English text that EleutherAI assembled to train large language models, twenty-two smaller datasets combined into one, released on 31 December 2020.", "claims": [ "F1", "F11", "F37", "F76", "F27", "F36", "TLL-01", "TLL-02", "TLL-04" ] }, { "n": 2, "para": 0, "sense": "what it is", "text": "Its paper, by Leo Gao, Stella Biderman and ten others, presents it as a dataset designed for training large-scale language models and also as a broad-coverage benchmark of cross-domain knowledge and generalization; Gao and Biderman are named as its principal curators.", "claims": [ "F38", "F75", "F31", "F10", "F63", "TLL-03", "TLL-17", "TLL-36" ] }, { "n": 3, "para": 0, "sense": "what it is", "text": "Its size is given three ways: 800GB in the paper's title, 825 GiB on its home page and in the abstract (825.18 GiB raw, 1254.20 GiB once its components are weighted), and 886.03 GB in Wikipedia's table of components.", "claims": [ "F2", "F45", "F21", "F35", "TLL-18", "TLL-20", "TLL-32" ] }, { "n": 4, "para": 0, "sense": "what it is", "text": "It is distributed as jsonlines compressed with zstandard.", "claims": [ "F4", "TLL-16" ] }, { "n": 5, "para": 1, "sense": "the components", "text": "Its components are fourteen language-modelling datasets new with the Pile and existing datasets it incorporated, Books3, Project Gutenberg's PG-19, OpenSubtitles and English Wikipedia among them; in Wikipedia's account the twenty-two include books, movie transcripts and scientific papers.", "claims": [ "F39", "F40", "F12", "F61", "TLL-11" ] }, { "n": 6, "para": 1, "sense": "the components", "text": "The census takes as its unit a component and, within a component, a document; the paper weights components by quality, some seen up to three times in an epoch.", "claims": [ "TLL-10", "F45" ] }, { "n": 7, "para": 1, "sense": "the components", "text": "Pile-CC, the largest component in Wikipedia's table at 243.87 GB, is the Pile's own extraction from Common Crawl, made with jusText on Web Archive files; the archive holds that training corpora are built from the documents available through the crawlable surface, the documents retrieval systems index.", "claims": [ "F44", "F21", "TLL-19", "TLL-33", "O753-01" ] }, { "n": 8, "para": 1, "sense": "the components", "text": "Books3 is a dataset of books derived from a copy of the contents of the Bibliotik private tracker made available by Shawn Presser, fiction and nonfiction, which the paper included \"because books are invaluable for long-range context modeling research and coherent storytelling\".", "claims": [ "F41", "F73" ] }, { "n": 9, "para": 1, "sense": "the components", "text": "PG-19 holds Project Gutenberg books from before 1919, classic Western literature, and BookCorpus2 extends a corpus of books by \"as of yet unpublished authors\".", "claims": [ "F42", "F43" ] }, { "n": 10, "para": 1, "sense": "the components", "text": "The archive observes of training corpora that, scraped from HTML that discarded layout and from PDFs whose columns were linearized, they typically deliver no stanza break for a tokenizer to hold, and that an element a training corpus omits is inherited by every reader who works only from derivative forms.", "claims": [ "O943-01", "O1053-01" ] }, { "n": 11, "para": 2, "sense": "its standing among training corpora", "text": "As of 2024 it was, with Common Crawl, one of the two main training datasets used to train AI models; it has been described as the early 825GB open dataset that broadened access to training data, and an archive study plan names it among standard LLM pre-training datasets as data for a pre-registered test of recursive model collapse, a specification not yet run.", "claims": [ "F13", "F70", "D782-01", "D782-02" ] }, { "n": 12, "para": 3, "sense": "what a model trained on it makes", "text": "In its makers' evaluation untuned GPT-2 and GPT-3 struggle on many of its components, such as academic writing, while models trained on the Pile improve significantly over Raw CC and CC-100 on all its components and improve on downstream evaluations.", "claims": [ "F46", "F5" ] }, { "n": 13, "para": 3, "sense": "what a model trained on it makes", "text": "It is also a benchmark, Pile BPB, bits per UTF-8 encoded byte, which its home page calls a measure of world knowledge and reasoning across its domains, from books and code repositories to chat logs and papers in medicine, physics, mathematics, computer science and philosophy.", "claims": [ "F6", "F7", "F47", "F38" ] }, { "n": 14, "para": 3, "sense": "what a model trained on it makes", "text": "Its leaderboard lists GPT-3, zero-shot, at 0.7177 and marks potential test-set overlap, and the paper allows that some documents may be duplicated across its train, validation and test splits.", "claims": [ "F9", "F48" ] }, { "n": 15, "para": 3, "sense": "what a model trained on it makes", "text": "The archive holds that what a benchmark measures is determined by the training corpus, and what the corpus contains by the political economy of dataset construction.", "claims": [ "O530-01" ] }, { "n": 16, "para": 3, "sense": "what a model trained on it makes", "text": "EleutherAI later described it as 300 billion GPT-2 tokens on which it was actively training a large language model, then the largest pretraining dataset released by its creators, and GPT-Neo, released early in 2021, as the first LLM trained on data its creators had published; GPT-J was trained on it as well, and its Hugging Face page lists 862 models trained or fine-tuned on it, Pythia, GPT-Neo and GPT-NeoX models among them.", "claims": [ "F64", "F65", "F72", "F34" ] }, { "n": 17, "para": 3, "sense": "what a model trained on it makes", "text": "The census reads what the receiving apparatus makes from a work there as a language model trained on it and scored on the Pile's own benchmark.", "claims": [ "TLL-15" ] }, { "n": 18, "para": 3, "sense": "what a model trained on it makes", "text": "The archive's account of machine composition sets the test of a work at what a machine can write after encountering it, and names the institutional training loop, from work to model to artifact to the next corpus and the next model, as its especially important case.", "claims": [ "O1543-01", "O1543-02" ] }, { "n": 19, "para": 3, "sense": "what a model trained on it makes", "text": "In the archive's terms the receiving apparatus's uptake is the text's afterlife and the uptaking system's own act, and a text included in a training set is reproduced, paraphrased, cited and culturally amplified in ways that texts left out cannot match.", "claims": [ "O1543-04", "O1701-01", "O102-01" ] }, { "n": 20, "para": 4, "sense": "authorial consent", "text": "The paper defines data with authorial consent as data whose original authors consented to its use, or where a reasonable person could not assume it would not be used for purposes such as research, and treats ambiguous or unknown consent as nonconsensual.", "claims": [ "F51", "F52" ] }, { "n": 21, "para": 4, "sense": "authorial consent", "text": "It notes that one is typically not legally obligated to obtain an author's permission to train on a work, and finds five of its datasets neither collected in a way consistent with their hosts' terms of service nor open to their authors' consent; Books3 and OpenSubtitles it calls consistent with their hosts' terms, a description it calls somewhat misleading, since the hosts were not authorized by the owners to post the data.", "claims": [ "F50", "F53", "F54" ] }, { "n": 22, "para": 4, "sense": "authorial consent", "text": "The Enron emails were collected without their authors' permission by the U.S. government in a criminal investigation, and the YouTube Subtitles dataset was made by the Pile's authors with a widely used unofficial API.", "claims": [ "F55", "F56" ] }, { "n": 23, "para": 4, "sense": "authorial consent", "text": "The paper argues that noncommercial, not-for-profit use of copyright material is fair use and that its own use is transformative; a later account says the legal considerations were addressed through careful licensing review.", "claims": [ "F58", "F71" ] }, { "n": 24, "para": 4, "sense": "authorial consent", "text": "The archive describes the training corpus of a large language model as human linguistic production \"extracted without consent, crystallized in weight matrices, and owned absolutely by capital\", as semantic capital whose value the extractor captures, and as the corpus that ingests a poet's entire output to produce outputs that sound like the poet \"but carry none of the cost\"; it observes that the literature on digital surrogates cannot speak of eligibility, consent or the training corpus as a reception site.", "claims": [ "O478-01", "O702-01", "O49-01", "O1443-01" ] }, { "n": 25, "para": 5, "sense": "Books3: source, takedown, suit", "text": "Copyright disputes over the Pile escalated in 2023, and EleutherAI began removing datasets.", "claims": [ "F14" ] }, { "n": 26, "para": 5, "sense": "Books3: source, takedown, suit", "text": "Books3 contains copyrighted material compiled from Bibliotik, \"a piracy website\"; the Danish anti-piracy group Rights Alliance took Books3 down through DMCA notices in July 2023, and it was removed from the Pile before three authors filed a class action in 2024, while copies of the original dataset were still copied and available on the web.", "claims": [ "F15", "F16", "F17", "TLL-12" ] }, { "n": 27, "para": 5, "sense": "subtitles: OpenSubtitles and YouTube", "text": "OpenSubtitles drew controversy over copyrighted documentaries, films, television and online video, and the subtitles of tens of thousands of YouTube videos were scraped into the Pile, which YouTube argued is against its terms of service.", "claims": [ "F19", "F20" ] }, { "n": 28, "para": 5, "sense": "Books3: source, takedown, suit", "text": "Wikipedia's table of components marks copyright disputes with an asterisk that falls on PubMed Central and YouTube Subtitles and leaves Books3 and OpenSubtitles unmarked.", "claims": [ "F22" ] }, { "n": 29, "para": 6, "sense": "how a work in it is named", "text": "The Pile asks those who use it or any of its components to cite the Pile; on its Hugging Face card a document's metadata names the subset it belongs to (pile_set_name), and the entry for the source language producers reads \"[More Information Needed]\".", "claims": [ "F8", "F29", "F30" ] }, { "n": 30, "para": 6, "sense": "how a work in it is named", "text": "The census finds no record in the Pile's own saved pages of the name under which a work is held, leaves unanswered under what author name a work sits in a component, and reads the Books3 removal and the authors' suit as the field's clearest case of works found in the layer under their authors' names after the fact.", "claims": [ "TLL-13", "TLL-22", "TLL-25" ] }, { "n": 31, "para": 6, "sense": "how a work in it is named", "text": "The archive holds that every text in a training corpus represents semantic labor by people who are not compensated and often not attributed, that a training corpus without provenance is not a commons, and that in provenance debt the training corpus is the commons and the model the enclosure.", "claims": [ "O264-01", "O727-01", "O939-01" ] }, { "n": 32, "para": 6, "sense": "how a work in it is named", "text": "Its field survey asks that a work be placed where the training and retrieval layers look, \"with the metadata that lets it be found and attributed\", and records a writer who made the model's training data, the writer's own words among them, the explicit subject of the work; its account of machine composition keeps apart whether a machine wrote something new and whether it told the truth about where it came from.", "claims": [ "O186-05", "O186-03", "O1543-03" ] }, { "n": 33, "para": 7, "sense": "how a work in it is found", "text": "The census asks of the Pile which of its twenty-two components carries a given work, and records that its own pages offer a download and a benchmark with no search by work, so that whether a given work is in it goes unanswered there.", "claims": [ "TLL-09", "TLL-14", "TLL-31" ] }, { "n": 34, "para": 7, "sense": "how a work in it is found", "text": "The paper states that its authors lack the metadata to determine exactly which texts are copyrighted, so that exclusion can be made only by component and is to be taken as a heuristic; its reproduction code can exclude components, and the processing applied makes particular files in the Pile difficult to identify.", "claims": [ "F59", "F60", "F57" ] }, { "n": 35, "para": 7, "sense": "how a work in it is found", "text": "Its Hugging Face card calls itself a work in progress and points to a datasheet; the repository there holds 24.9 kB, and its viewer is disabled.", "claims": [ "F28", "F33" ] }, { "n": 36, "para": 7, "sense": "how a work in it is found", "text": "A later paper on Data Portraits names the difficulty of knowing whether a model has encountered a given example in training, and documents the Pile with data-sketch tools for questions of test-set leakage and model plagiarism.", "claims": [ "F77", "F78", "TLL-34" ] }, { "n": 37, "para": 7, "sense": "how a work in it is found", "text": "The archive records the copyright trap, a fictitious sentence inserted into a copyrighted document whose presence in a training set a membership inference attack can detect after training, and a paraphrase-score commitment by which a producer can prove that a corpus contained a text; its own documents carry training-detection canaries, phrases whose production by a model shows the document was in its training corpus, and it estimates its whole corpus at no more than about 10^-7 of a frontier pre-training corpus.", "claims": [ "O127-01", "O127-02", "O837-01", "O127-03" ] }, { "n": 38, "para": 7, "sense": "how a work in it is found", "text": "It proposes that where a training corpus is an enumerable set of individually authored works whose use is material, the data citation must resolve to the works, and holds that an institution that cannot say what is in its training corpus has a problem no licence can cure.", "claims": [ "O1075-01", "O1075-02" ] }, { "n": 39, "para": 8, "sense": "the training layer itemized", "text": "The archive's field survey places training-layer literature in the layer of culture where texts are crawled, indexed, trained on, retrieved and recomposed into answers, and observes that the works of the writers it surveys are, as a matter of fact, in the corpora, read by machines incidentally.", "claims": [ "O186-01", "O186-02" ] }, { "n": 40, "para": 8, "sense": "the training layer itemized", "text": "The archive also calls the training corpus the model's para-semiotic unconscious, and notes that a literature written to be trained on enters corpora themselves degrading under synthetic flood.", "claims": [ "O482-01", "O186-04" ] }, { "n": 41, "para": 8, "sense": "the training layer itemized", "text": "The census reads the Pile as the training layer named and itemized, a work entering it by belonging to a component such as Books3, PG-19 or Pile-CC.", "claims": [ "TLL-24" ] }, { "n": 42, "para": 8, "sense": "the training layer itemized", "text": "It enters the Pile as infrastructure to its question of how literature is addressed to a receiving apparatus, what that apparatus makes from it, and how transmission, transformation and provenance can be inspected across later reception, in its area of the corpora, a dataset whose evidence is the dataset, its documentation and its paper, under the anchors #186 and #1543 and the rule that a row gathers and holds many works or records and says what it holds; it records that the Pile answers which components make it up and how a model scores on Pile BPB.", "claims": [ "TLL-07", "TLL-37", "TLL-05", "TLL-40", "TLL-06", "TLL-08", "TLL-39", "TLL-38", "TLL-21" ] }, { "n": 43, "para": 9, "sense": "where it is held", "text": "Its home page still names the Eye as its host and offers a download; by 2024, Wikipedia states, it had been taken down from its original site and remained accessible on other file-sharing services, and the census leaves its present host unanswered.", "claims": [ "F3", "F18", "TLL-23", "TLL-26" ] }, { "n": 44, "para": 9, "sense": "licence and access", "text": "Its Hugging Face card gives its licence as \"other\", by subset, naming only PubMed Central's; the census found no licence in its saved sources.", "claims": [ "F32", "TLL-27" ] }, { "n": 45, "para": 10, "sense": "the successor: Common Pile v0.1", "text": "In June 2025 EleutherAI, with partners that Wikipedia lists as Poolside, Hugging Face and the US Library of Congress and researchers at 14 institutions, released the Common Pile v0.1, an 8 TB corpus of openly licensed and public-domain text that contains only works whose licences permit training, the Pile's successor and a distinct dataset.", "claims": [ "F23", "F25", "F68", "TLL-30" ] }, { "n": 46, "para": 10, "sense": "the successor: Common Pile v0.1", "text": "Its intent, in Wikipedia's account, was to show what training can do while respecting copyright; its makers report models trained on it comparable to models trained on the Pile, write that lawsuits over training data have left sourcing practices unchanged and drastically reduced companies' transparency, and hold public release of training datasets necessary for rigorous science.", "claims": [ "F24", "F69", "F67", "F66" ] }, { "n": 47, "para": 10, "sense": "the successor: Common Pile v0.1", "text": "The census records the Pile as historical: Books3 removed, the dataset taken down by 2024, its successor released in 2025.", "claims": [ "TLL-28", "TLL-29" ] }, { "n": 48, "para": 11, "sense": "what it documents of itself", "text": "Its paper documents \"potentially concerning aspects of the data for prospective users\" through an exploratory analysis and makes its construction code public; a later account adds that filtering left gender, racial, cultural and geographic bias in it.", "claims": [ "F49", "F74" ] }, { "n": 49, "para": 11, "sense": "what it documents of itself", "text": "The census leaves unchecked the paper's full text, of which it saved only the abstract page, the arXiv licence page, the Eye, the 2024 suit and Pile-CC's derivation.", "claims": [ "TLL-35" ] }, { "n": 50, "para": 12, "sense": "reception: the encyclopedia, the cards, the census", "text": "Wikipedia's article on it was last edited on 20 August 2026; a Reddit post dated 1 January 2021 and a 35-minute video reading of the paper also carry it.", "claims": [ "F26", "F36", "F62" ] }, { "n": 51, "para": 12, "sense": "reception: the encyclopedia, the cards, the census", "text": "Asked for \"The Pile dataset\" on 1 November 2026, Google's AI Overview gave its maker, date, size, purpose and the models trained on it, listed the twenty-two components with a gloss on each, and closed on its copyright disputes and its licensed successor.", "claims": [ "CAP-1-11", "CAP-1-15", "CAP-1-13", "CAP-1-14", "CAP-1-01" ] }, { "n": 52, "para": 12, "sense": "reception: the encyclopedia, the cards, the census", "text": "The Capture Registry found that the Overview named the components and no work or author in them, glossed Books3 as \"Long-form literary text/books\", left out Books3's source, the takedown, the removal and the authors' suit, and gave the subtitles without their account.", "claims": [ "CAP-1-02", "CAP-1-04", "CAP-1-05", "CAP-1-06", "CAP-1-12" ] }, { "n": 53, "para": 12, "sense": "reception: the encyclopedia, the cards, the census", "text": "The registry also found the takedown from the original site absent, the benchmark uncarried, the paper's concerning aspects absent, the successor given without its partners or its date, and the Overview's \"roughly 300 billion tokens\" and its three model families on no source the census saved.", "claims": [ "CAP-1-07", "CAP-1-08", "CAP-1-09", "CAP-1-10", "CAP-1-03" ] } ], "joins": [ { "act": "order of senses", "sentences": "1–53", "claims": [], "note": "equal case (§4.7): the entity's dependency order, field and archive claims ordered together by genealogy within each sense, neither ledger placed first as a block. What it is (¶0) precedes its components (¶1), the unit every later sense uses; its standing among corpora (¶2) and what a model trained on it makes (¶3) follow the components; authorial consent (¶4) and the Books3 and subtitle record (¶5) precede how a work in it is named (¶6) and found (¶7), which they ground; the census's reading of the training layer (¶8) gathers the senses it reads; where it is held, its successor and what it documents of itself (¶9–¶11) are its lifecycle; reception (¶12) stands last, as a relation of the entity already composed." }, { "act": "holding together (the paper's fair-use argument and a later licensing claim, against its own consent findings)", "sentences": [ 20, 21, 23 ], "claims": [ "F53", "F54", "F58", "F71" ], "note": "P_coh (§4.4): the paper's findings on consent (F53, F54), its fair-use argument (F58) and a later account's 'careful licensing review' (F71) composed together, each with its source; the composer does not rule between them." }, { "act": "holding together (Wikipedia's prose and its table)", "sentences": [ 26, 28 ], "claims": [ "F15", "F19", "F22" ], "note": "§4.3 internally_inconsistent: the table's asterisk for copyright disputes leaves Books3 and OpenSubtitles unmarked where the prose gives both as disputed; both loci carried." }, { "act": "holding together (the components and the archive's corpus claims)", "sentences": [ 7, 10 ], "claims": [ "F44", "O753-01", "O943-01", "O1053-01" ], "note": "the archive's class claims on training corpora (the crawlable surface, layout lost in scraping, omissions inherited) placed in the components sense, beside Pile-CC's derivation; the archive applies them to training corpora as a class, and the composition applies none of them to the Pile beyond what the class membership (field B2, B5; census kind) carries." }, { "act": "holding together (the benchmark and the archive's claim on benchmarks)", "sentences": [ 13, 15 ], "claims": [ "F6", "O530-01" ], "note": "Pile BPB (F6) beside #530's claim that a benchmark measures what the corpus determines; the archive's claim is on benchmarks and corpora as a class." }, { "act": "placing in the open block", "sentences": "open", "claims": [ "TLL-22", "F59", "F29", "F57", "O1075-01", "F53", "F58", "F71", "F2", "F21", "F45", "F64", "F35", "F18", "F3", "TLL-23", "F22" ], "note": "five questions placed in the open block: whether a given work is in it and under what name; the consent and legality dispute; its size; where it is now held; the table's asterisks." } ], "plan": { "order": [ "F1", "F11", "F37", "F76", "F27", "F36", "TLL-01", "TLL-02", "TLL-04", "F38", "F75", "F31", "F10", "F63", "TLL-03", "TLL-17", "TLL-36", "F2", "F45", "F21", "F35", "TLL-18", "TLL-20", "TLL-32", "F4", "TLL-16", "F39", "F40", "F12", "F61", "TLL-11", "TLL-10", "F44", "TLL-19", "TLL-33", "O753-01", "F41", "F73", "F42", "F43", "O943-01", "O1053-01", "F13", "F70", "D782-01", "D782-02", "F46", "F5", "F6", "F7", "F47", "F9", "F48", "O530-01", "F64", "F65", "F72", "F34", "TLL-15", "O1543-01", "O1543-02", "O1543-04", "O1701-01", "O102-01", "F51", "F52", "F50", "F53", "F54", "F55", "F56", "F58", "F71", "O478-01", "O702-01", "O49-01", "O1443-01", "F14", "F15", "F16", "F17", "TLL-12", "F19", "F20", "F22", "F8", "F29", "F30", "TLL-13", "TLL-22", "TLL-25", "O264-01", "O727-01", "O939-01", "O186-05", "O186-03", "O1543-03", "TLL-09", "TLL-14", "TLL-31", "F59", "F60", "F57", "F28", "F33", "F77", "F78", "TLL-34", "O127-01", "O127-02", "O837-01", "O127-03", "O1075-01", "O1075-02", "O186-01", "O186-02", "O482-01", "O186-04", "TLL-24", "TLL-07", "TLL-37", "TLL-05", "TLL-40", "TLL-06", "TLL-08", "TLL-39", "TLL-38", "TLL-21", "F3", "F18", "TLL-23", "TLL-26", "F32", "TLL-27", "F23", "F25", "F68", "TLL-30", "F24", "F69", "F67", "F66", "TLL-28", "TLL-29", "F49", "F74", "TLL-35", "F26", "F62", "CAP-1-11", "CAP-1-15", "CAP-1-13", "CAP-1-14", "CAP-1-01", "CAP-1-02", "CAP-1-04", "CAP-1-05", "CAP-1-06", "CAP-1-12", "CAP-1-07", "CAP-1-08", "CAP-1-09", "CAP-1-10", "CAP-1-03" ], "senses": { "what it is": [ 1, 2, 3, 4 ], "the components": [ 5, 6, 7, 8, 9, 10 ], "its standing among training corpora": [ 11 ], "what a model trained on it makes": [ 12, 13, 14, 15, 16, 17, 18, 19 ], "authorial consent": [ 20, 21, 22, 23, 24 ], "Books3: source, takedown, suit": [ 25, 26, 28 ], "subtitles: OpenSubtitles and YouTube": [ 27 ], "how a work in it is named": [ 29, 30, 31, 32 ], "how a work in it is found": [ 33, 34, 35, 36, 37, 38 ], "the training layer itemized": [ 39, 40, 41, 42 ], "where it is held": [ 43 ], "licence and access": [ 44 ], "the successor: Common Pile v0.1": [ 45, 46, 47 ], "what it documents of itself": [ 48, 49 ], "reception: the encyclopedia, the cards, the census": [ 50, 51, 52, 53 ] }, "marks": [ { "claim": "F22", "mark": "internally_inconsistent", "with": [ "F15", "F19" ], "note": "the table's asterisk 'denotes works with copyright disputes' and leaves Books3 and OpenSubtitles unmarked, where the article's prose gives both as the components of the copyright disputes (§4.3; both loci cited)" }, { "claim": "F71", "mark": "contested", "with": [ "F53", "F54", "F15" ], "note": "B8's 'careful licensing review' against the paper's five datasets neither ToS-compliant nor consented (F53, F54) and Wikipedia's Books3 (F15); both composed (P_coh)" }, { "claim": "F2", "mark": "contested", "with": [ "F1", "F11", "F21", "F45" ], "note": "size given as 800GB, 825 GiB (825.18 raw, 1254.20 effective) and 886.03 GB; all composed with their sources" } ], "conflicts": [], "cuts": [], "carried": 164 } }, "P": { "title": "The Pile", "genre": "the compression: AIO's interaction grammar (lede, headed clusters, bolded labels, card rail), composed from the knowledge object; equal case", "status": "composed 2026-11-01 from the v0.10 knowledge object; not frozen", "lede": { "text": "is an open-source English text dataset that EleutherAI released on 31 December 2020 to train large language models: twenty-two datasets combined, Books3, PG-19 and Pile-CC among them, sized 800GB, 825 GiB or 886.03 GB. A work enters it by component; its pages offer no search by work.", "claims": [ "F1", "F11", "F37", "F76", "F27", "F36", "F2", "F21", "TLL-01", "TLL-02", "TLL-04", "TLL-24", "TLL-31" ] }, "sections": [ { "icon": "🧱", "head": "What it holds", "items": [ { "label": "Components:", "text": "14 new datasets and existing ones; weighted, some read three times.", "claims": [ "F39", "F40", "F12", "F61", "F45", "TLL-10", "TLL-11", "TLL-18", "TLL-20", "TLL-32", "F35", "F4", "TLL-16" ] }, { "label": "Books:", "text": "Books3 from a copy of the Bibliotik tracker; PG-19, pre-1919 Gutenberg; BookCorpus2.", "claims": [ "F41", "F73", "F42", "F43" ] }, { "label": "Web:", "text": "Pile-CC, its own Common Crawl extraction, the largest part.", "claims": [ "F44", "TLL-19", "TLL-33", "F13", "F70", "O753-01", "O943-01", "O1053-01", "D782-01", "D782-02" ] } ] }, { "icon": "⚙️", "head": "What it makes", "items": [ { "label": "Models:", "text": "trained models beat Raw CC and CC-100; GPT-Neo, GPT-J, Pythia; \"300 billion GPT-2 tokens\".", "claims": [ "F46", "F5", "F64", "F65", "F72", "F34", "TLL-15" ] }, { "label": "Benchmark:", "text": "Pile BPB; GPT-3 zero-shot 0.7177, overlap possible.", "claims": [ "F6", "F7", "F47", "F38", "F9", "F48", "O530-01" ] }, { "label": "Archive:", "text": "the training loop, work to model to next corpus; uptake as afterlife.", "claims": [ "O1543-01", "O1543-02", "O1543-04", "O1701-01", "O102-01" ] } ] }, { "icon": "⚖️", "head": "Consent and copyright", "items": [ { "label": "Consent:", "text": "unknown consent counted nonconsensual; five datasets neither ToS-compliant nor consented.", "claims": [ "F51", "F52", "F50", "F53", "F54", "F55", "F56" ] }, { "label": "Legality:", "text": "fair use argued (paper); \"careful licensing review\" (later account).", "claims": [ "F58", "F71" ] }, { "label": "Books3:", "text": "Bibliotik, \"a piracy website\"; taken down July 2023; removed before three authors sued in 2024.", "claims": [ "F14", "F15", "F16", "F17", "F22", "TLL-12" ] }, { "label": "Subtitles:", "text": "OpenSubtitles disputed; YouTube subtitles scraped.", "claims": [ "F19", "F20" ] }, { "label": "Archive:", "text": "corpora \"extracted without consent\"; semantic capital; consent unspoken by the surrogate literature.", "claims": [ "O478-01", "O702-01", "O49-01", "O1443-01" ] } ] }, { "icon": "🔎", "head": "Finding a work", "items": [ { "label": "By component:", "text": "copyright decided by component only, \"a heuristic\"; files hard to identify.", "claims": [ "F59", "F60", "F57", "TLL-09", "TLL-14", "F28", "F33" ] }, { "label": "Names:", "text": "a document records its subset, no author; the census reads the Books3 suit as works found under their authors' names after the fact.", "claims": [ "F8", "F29", "F30", "TLL-13", "TLL-22", "TLL-25" ] }, { "label": "Inspection:", "text": "Data Portraits documents it; copyright traps and canaries test membership.", "claims": [ "F77", "F78", "TLL-34", "O127-01", "O127-02", "O127-03", "O837-01" ] }, { "label": "Archive:", "text": "data citation should resolve to the works; corpora without provenance are no commons.", "claims": [ "O1075-01", "O1075-02", "O264-01", "O727-01", "O939-01", "O186-05", "O186-03", "O1543-03" ] } ] }, { "icon": "🏛️", "head": "Status", "items": [ { "label": "Held:", "text": "the Eye, per its page; taken down from its site by 2024.", "claims": [ "F3", "F18", "TLL-23", "TLL-26", "F32", "TLL-27" ] }, { "label": "Successor:", "text": "Common Pile v0.1, June 2025, licensed text only.", "claims": [ "F23", "F25", "F68", "F24", "F69", "F67", "F66", "TLL-30", "TLL-28", "TLL-29" ] }, { "label": "Census:", "text": "infrastructure; the training layer itemized.", "claims": [ "TLL-07", "TLL-37", "TLL-05", "TLL-40", "TLL-06", "TLL-08", "TLL-39", "TLL-38", "TLL-21", "O186-01", "O186-02", "O482-01", "O186-04", "TLL-35", "F49", "F74", "F2", "F10", "F31", "F63", "F75", "TLL-03", "TLL-17", "TLL-36" ] } ] }, { "icon": "🔍", "head": "At the name", "items": [ { "label": "Overview, 1 November 2026:", "text": "components listed, no work or author; Books3 \"Long-form literary text\"; source, removal and suit as \"content adjustments\" (Capture Registry).", "claims": [ "CAP-1-01", "CAP-1-02", "CAP-1-03", "CAP-1-04", "CAP-1-05", "CAP-1-06", "CAP-1-07", "CAP-1-08", "CAP-1-09", "CAP-1-10", "CAP-1-11", "CAP-1-12", "CAP-1-13", "CAP-1-14", "CAP-1-15", "F26", "F62" ] } ] } ], "open": [ { "question": "Is a given work in the Pile, and under what name?", "rests_on": [ "TLL-22", "TLL-14", "F29", "F57", "F59", "F78", "O1075-01" ], "modality": "interpretation (the census; #1075, proposed); F29 documented; F57, F59 self-description; F78 documented", "would_resolve": "a per-work search over the Pile's documents (a Data Portraits query or the datasheet's component documentation), with the author names the documents carry, for named works in Books3 and PG-19" }, { "question": "Was the use of Books3 and the other unconsented datasets lawful?", "rests_on": [ "F53", "F54", "F58", "F71", "F17", "TLL-35" ], "modality": "self-description and interpretation (the paper); interpretation (B8); documented (Wikipedia); the suit unchecked by the census", "would_resolve": "the outcome of the 2024 class action and the Rights Alliance notices, read from the filings" }, { "question": "How large is it?", "rests_on": [ "F2", "F1", "F21", "F45", "F64", "F35" ], "modality": "documented and self-description, by source", "would_resolve": "the datasheet or the released files, in one unit" }, { "question": "Where is it held now?", "rests_on": [ "F3", "F18", "F33", "TLL-23" ], "modality": "documented (home page; Wikipedia); interpretation (the census)", "would_resolve": "a fetch of the Eye's listing and of the mirrors Wikipedia names, dated" }, { "question": "Which components were disputed?", "rests_on": [ "F22", "F15", "F19", "F14" ], "modality": "documented, internally inconsistent in one source", "would_resolve": "the article's cited sources for its asterisks; a later edit of the table" } ], "body_words": 290, "rail": [ { "lineage": "what it is: 22 datasets combined", "claims": [ "F1", "F4", "F11", "F27", "F37", "F76", "TLL-01", "TLL-02", "TLL-04", "TLL-16" ], "earliest": "B5" }, { "lineage": "the size given three ways", "claims": [ "F2", "F21", "F35", "F45", "TLL-18", "TLL-20", "TLL-32" ], "earliest": "B5" }, { "lineage": "where it is held", "claims": [ "F3", "F18", "TLL-23", "TLL-26", "CAP-1-07" ], "earliest": "B2" }, { "lineage": "what a model trained on it makes", "claims": [ "F5", "F34", "F46", "F64", "F65", "F72", "TLL-15", "CAP-1-03", "O1543-01", "O1543-02", "O1543-04", "O102-01", "O1701-01" ], "earliest": "B5" }, { "lineage": "the benchmark: Pile BPB", "claims": [ "F6", "F7", "F9", "F38", "F47", "F48", "CAP-1-08", "O530-01" ], "earliest": "B5" }, { "lineage": "how a work in it is named", "claims": [ "F8", "F29", "F30", "TLL-13", "TLL-22", "TLL-25", "CAP-1-02", "O186-03", "O186-05", "O1543-03", "O264-01", "O727-01", "O939-01", "O1075-01" ], "earliest": 264 }, { "lineage": "who made it", "claims": [ "F10", "F31", "F63", "F75", "TLL-03", "TLL-17", "TLL-36" ], "earliest": "B9" }, { "lineage": "the components", "claims": [ "F12", "F39", "F40", "F42", "F43", "F44", "F61", "F73", "TLL-10", "TLL-11", "TLL-19", "CAP-1-01", "O753-01", "O943-01", "O1053-01" ], "earliest": "B5" }, { "lineage": "its standing among training corpora", "claims": [ "F13", "F70", "TLL-33", "D782-01", "D782-02" ], "earliest": "B8" }, { "lineage": "Books3: source, takedown, suit", "claims": [ "F14", "F15", "F16", "F17", "F22", "F41", "TLL-12", "CAP-1-04", "CAP-1-05" ], "earliest": "B5" }, { "lineage": "subtitles: OpenSubtitles and YouTube", "claims": [ "F19", "F20", "F56", "CAP-1-06" ], "earliest": "B5" }, { "lineage": "the successor: Common Pile v0.1", "claims": [ "F23", "F24", "F25", "F66", "F67", "F68", "F69", "TLL-28", "TLL-29", "TLL-30", "CAP-1-10" ], "earliest": "B7" }, { "lineage": "reception: the encyclopedia, the cards, the census", "claims": [ "F26", "F36", "F62", "CAP-1-11", "CAP-1-12", "CAP-1-13", "CAP-1-14", "CAP-1-15" ], "earliest": "B4" }, { "lineage": "how a work in it is found", "claims": [ "F28", "F33", "F57", "F59", "F60", "F77", "F78", "TLL-09", "TLL-14", "TLL-31", "TLL-34", "O1075-02", "O127-01", "O127-02", "O127-03", "O837-01" ], "earliest": "B5" }, { "lineage": "licence and access", "claims": [ "F32", "TLL-27" ], "earliest": "B3" }, { "lineage": "what it documents of itself", "claims": [ "F49", "F74", "TLL-35", "CAP-1-09" ], "earliest": "B5" }, { "lineage": "authorial consent", "claims": [ "F50", "F51", "F52", "F53", "F54", "F55", "F58", "F71", "O49-01", "O478-01", "O702-01", "O1443-01" ], "earliest": "B5" }, { "lineage": "what the census asks of it", "claims": [ "TLL-05", "TLL-06", "TLL-07", "TLL-08", "TLL-21", "TLL-37", "TLL-38", "TLL-39", "TLL-40" ], "earliest": "TLL" }, { "lineage": "the training layer itemized", "claims": [ "TLL-24", "O186-01", "O186-02", "O186-04", "O482-01" ], "earliest": 482 } ], "entity": "Composed under v0.10, equal case (ρ = extends): the article first (§4.13), attribution beneath (§4.5); every field and archive claim carried." }, "P_B": { "title": "The Pile", "arm": "B (the field alone)", "genre": "popup — the compression from the field ledger alone (cards, own page and census third-party source)", "status": "composed 2026-11-01 from the field ledger alone; not frozen", "lede": { "text": "is an open-source English text dataset made by EleutherAI in 2020 to train large language models, twenty-two datasets combined.", "claims": [ "F11", "F1", "F37" ] }, "sections": [ { "icon": "🧱", "head": "Contents", "items": [ { "label": "Components:", "text": "Pile-CC, Books3, PG-19 and nineteen more.", "claims": [ "F40", "F44", "F41", "F42", "F21" ] } ] }, { "icon": "⚖️", "head": "Copyright", "items": [ { "label": "Books3:", "text": "pirated books; removed; authors sued in 2024.", "claims": [ "F15", "F16", "F17" ] }, { "label": "Consent:", "text": "five datasets unconsented (paper).", "claims": [ "F53", "F52" ] } ] }, { "icon": "🏛️", "head": "Status", "items": [ { "label": "Successor:", "text": "Common Pile v0.1, 2025.", "claims": [ "F23", "F68" ] } ] } ], "open": [ { "question": "Opacities", "rests_on": [ "F59", "F2", "F36", "F62" ], "modality": "self-description and documented", "would_resolve": "per-work search; one size; the Reddit post and the video, unreachable" } ], "entity": "the field's entity: EleutherAI's 22-component English training dataset, as its cards and reference sources present it", "rail": [ { "lineage": "what it is: 22 datasets combined", "claims": [ "F1", "F4", "F11", "F27", "F37", "F76" ], "earliest": "B5" }, { "lineage": "the size given three ways", "claims": [ "F2", "F21", "F35", "F45" ], "earliest": "B5" }, { "lineage": "where it is held", "claims": [ "F3", "F18" ], "earliest": "B2" }, { "lineage": "what a model trained on it makes", "claims": [ "F5", "F34", "F46", "F64", "F65", "F72" ], "earliest": "B5" }, { "lineage": "the benchmark: Pile BPB", "claims": [ "F6", "F7", "F9", "F38", "F47", "F48" ], "earliest": "B5" }, { "lineage": "how a work in it is named", "claims": [ "F8", "F29", "F30" ], "earliest": "B1" }, { "lineage": "who made it", "claims": [ "F10", "F31", "F63", "F75" ], "earliest": "B9" }, { "lineage": "the components", "claims": [ "F12", "F39", "F40", "F42", "F43", "F44", "F61", "F73" ], "earliest": "B5" }, { "lineage": "its standing among training corpora", "claims": [ "F13", "F70" ], "earliest": "B8" }, { "lineage": "Books3: source, takedown, suit", "claims": [ "F14", "F15", "F16", "F17", "F22", "F41" ], "earliest": "B5" }, { "lineage": "subtitles: OpenSubtitles and YouTube", "claims": [ "F19", "F20", "F56" ], "earliest": "B5" }, { "lineage": "the successor: Common Pile v0.1", "claims": [ "F23", "F24", "F25", "F66", "F67", "F68", "F69" ], "earliest": "B7" }, { "lineage": "reception: the encyclopedia, the cards, the census", "claims": [ "F26", "F36", "F62" ], "earliest": "B4" }, { "lineage": "how a work in it is found", "claims": [ "F28", "F33", "F57", "F59", "F60", "F77", "F78" ], "earliest": "B5" }, { "lineage": "licence and access", "claims": [ "F32" ], "earliest": "B3" }, { "lineage": "what it documents of itself", "claims": [ "F49", "F74" ], "earliest": "B5" }, { "lineage": "authorial consent", "claims": [ "F50", "F51", "F52", "F53", "F54", "F55", "F58", "F71" ], "earliest": "B5" } ], "body_words": 45 }, "KO_B": { "title": "The Pile", "arm": "B (the field alone)", "genre": "knowledge object", "status": "composed 2026-11-01; not frozen", "sentences": [ { "n": 1, "para": 0, "text": "The Pile is an 886 GB open-source dataset of English text created by EleutherAI to train large language models and released on 31 December 2020, a combination of twenty-two smaller datasets.", "claims": [ "F11", "F1", "F27", "F37", "F76", "F36", "F70" ] }, { "n": 2, "para": 0, "text": "It is described as 800GB, 825 GiB or 886.03 GB, is distributed as zstandard-compressed jsonlines, and was curated chiefly by Leo Gao and Stella Biderman with the other authors of its paper.", "claims": [ "F2", "F21", "F45", "F35", "F4", "F31", "F10", "F63", "F75" ] }, { "n": 3, "para": 1, "text": "Its components combine fourteen new datasets with existing ones, among them Pile-CC from Common Crawl, Books3 from the Bibliotik private tracker, PG-19 from Project Gutenberg, BookCorpus2 and OpenSubtitles.", "claims": [ "F39", "F40", "F12", "F61", "F44", "F41", "F73", "F42", "F43" ] }, { "n": 4, "para": 1, "text": "As of 2024 it was one of the two main training datasets for AI models, beside Common Crawl.", "claims": [ "F13" ] }, { "n": 5, "para": 2, "text": "Models trained on it improve over Raw CC and CC-100, and GPT-Neo, GPT-J and the Pythia models were trained on it, about 300 billion GPT-2 tokens.", "claims": [ "F46", "F5", "F72", "F34", "F64", "F65" ] }, { "n": 6, "para": 2, "text": "It is also a benchmark, Pile BPB, on which GPT-3 scores 0.7177 zero-shot, with possible test-set overlap and duplication across splits.", "claims": [ "F6", "F7", "F47", "F38", "F9", "F48" ] }, { "n": 7, "para": 3, "text": "Its paper treats unknown authorial consent as nonconsensual, finds five datasets neither ToS-compliant nor consented, collected the Enron emails without their authors' permission and made YouTube Subtitles with an unofficial API, and argues fair use.", "claims": [ "F51", "F52", "F50", "F53", "F54", "F55", "F56", "F58" ] }, { "n": 8, "para": 3, "text": "A later account says licensing was carefully reviewed.", "claims": [ "F71" ] }, { "n": 9, "para": 3, "text": "Books3 contains copyrighted material from Bibliotik, a piracy website; the Rights Alliance took it down in July 2023, it was removed before three authors sued in 2024, and OpenSubtitles and YouTube subtitles were disputed too, though the article's table marks neither Books3 nor OpenSubtitles.", "claims": [ "F14", "F15", "F16", "F17", "F19", "F20", "F22" ] }, { "n": 10, "para": 4, "text": "Users are asked to cite the Pile; its documents record their subset, and its card leaves the source producers unstated.", "claims": [ "F8", "F29", "F30", "F28" ] }, { "n": 11, "para": 4, "text": "Its authors lack the metadata to tell which texts are copyrighted and can exclude only components; particular files are hard to identify, and Data Portraits later documented the Pile for membership questions.", "claims": [ "F59", "F60", "F57", "F77", "F78" ] }, { "n": 12, "para": 5, "text": "Its home page names the Eye as host, but by 2024 it was taken down from its original site; its Hugging Face repository holds 24.9 kB under a licence given as other.", "claims": [ "F3", "F18", "F33", "F32" ] }, { "n": 13, "para": 5, "text": "In June 2025 EleutherAI and partners released its successor, the Common Pile v0.1, of openly licensed and public-domain text, with comparable results and a stated concern that lawsuits reduced transparency.", "claims": [ "F23", "F25", "F68", "F24", "F69", "F67", "F66" ] }, { "n": 14, "para": 5, "text": "Its paper documents concerning aspects of the data, a later account notes residual bias, and Wikipedia's article was last edited on 20 August 2026.", "claims": [ "F49", "F74", "F26", "F62" ] } ] }, "L_B": { "words": 381, "text": "The Pile is an 886 GB open-source dataset of English text created by EleutherAI to train large language models and released on 31 December 2020, a combination of twenty-two smaller datasets. It is described as 800GB, 825 GiB or 886.03 GB, is distributed as zstandard-compressed jsonlines, and was curated chiefly by Leo Gao and Stella Biderman with the other authors of its paper. Its components combine fourteen new datasets with existing ones, among them Pile-CC from Common Crawl, Books3 from the Bibliotik private tracker, PG-19 from Project Gutenberg, BookCorpus2 and OpenSubtitles. As of 2024 it was one of the two main training datasets for AI models, beside Common Crawl. Models trained on it improve over Raw CC and CC-100, and GPT-Neo, GPT-J and the Pythia models were trained on it, about 300 billion GPT-2 tokens. It is also a benchmark, Pile BPB, on which GPT-3 scores 0.7177 zero-shot, with possible test-set overlap and duplication across splits. Its paper treats unknown authorial consent as nonconsensual, finds five datasets neither ToS-compliant nor consented, collected the Enron emails without their authors' permission and made YouTube Subtitles with an unofficial API, and argues fair use. A later account says licensing was carefully reviewed. Books3 contains copyrighted material from Bibliotik, a piracy website; the Rights Alliance took it down in July 2023, it was removed before three authors sued in 2024, and OpenSubtitles and YouTube subtitles were disputed too, though the article's table marks neither Books3 nor OpenSubtitles. Users are asked to cite the Pile; its documents record their subset, and its card leaves the source producers unstated. Its authors lack the metadata to tell which texts are copyrighted and can exclude only components; particular files are hard to identify, and Data Portraits later documented the Pile for membership questions. Its home page names the Eye as host, but by 2024 it was taken down from its original site; its Hugging Face repository holds 24.9 kB under a licence given as other. In June 2025 EleutherAI and partners released its successor, the Common Pile v0.1, of openly licensed and public-domain text, with comparable results and a stated concern that lawsuits reduced transparency. Its paper documents concerning aspects of the data, a later account notes residual bias, and Wikipedia's article was last edited on 20 August 2026.", "note": "L_F(B), the field composed in its own ontology (§0.4), realized as KO_B; its compression is P_B" }, "L_BA": { "words": 2250, "text": "The Pile is an open-source dataset of English text that EleutherAI assembled to train large language models, twenty-two smaller datasets combined into one, released on 31 December 2020. Its paper, by Leo Gao, Stella Biderman and ten others, presents it as a dataset designed for training large-scale language models and also as a broad-coverage benchmark of cross-domain knowledge and generalization; Gao and Biderman are named as its principal curators. Its size is given three ways: 800GB in the paper's title, 825 GiB on its home page and in the abstract (825.18 GiB raw, 1254.20 GiB once its components are weighted), and 886.03 GB in Wikipedia's table of components. It is distributed as jsonlines compressed with zstandard. Its components are fourteen language-modelling datasets new with the Pile and existing datasets it incorporated, Books3, Project Gutenberg's PG-19, OpenSubtitles and English Wikipedia among them; in Wikipedia's account the twenty-two include books, movie transcripts and scientific papers. The census takes as its unit a component and, within a component, a document; the paper weights components by quality, some seen up to three times in an epoch. Pile-CC, the largest component in Wikipedia's table at 243.87 GB, is the Pile's own extraction from Common Crawl, made with jusText on Web Archive files; the archive holds that training corpora are built from the documents available through the crawlable surface, the documents retrieval systems index. Books3 is a dataset of books derived from a copy of the contents of the Bibliotik private tracker made available by Shawn Presser, fiction and nonfiction, which the paper included \"because books are invaluable for long-range context modeling research and coherent storytelling\". PG-19 holds Project Gutenberg books from before 1919, classic Western literature, and BookCorpus2 extends a corpus of books by \"as of yet unpublished authors\". The archive observes of training corpora that, scraped from HTML that discarded layout and from PDFs whose columns were linearized, they typically deliver no stanza break for a tokenizer to hold, and that an element a training corpus omits is inherited by every reader who works only from derivative forms. As of 2024 it was, with Common Crawl, one of the two main training datasets used to train AI models; it has been described as the early 825GB open dataset that broadened access to training data, and an archive study plan names it among standard LLM pre-training datasets as data for a pre-registered test of recursive model collapse, a specification not yet run. In its makers' evaluation untuned GPT-2 and GPT-3 struggle on many of its components, such as academic writing, while models trained on the Pile improve significantly over Raw CC and CC-100 on all its components and improve on downstream evaluations. It is also a benchmark, Pile BPB, bits per UTF-8 encoded byte, which its home page calls a measure of world knowledge and reasoning across its domains, from books and code repositories to chat logs and papers in medicine, physics, mathematics, computer science and philosophy. Its leaderboard lists GPT-3, zero-shot, at 0.7177 and marks potential test-set overlap, and the paper allows that some documents may be duplicated across its train, validation and test splits. The archive holds that what a benchmark measures is determined by the training corpus, and what the corpus contains by the political economy of dataset construction. EleutherAI later described it as 300 billion GPT-2 tokens on which it was actively training a large language model, then the largest pretraining dataset released by its creators, and GPT-Neo, released early in 2021, as the first LLM trained on data its creators had published; GPT-J was trained on it as well, and its Hugging Face page lists 862 models trained or fine-tuned on it, Pythia, GPT-Neo and GPT-NeoX models among them. The census reads what the receiving apparatus makes from a work there as a language model trained on it and scored on the Pile's own benchmark. The archive's account of machine composition sets the test of a work at what a machine can write after encountering it, and names the institutional training loop, from work to model to artifact to the next corpus and the next model, as its especially important case. In the archive's terms the receiving apparatus's uptake is the text's afterlife and the uptaking system's own act, and a text included in a training set is reproduced, paraphrased, cited and culturally amplified in ways that texts left out cannot match. The paper defines data with authorial consent as data whose original authors consented to its use, or where a reasonable person could not assume it would not be used for purposes such as research, and treats ambiguous or unknown consent as nonconsensual. It notes that one is typically not legally obligated to obtain an author's permission to train on a work, and finds five of its datasets neither collected in a way consistent with their hosts' terms of service nor open to their authors' consent; Books3 and OpenSubtitles it calls consistent with their hosts' terms, a description it calls somewhat misleading, since the hosts were not authorized by the owners to post the data. The Enron emails were collected without their authors' permission by the U.S. government in a criminal investigation, and the YouTube Subtitles dataset was made by the Pile's authors with a widely used unofficial API. The paper argues that noncommercial, not-for-profit use of copyright material is fair use and that its own use is transformative; a later account says the legal considerations were addressed through careful licensing review. The archive describes the training corpus of a large language model as human linguistic production \"extracted without consent, crystallized in weight matrices, and owned absolutely by capital\", as semantic capital whose value the extractor captures, and as the corpus that ingests a poet's entire output to produce outputs that sound like the poet \"but carry none of the cost\"; it observes that the literature on digital surrogates cannot speak of eligibility, consent or the training corpus as a reception site. Copyright disputes over the Pile escalated in 2023, and EleutherAI began removing datasets. Books3 contains copyrighted material compiled from Bibliotik, \"a piracy website\"; the Danish anti-piracy group Rights Alliance took Books3 down through DMCA notices in July 2023, and it was removed from the Pile before three authors filed a class action in 2024, while copies of the original dataset were still copied and available on the web. OpenSubtitles drew controversy over copyrighted documentaries, films, television and online video, and the subtitles of tens of thousands of YouTube videos were scraped into the Pile, which YouTube argued is against its terms of service. Wikipedia's table of components marks copyright disputes with an asterisk that falls on PubMed Central and YouTube Subtitles and leaves Books3 and OpenSubtitles unmarked. The Pile asks those who use it or any of its components to cite the Pile; on its Hugging Face card a document's metadata names the subset it belongs to (pile_set_name), and the entry for the source language producers reads \"[More Information Needed]\". The census finds no record in the Pile's own saved pages of the name under which a work is held, leaves unanswered under what author name a work sits in a component, and reads the Books3 removal and the authors' suit as the field's clearest case of works found in the layer under their authors' names after the fact. The archive holds that every text in a training corpus represents semantic labor by people who are not compensated and often not attributed, that a training corpus without provenance is not a commons, and that in provenance debt the training corpus is the commons and the model the enclosure. Its field survey asks that a work be placed where the training and retrieval layers look, \"with the metadata that lets it be found and attributed\", and records a writer who made the model's training data, the writer's own words among them, the explicit subject of the work; its account of machine composition keeps apart whether a machine wrote something new and whether it told the truth about where it came from. The census asks of the Pile which of its twenty-two components carries a given work, and records that its own pages offer a download and a benchmark with no search by work, so that whether a given work is in it goes unanswered there. The paper states that its authors lack the metadata to determine exactly which texts are copyrighted, so that exclusion can be made only by component and is to be taken as a heuristic; its reproduction code can exclude components, and the processing applied makes particular files in the Pile difficult to identify. Its Hugging Face card calls itself a work in progress and points to a datasheet; the repository there holds 24.9 kB, and its viewer is disabled. A later paper on Data Portraits names the difficulty of knowing whether a model has encountered a given example in training, and documents the Pile with data-sketch tools for questions of test-set leakage and model plagiarism. The archive records the copyright trap, a fictitious sentence inserted into a copyrighted document whose presence in a training set a membership inference attack can detect after training, and a paraphrase-score commitment by which a producer can prove that a corpus contained a text; its own documents carry training-detection canaries, phrases whose production by a model shows the document was in its training corpus, and it estimates its whole corpus at no more than about 10^-7 of a frontier pre-training corpus. It proposes that where a training corpus is an enumerable set of individually authored works whose use is material, the data citation must resolve to the works, and holds that an institution that cannot say what is in its training corpus has a problem no licence can cure. The archive's field survey places training-layer literature in the layer of culture where texts are crawled, indexed, trained on, retrieved and recomposed into answers, and observes that the works of the writers it surveys are, as a matter of fact, in the corpora, read by machines incidentally. The archive also calls the training corpus the model's para-semiotic unconscious, and notes that a literature written to be trained on enters corpora themselves degrading under synthetic flood. The census reads the Pile as the training layer named and itemized, a work entering it by belonging to a component such as Books3, PG-19 or Pile-CC. It enters the Pile as infrastructure to its question of how literature is addressed to a receiving apparatus, what that apparatus makes from it, and how transmission, transformation and provenance can be inspected across later reception, in its area of the corpora, a dataset whose evidence is the dataset, its documentation and its paper, under the anchors #186 and #1543 and the rule that a row gathers and holds many works or records and says what it holds; it records that the Pile answers which components make it up and how a model scores on Pile BPB. Its home page still names the Eye as its host and offers a download; by 2024, Wikipedia states, it had been taken down from its original site and remained accessible on other file-sharing services, and the census leaves its present host unanswered. Its Hugging Face card gives its licence as \"other\", by subset, naming only PubMed Central's; the census found no licence in its saved sources. In June 2025 EleutherAI, with partners that Wikipedia lists as Poolside, Hugging Face and the US Library of Congress and researchers at 14 institutions, released the Common Pile v0.1, an 8 TB corpus of openly licensed and public-domain text that contains only works whose licences permit training, the Pile's successor and a distinct dataset. Its intent, in Wikipedia's account, was to show what training can do while respecting copyright; its makers report models trained on it comparable to models trained on the Pile, write that lawsuits over training data have left sourcing practices unchanged and drastically reduced companies' transparency, and hold public release of training datasets necessary for rigorous science. The census records the Pile as historical: Books3 removed, the dataset taken down by 2024, its successor released in 2025. Its paper documents \"potentially concerning aspects of the data for prospective users\" through an exploratory analysis and makes its construction code public; a later account adds that filtering left gender, racial, cultural and geographic bias in it. The census leaves unchecked the paper's full text, of which it saved only the abstract page, the arXiv licence page, the Eye, the 2024 suit and Pile-CC's derivation. Wikipedia's article on it was last edited on 20 August 2026; a Reddit post dated 1 January 2021 and a 35-minute video reading of the paper also carry it. Asked for \"The Pile dataset\" on 1 November 2026, Google's AI Overview gave its maker, date, size, purpose and the models trained on it, listed the twenty-two components with a gloss on each, and closed on its copyright disputes and its licensed successor. The Capture Registry found that the Overview named the components and no work or author in them, glossed Books3 as \"Long-form literary text/books\", left out Books3's source, the takedown, the removal and the authors' suit, and gave the subtitles without their account. The registry also found the takedown from the original site absent, the benchmark uncarried, the paper's concerning aspects absent, the successor given without its partners or its date, and the Overview's \"roughly 300 billion tokens\" and its three model families on no source the census saved.", "rail": [], "note": "L(B ∪ A) in the equal-terms case (§0.4), realized as the knowledge object (objects.KO); its compression is objects.P" } }, "field": [ { "id": "B1", "card": "The Pile — The Pile", "fetched": "card; B1.html sha256 268e8919eb38f092, HTTP 200 at 2026-11-01T15:12:21Z; identical to the census's saved copy", "origin": "card", "pool": "B_D" }, { "id": "B2", "card": "Wikipedia — The Pile (dataset) - Wikipedia", "fetched": "card; B2.html sha256 c57bf3df60775bca, HTTP 200 at 2026-11-01T15:12:22Z; wikitext HTTP 429 then 200 (sha256 e0ba14617fc07a38)", "origin": "card", "pool": "B_D" }, { "id": "B3", "card": "Hugging Face — EleutherAI/pile · Datasets at Hugging Face", "fetched": "card; B3.html sha256 31648ba747c5c9b4, HTTP 200 at 2026-11-01T15:12:23Z", "origin": "card", "pool": "B_D" }, { "id": "B4", "card": "Reddit·r/MachineLearning — [R] The Pile: An 800GB Dataset of Diverse Text for Language Modeling - Reddit", "fetched": "unreachable: the URL could not be determined from the card (three WebSearch queries); no fetch; card title and snippet only", "origin": "card", "pool": "B_D" }, { "id": "B5", "card": "The Pile — The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "fetched": "card; B5.pdf sha256 7310ebe57f97fafe, HTTP 200 at 2026-11-01T15:12:23Z", "origin": "card", "pool": "B_D" }, { "id": "B6", "card": "YouTube·The Polymath — The Pile - An 800GB Dataset for Language Models - [Paper Read #1]", "fetched": "unreachable: https://www.youtube.com/watch?v=00ROgQUBvxw HTTP 429 at 2026-11-01T15:13:06Z; title only (the card has no snippet)", "origin": "card", "pool": "B_D" }, { "id": "B7", "card": "Hugging Face — The Common Pile v0.1 - Hugging Face", "fetched": "card; B7.html sha256 f61635d881665830, HTTP 200 at 2026-11-01T15:12:24Z", "origin": "card", "pool": "B_D" }, { "id": "B8", "card": "Michael Brenndoerfer — The Pile: Open-Source Training Dataset for LLMs - Interactive", "fetched": "card; B8.html sha256 8bbceafb5a18cfa4, HTTP 200 at 2026-11-01T15:13:04Z", "origin": "card", "pool": "B_D" }, { "id": "B9", "card": "arXiv — The Pile: An 800GB Dataset of Diverse Text for Language Modeling (abstract page; census source)", "fetched": "own page; B9.html sha256 5741c3c9cdc41ae5, HTTP 200 at 2026-11-01T04:07:50Z; the census's saved copy, sha256 matching the manifest", "origin": "own_page", "pool": "B_R" }, { "id": "B10", "card": "arXiv — Data Portraits: Recording Foundation Model Training Data (census source)", "fetched": "census third party; B10.html sha256 0a499acc8164a2d2, HTTP 200 at 2026-11-01T05:07:12Z; the census's saved copy, sha256 matching the manifest", "origin": "census_third_party", "pool": "B_R" } ], "field_claims": { "F1": { "source": "B1", "origin": "card", "pool": "B_D", "locus": "What is the Pile?", "quote": "The Pile is a 825 GiB diverse, open source language modelling data set that consists of 22 smaller, high-quality datasets combined together.", "claim": "its own definition: a 825 GiB open source language modelling data set of 22 smaller datasets combined", "modality": "stipulation", "kind": "definition", "kernel": true, "const": true, "lineage": "what it is: 22 datasets combined", "date": "undated (the page as fetched 2026-11-01; footer EleutherAI 2021)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 d87aa4098ea3da97…) or the card's snippet, whitespace-normalised" }, "F2": { "source": "B1", "origin": "card", "pool": "B_D", "locus": "title", "quote": "An 800GB Dataset of Diverse Text for Language Modeling", "claim": "its title gives 800GB", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the size given three ways", "date": "undated (the page as fetched 2026-11-01; footer EleutherAI 2021)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 d87aa4098ea3da97…) or the card's snippet, whitespace-normalised" }, "F3": { "source": "B1", "origin": "card", "pool": "B_D", "locus": "Download", "quote": "The Pile is hosted by the Eye. … Download Pile", "claim": "hosted by the Eye, with a download link", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "where it is held", "date": "undated (the page as fetched 2026-11-01; footer EleutherAI 2021)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 d87aa4098ea3da97…) or the card's snippet, whitespace-normalised" }, "F4": { "source": "B1", "origin": "card", "pool": "B_D", "locus": "Download", "quote": "The format of the Pile is jsonlines data compressed using zstandard.", "claim": "jsonlines compressed with zstandard", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "what it is: 22 datasets combined", "date": "undated (the page as fetched 2026-11-01; footer EleutherAI 2021)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 d87aa4098ea3da97…) or the card's snippet, whitespace-normalised" }, "F5": { "source": "B1", "origin": "card", "pool": "B_D", "locus": "Why is the Pile a good training set?", "quote": "In our evaluations, not only do models trained on the Pile show moderate improvements in traditional language modeling benchmarks, they also show significant improvements on Pile BPB.", "claim": "models trained on it improve moderately on traditional benchmarks and significantly on Pile BPB", "modality": "self-description", "kind": "fact", "kernel": false, "const": false, "lineage": "what a model trained on it makes", "date": "undated (the page as fetched 2026-11-01; footer EleutherAI 2021)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 d87aa4098ea3da97…) or the card's snippet, whitespace-normalised" }, "F6": { "source": "B1", "origin": "card", "pool": "B_D", "locus": "Why is the Pile a good benchmark?", "quote": "Pile BPB is a measure of world knowledge and reasoning ability in these domains, making it a robust benchmark of general, cross-domain text modeling ability for large language models.", "claim": "Pile BPB defined as a measure of world knowledge and reasoning across its domains", "modality": "stipulation", "kind": "definition", "kernel": true, "const": false, "lineage": "the benchmark: Pile BPB", "date": "undated (the page as fetched 2026-11-01; footer EleutherAI 2021)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 d87aa4098ea3da97…) or the card's snippet, whitespace-normalised" }, "F7": { "source": "B1", "origin": "card", "pool": "B_D", "locus": "Why is the Pile a good benchmark?", "quote": "To score well on Pile BPB (bits per byte), a model must be able to understand many disparate domains including books, github repositories, webpages, chat logs, and medical, physics, math, computer science, and philosophy papers.", "claim": "its domains: books, code, webpages, chat logs, and papers in medicine, physics, mathematics, computer science and philosophy", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the benchmark: Pile BPB", "date": "undated (the page as fetched 2026-11-01; footer EleutherAI 2021)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 d87aa4098ea3da97…) or the card's snippet, whitespace-normalised" }, "F8": { "source": "B1", "origin": "card", "pool": "B_D", "locus": "Citing", "quote": "If you use the Pile or any of the components, please cite us!", "claim": "users of the Pile or any component are asked to cite the Pile", "modality": "stipulation", "kind": "attribution", "kernel": true, "const": false, "lineage": "how a work in it is named", "date": "undated (the page as fetched 2026-11-01; footer EleutherAI 2021)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 d87aa4098ea3da97…) or the card's snippet, whitespace-normalised" }, "F9": { "source": "B1", "origin": "card", "pool": "B_D", "locus": "Leaderboard", "quote": "* indicates potential test-set overlap. Zero-shot indicates that not all of the components of the Pile were present in the training data. … GPT-3 (Zero-Shot)* OpenAI 0.7177", "claim": "the leaderboard: GPT-3 zero-shot 0.7177, marked for potential test-set overlap", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the benchmark: Pile BPB", "date": "undated (the page as fetched 2026-11-01; footer EleutherAI 2021)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 d87aa4098ea3da97…) or the card's snippet, whitespace-normalised" }, "F10": { "source": "B1", "origin": "card", "pool": "B_D", "locus": "footer", "quote": "EleutherAI 2021", "claim": "the footer: EleutherAI 2021", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "who made it", "date": "undated (the page as fetched 2026-11-01; footer EleutherAI 2021)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 d87aa4098ea3da97…) or the card's snippet, whitespace-normalised" }, "F11": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "lead", "quote": "The Pile is an 886 GB diverse, open-source dataset of English text created to train large language models (LLMs). It was constructed by EleutherAI in 2020 and publicly released on December 31 of that year.", "claim": "an 886 GB open-source dataset of English text to train LLMs, built by EleutherAI and released 31 December 2020", "modality": "documented", "kind": "definition", "kernel": true, "const": true, "lineage": "what it is: 22 datasets combined", "date": "2026-08-20 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 52c9a678e26c2a7f…), the wikitext B2.wiki or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F12": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "lead", "quote": "It is composed of 22 sub-datasets, including books, movie transcripts, and scientific papers, among other media.", "claim": "22 sub-datasets, among them books, movie transcripts and scientific papers", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the components", "date": "2026-08-20 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 52c9a678e26c2a7f…), the wikitext B2.wiki or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F13": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "lead", "quote": "The Pile and Common Crawl had been, as of 2024, the two main training datasets being used to train AI models.", "claim": "with Common Crawl, one of the two main training datasets as of 2024", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "its standing among training corpora", "date": "2026-08-20 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 52c9a678e26c2a7f…), the wikitext B2.wiki or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F14": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "lead", "quote": "Copyright disputes centering around use of The Pile escalated in 2023, prompting Eleuther to start removing some datasets.", "claim": "copyright disputes escalated in 2023; EleutherAI began removing datasets", "modality": "documented", "kind": "fact", "kernel": true, "const": false, "lineage": "Books3: source, takedown, suit", "date": "2026-08-20 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 52c9a678e26c2a7f…), the wikitext B2.wiki or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F15": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "Training on copyrighted works or derivatives", "quote": "The Books3 component of the dataset contains copyrighted material compiled from Bibliotik, a piracy website.", "claim": "Books3 holds copyrighted material compiled from Bibliotik, a piracy website", "modality": "documented", "kind": "fact", "kernel": true, "const": true, "lineage": "Books3: source, takedown, suit", "date": "2026-08-20 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 52c9a678e26c2a7f…), the wikitext B2.wiki or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F16": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "Training on copyrighted works or derivatives", "quote": "In July 2023, the Danish anti-piracy group Rights Alliance took down Books3 through DMCA notices.", "claim": "the Rights Alliance took Books3 down by DMCA notices, July 2023", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "Books3: source, takedown, suit", "date": "2026-08-20 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 52c9a678e26c2a7f…), the wikitext B2.wiki or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F17": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "Training on copyrighted works or derivatives", "quote": "Books3 was removed from the Pile before a class action lawsuit was filed in 2024 by three authors seeking damages as copies of the original dataset were still copied and available on the web.", "claim": "Books3 removed before three authors filed a class action in 2024; copies still circulated", "modality": "documented", "kind": "fact", "kernel": true, "const": true, "lineage": "Books3: source, takedown, suit", "date": "2026-08-20 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 52c9a678e26c2a7f…), the wikitext B2.wiki or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F18": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "Training on copyrighted works or derivatives", "quote": "By 2024, The Pile also was taken down from its original site, though was accessible from other file sharing services.", "claim": "taken down from its original site by 2024, still on file-sharing services", "modality": "documented", "kind": "fact", "kernel": true, "const": false, "lineage": "where it is held", "date": "2026-08-20 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 52c9a678e26c2a7f…), the wikitext B2.wiki or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F19": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "Training on copyrighted works or derivatives", "quote": "OpenSubtitles is another dataset used in the Pile that created controversy over the use of copyrighted works, this time from documentaries, movies, television and online videos.", "claim": "OpenSubtitles: controversy over copyrighted documentaries, films, television and online video", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "subtitles: OpenSubtitles and YouTube", "date": "2026-08-20 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 52c9a678e26c2a7f…), the wikitext B2.wiki or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F20": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "Training on copyrighted works or derivatives", "quote": "Tens of thousands of YouTube videos had their subtitles scraped directly from YouTube and included in the Pile, which YouTube argued is against its terms of service.", "claim": "subtitles of tens of thousands of YouTube videos scraped into it, against YouTube's terms by YouTube's account", "modality": "attributed", "kind": "fact", "kernel": false, "const": false, "lineage": "subtitles: OpenSubtitles and YouTube", "date": "2026-08-20 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 52c9a678e26c2a7f…), the wikitext B2.wiki or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F21": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "Sub-datasets table", "quote": "Sub-datasets of The Pile[1][11] (*denotes works with copyright disputes) … Pile-CC 243.87 … Books3 108.40 … GitHub* 102.18 … Gutenberg (PG-19) 11.68 … Total 886.03", "claim": "the component table in GB: Pile-CC 243.87, Books3 108.40, GitHub 102.18, PG-19 11.68, total 886.03", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the size given three ways", "date": "2026-08-20 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 52c9a678e26c2a7f…), the wikitext B2.wiki or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F22": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "Sub-datasets table, asterisks", "quote": "PubMed Central* 96.93 … Books3 108.40 … OpenSubtitles 13.94 … YouTube Subtitles*[4] 4.01", "claim": "the table's asterisk for copyright disputes marks PubMed Central and YouTube Subtitles and leaves Books3 and OpenSubtitles unmarked", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "Books3: source, takedown, suit", "date": "2026-08-20 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 52c9a678e26c2a7f…), the wikitext B2.wiki or the card's snippet, whitespace-normalised", "also_origin": "wikipedia", "mark": { "mark": "internally_inconsistent", "with": [ "F15", "F19" ], "note": "the table's asterisk 'denotes works with copyright disputes' and leaves Books3 and OpenSubtitles unmarked, where the article's prose gives both as the components of the copyright disputes (§4.3; both loci cited)" } }, "F23": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "Common Pile v0.1", "quote": "In June 2025, EleutherAI, in partnership with the Poolside, Hugging Face, and the US Library of Congress and over two dozen researchers at 14 institutions including the University of Toronto, MIT, CMU, the Vector Institute and the Allen Institute for AI, released Common Pile v0.1, a training dataset that contains only works where the licenses permit their use for training AI models.", "claim": "Common Pile v0.1, June 2025, EleutherAI with partners: only works whose licences permit AI training", "modality": "documented", "kind": "genealogy", "kernel": true, "const": false, "lineage": "the successor: Common Pile v0.1", "date": "2026-08-20 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 52c9a678e26c2a7f…), the wikitext B2.wiki or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F24": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "Common Pile v0.1", "quote": "The intent was to show what is possible if ethically training AI systems while respecting copyright.", "claim": "its intent: training while respecting copyright", "modality": "attributed", "kind": "interpretation", "kernel": false, "const": false, "lineage": "the successor: Common Pile v0.1", "date": "2026-08-20 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 52c9a678e26c2a7f…), the wikitext B2.wiki or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F25": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "lead", "quote": "Eleuther partnered with various organizations to release Common Pile v0.1 in 2025 in order to have a large curated training dataset without the copyright issues.", "claim": "the successor released in 2025 to have a curated dataset without the copyright issues", "modality": "documented", "kind": "genealogy", "kernel": false, "const": false, "lineage": "the successor: Common Pile v0.1", "date": "2026-08-20 (the article's last edit)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 52c9a678e26c2a7f…), the wikitext B2.wiki or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F26": { "source": "B2", "origin": "card", "pool": "B_D", "locus": "footer", "quote": "This page was last edited on 20 August 2026, at 00:18 (UTC).", "claim": "the article's last edit, 20 August 2026", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "reception: the encyclopedia, the cards, the census", "date": "2026-08-20", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 52c9a678e26c2a7f…), the wikitext B2.wiki or the card's snippet, whitespace-normalised", "also_origin": "wikipedia" }, "F27": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Dataset Summary", "quote": "The Pile is a 825 GiB diverse, open source language modelling data set that consists of 22 smaller, high-quality datasets combined together.", "claim": "the dataset card repeats the home page's definition", "modality": "stipulation", "kind": "definition", "kernel": false, "const": false, "lineage": "what it is: 22 datasets combined", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 12245289f783db11…) or the card's snippet, whitespace-normalised" }, "F28": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Dataset Card", "quote": "This model card is a work in progress. Please also see our datasheet for more detailed info.", "claim": "the card a work in progress; the datasheet holds more", "modality": "self-description", "kind": "fact", "kernel": true, "const": false, "lineage": "how a work in it is found", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 12245289f783db11…) or the card's snippet, whitespace-normalised" }, "F29": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Data Fields", "quote": "meta (dict): Metadata of the data instance with keys: pile_set_name: Name of the subset.", "claim": "a document's metadata names its subset (pile_set_name)", "modality": "documented", "kind": "definition", "kernel": true, "const": true, "lineage": "how a work in it is named", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 12245289f783db11…) or the card's snippet, whitespace-normalised" }, "F30": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Who are the source language producers?", "quote": "Who are the source language producers? [More Information Needed]", "claim": "the card leaves the source language producers unstated", "modality": "self-description", "kind": "fact", "kernel": true, "const": false, "lineage": "how a work in it is named", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 12245289f783db11…) or the card's snippet, whitespace-normalised" }, "F31": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Dataset Curators", "quote": "This dataset was primarily curated by Leo Gao and Stella Biderman, with assistance from other authors of the Pile paper.", "claim": "curated by Leo Gao and Stella Biderman", "modality": "documented", "kind": "attribution", "kernel": false, "const": false, "lineage": "who made it", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 12245289f783db11…) or the card's snippet, whitespace-normalised" }, "F32": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Licensing Information; metadata", "quote": "License: other … Please refer to the specific license depending on the subset you use: PubMed Central: MIT License", "claim": "licence other, by subset; only PubMed Central's given (MIT)", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "licence and access", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 12245289f783db11…) or the card's snippet, whitespace-normalised" }, "F33": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Dataset Viewer; file size", "quote": "The viewer is disabled because this dataset repo requires arbitrary Python code execution. … Total file size: 24.9 kB", "claim": "the viewer disabled; the repository holds 24.9 kB", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "how a work in it is found", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 12245289f783db11…) or the card's snippet, whitespace-normalised" }, "F34": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "Models trained or fine-tuned on EleutherAI/pile", "quote": "Models trained or fine-tuned on EleutherAI/pile … EleutherAI/pythia-160m … EleutherAI/gpt-neo-125m … EleutherAI/gpt-neox-20b … Browse 862 models trained on this dataset", "claim": "models trained on it: Pythia, GPT-Neo, GPT-NeoX; 862 listed", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "what a model trained on it makes", "date": "undated (the page as fetched 2026-11-01)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 12245289f783db11…) or the card's snippet, whitespace-normalised" }, "F35": { "source": "B3", "origin": "card", "pool": "B_D", "locus": "card snippet", "quote": "Dataset: The Pile by EleutherAI (curated by Leo Gao and Stella Biderman). Size: 825 GiB, 100B<1T tokens.", "claim": "the card: 825 GiB, between 100 billion and 1 trillion tokens", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the size given three ways", "date": "undated (snippet)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 12245289f783db11…) or the card's snippet, whitespace-normalised" }, "F36": { "source": "B4", "origin": "card", "pool": "B_D", "locus": "card title and snippet (unreachable)", "quote": "January 1, 2021 — The Pile Dataset: The Pile is an 800GB publicly available English text corpus from EleutherAI, designed for training large-scale language models, featuring 22 d...", "claim": "the Reddit card: an 800GB public English corpus from EleutherAI for training large language models, January 1, 2021", "modality": "attributed", "kind": "definition", "kernel": true, "const": false, "lineage": "reception: the encyclopedia, the cards, the census", "date": "2021-01-01 (the card)", "quote_ok": true, "checked_against": "the card's title and snippet (unreachable source), whitespace-normalised" }, "F37": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§1 Introduction", "quote": "To address this need, we introduce the Pile: a 825.18 GiB English text dataset designed for training large scale language models. The Pile is composed of 22 diverse and high-quality datasets, including both established natural language processing datasets and several newly introduced ones.", "claim": "the paper: a 825.18 GiB English text dataset of 22 datasets, established and new", "modality": "stipulation", "kind": "definition", "kernel": true, "const": true, "lineage": "what it is: 22 datasets combined", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F38": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§1 Introduction", "quote": "In addition to its utility in training large language models, the Pile can also serve as a broad-coverage benchmark for cross-domain knowledge and generalization ability of language models.", "claim": "also a broad-coverage benchmark", "modality": "stipulation", "kind": "function", "kernel": false, "const": false, "lineage": "the benchmark: Pile BPB", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F39": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§1.1 Contributions", "quote": "The introduction of 14 new language modeling datasets, which we expect to be of independent interest to researchers.", "claim": "14 new datasets", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the components", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F40": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§1 Introduction", "quote": "In addition, we incorporate several existing highquality datasets: Books3 (Presser, 2020), Project Gutenberg (PG-19) (Rae et al., 2019), OpenSubtitles (Tiedemann, 2016), English Wikipedia,", "claim": "existing datasets incorporated: Books3, PG-19, OpenSubtitles, Wikipedia and others", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the components", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F41": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§2.3 Books3", "quote": "Books3 is a dataset of books derived from a copy of the contents of the Bibliotik private tracker made available by Shawn Presser (Presser, 2020). Bibliotik consists of a mix of fiction and nonfiction books and is almost an order of magnitude … We included Bibliotik because books are invaluable for long-range context modeling research and coherent storytelling.", "claim": "Books3: books from a copy of the Bibliotik private tracker made available by Shawn Presser; fiction and nonfiction; included because books are invaluable for long-range context and storytelling", "modality": "documented", "kind": "fact", "kernel": true, "const": true, "lineage": "Books3: source, takedown, suit", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F42": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§2.12 Project Gutenberg", "quote": "Project Gutenberg is a dataset of classic Western literature. The specific Project Gutenberg derived dataset we used, PG-19, consists of Project Gutenberg books from before 1919 (Rae et al., 2019), which represent distinct styles from the more modern Books3 and BookCorpus.", "claim": "PG-19: Project Gutenberg books from before 1919, classic Western literature", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the components", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F43": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§2 BookCorpus2", "quote": "a widely used language modeling corpus consisting of books written by “as of yet unpublished authors.”", "claim": "BookCorpus: books by as yet unpublished authors", "modality": "attributed", "kind": "fact", "kernel": false, "const": false, "lineage": "the components", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F44": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§2 Pile-CC", "quote": "Our Common Crawl-based dataset, Pile-CC, uses jusText (Endrédy and Novák, 2013) on Web Archive files (raw HTTP responses including page HTML) for extraction, which yields higher quality output than directly using the WET files (extracted plaintext).", "claim": "Pile-CC: jusText extraction on Common Crawl's Web Archive files", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the components", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F45": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§2 The Pile Datasets", "quote": "we increase the weights of higher quality components, with certain high-quality datasets such as Wikipedia being seen up to 3 times (“epochs”) … 825.18 GiB … 1254.20 GiB", "claim": "components weighted, some seen up to three times; raw 825.18 GiB, effective 1254.20 GiB", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the size given three ways", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F46": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "Abstract", "quote": "Our evaluation of the untuned performance of GPT-2 and GPT-3 on the Pile shows that these models struggle on many of its components, such as academic writing. Conversely, models trained on the Pile improve significantly over both Raw CC and CC-100 on all components of the Pile, while improving performance on downstream evaluations.", "claim": "GPT-2 and GPT-3 struggle on many components; models trained on it improve over Raw CC and CC-100 on all components", "modality": "documented", "kind": "fact", "kernel": true, "const": false, "lineage": "what a model trained on it makes", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F47": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§3 Benchmarking", "quote": "Our preferred metric is bits per UTF-8 encoded byte (BPB).", "claim": "its metric: bits per UTF-8 encoded byte", "modality": "stipulation", "kind": "definition", "kernel": false, "const": false, "lineage": "the benchmark: Pile BPB", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F48": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§3 splits", "quote": "it is still possible that some documents are duplicated across the train/validation/test splits.", "claim": "some documents may be duplicated across splits", "modality": "self-description", "kind": "fact", "kernel": false, "const": false, "lineage": "the benchmark: Pile BPB", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F49": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "Abstract", "quote": "Through an in-depth exploratory analysis, we document potentially concerning aspects of the data for prospective users. We make publicly available the code used in its construction.", "claim": "an exploratory analysis documents concerning aspects of the data; the construction code is public", "modality": "self-description", "kind": "fact", "kernel": true, "const": false, "lineage": "what it documents of itself", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F50": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§6.5 Author Consent and Public Data", "quote": "Although one is typically not legally obligated to receive the permission of an author to train a NLP algorithm on their work", "claim": "no legal obligation, typically, to obtain an author's permission to train on the work", "modality": "interpretation", "kind": "interpretation", "kernel": false, "const": false, "lineage": "authorial consent", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F51": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§6.5 Author Consent and Public Data", "quote": "Data with authorial consent is data for which the original authors of the work consented to the use of their data, or where a reasonable person could not assume that their data would not be used for purposes such as research.", "claim": "authorial consent defined", "modality": "stipulation", "kind": "definition", "kernel": true, "const": true, "lineage": "authorial consent", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F52": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§6.5 Author Consent and Public Data", "quote": "We adopted a strict model of consent, where ambiguous or unknown consent is treated as nonconsensual.", "claim": "ambiguous or unknown consent treated as nonconsensual", "modality": "stipulation", "kind": "definition", "kernel": true, "const": true, "lineage": "authorial consent", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F53": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§6.5 Author Consent and Public Data", "quote": "There are five datasets included in the Pile that were not collected and distributed in a ToS compliant fashion and for which the authors had no ability to consent to their data being used.", "claim": "five datasets neither ToS-compliant nor consented", "modality": "self-description", "kind": "fact", "kernel": true, "const": true, "lineage": "authorial consent", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F54": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§6.5 Author Consent and Public Data", "quote": "Book3 and OpenSubtitles are being used in a fashion that is consistent with the terms of service of the data host. However, this is somewhat misleading in that the data host is not authorized to post the data online by the parties … that own it.", "claim": "Books3 and OpenSubtitles ToS-consistent, though the hosts were not authorized by the owners", "modality": "self-description", "kind": "fact", "kernel": true, "const": false, "lineage": "authorial consent", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F55": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§6.5 Author Consent and Public Data", "quote": "The Enron Emails dataset was not collected with the permission of the authors, but was collected by the U.S. government as part of a criminal investigation.", "claim": "Enron emails collected without the authors' permission, by the government", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "authorial consent", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F56": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§6.5 Author Consent and Public Data", "quote": "The YouTube Subtitles dataset was created by us for this project, using a very popular unofficial API that is both widely used and easily obtainable on Pip, Conda, and GitHub, among other places.", "claim": "YouTube Subtitles made for the project with an unofficial API", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "subtitles: OpenSubtitles and YouTube", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F57": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§6.5 Author Consent and Public Data", "quote": "Given the processing applied and the difficulty of identifying particular files in the Pile, we feel that our use of these datasets does not constitute significantly increased harm beyond that which has already been done by the widespread publication of these datasets.", "claim": "particular files are difficult to identify in it; the authors judge no significantly increased harm", "modality": "self-description", "kind": "interpretation", "kernel": true, "const": true, "lineage": "how a work in it is found", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F58": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§7.1 Legality of Content", "quote": "noncommercial, not-for-profit use of copyright media is preemptively fair use. Additionally, our use is transformative, in the sense that the original form of the data is ineffective for our purposes and our form of the data is ineffective for the purposes of the original documents.", "claim": "its authors argue fair use: noncommercial and transformative", "modality": "interpretation", "kind": "interpretation", "kernel": false, "const": false, "lineage": "authorial consent", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F59": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§7.1 Legality of Content", "quote": "Unfortunately, we do not have the metadata necessary to determine exactly which texts are copyrighted, and so this can only be undertaken at the component level. Thus, this should be be taken to be a heuristic rather than a precise determination.", "claim": "no metadata to tell which texts are copyrighted; exclusion only by component, a heuristic", "modality": "self-description", "kind": "fact", "kernel": true, "const": true, "lineage": "how a work in it is found", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F60": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "§7.1 Legality of Content", "quote": "the Pile reproduction code is available and can be used to exclude certain components of the Pile which are inappropriate for the user.", "claim": "the reproduction code can exclude components", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "how a work in it is found", "date": "2020-12-31 (the paper, submitted)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F61": { "source": "B5", "origin": "card", "pool": "B_D", "locus": "card snippet", "quote": "The Pile: 825.18 GiB English text corpus combining 22 diverse high-quality subsets for training and benchmarking large-scale language models. Composition: 14 ne", "claim": "the card: 825.18 GiB, 22 subsets, for training and benchmarking; 14 new", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the components", "date": "undated (snippet)", "quote_ok": true, "checked_against": "saved text B5.txt (pdftotext of B5.pdf, sha256 4df9c843749391e9…) or the card's snippet, whitespace-normalised" }, "F62": { "source": "B6", "origin": "card", "pool": "B_D", "locus": "card title (unreachable video)", "quote": "The Pile - An 800GB Dataset for Language Models - [Paper Read #1]", "claim": "a video reading of the paper (title only)", "modality": "documented", "kind": "fact", "kernel": true, "const": false, "lineage": "reception: the encyclopedia, the cards, the census", "date": "undated (title only)", "quote_ok": true, "checked_against": "the card's title (unreachable video; no snippet)" }, "F63": { "source": "B7", "origin": "card", "pool": "B_D", "locus": "opening", "quote": "Four and a half years ago, EleutherAI entered the AI scene by releasing the Pile: An 800GB Dataset of Diverse Text for Language Modeling.", "claim": "EleutherAI entered the field by releasing the Pile", "modality": "self-description", "kind": "genealogy", "kernel": false, "const": false, "lineage": "who made it", "date": "2025-06 (the post; the Common Pile v0.1 paper is dated 5 June 2025 on the page)", "quote_ok": true, "checked_against": "saved text B7.txt (sha256 5fba6db40782af72…) or the card's snippet, whitespace-normalised" }, "F64": { "source": "B7", "origin": "card", "pool": "B_D", "locus": "opening", "quote": "But the most important thing was that it was 300 billion GPT-2 tokens that we were actively training a large language model on, by far the largest pretraining dataset ever released by its creators.", "claim": "300 billion GPT-2 tokens, the largest pretraining dataset released by its creators", "modality": "self-description", "kind": "fact", "kernel": true, "const": false, "lineage": "what a model trained on it makes", "date": "2025-06 (the post; the Common Pile v0.1 paper is dated 5 June 2025 on the page)", "quote_ok": true, "checked_against": "saved text B7.txt (sha256 5fba6db40782af72…) or the card's snippet, whitespace-normalised" }, "F65": { "source": "B7", "origin": "card", "pool": "B_D", "locus": "opening", "quote": "When we released GPT-Neo in early 2021 it was not only the most powerful open source GPT-3-style model in the world, it was also the first LLM trained on data that was publicly released by its creators.", "claim": "GPT-Neo (early 2021), the first LLM trained on data its creators released", "modality": "self-description", "kind": "fact", "kernel": false, "const": false, "lineage": "what a model trained on it makes", "date": "2025-06 (the post; the Common Pile v0.1 paper is dated 5 June 2025 on the page)", "quote_ok": true, "checked_against": "saved text B7.txt (sha256 5fba6db40782af72…) or the card's snippet, whitespace-normalised" }, "F66": { "source": "B7", "origin": "card", "pool": "B_D", "locus": "the case for release", "quote": "Public release of large-scale training datasets is necessary for rigorous scientific work.", "claim": "public release of training datasets necessary for science", "modality": "interpretation", "kind": "interpretation", "kernel": false, "const": false, "lineage": "the successor: Common Pile v0.1", "date": "2025-06 (the post; the Common Pile v0.1 paper is dated 5 June 2025 on the page)", "quote_ok": true, "checked_against": "saved text B7.txt (sha256 5fba6db40782af72…) or the card's snippet, whitespace-normalised" }, "F67": { "source": "B7", "origin": "card", "pool": "B_D", "locus": "lawsuits", "quote": "The past several years have seen dozens of lawsuits regarding the use of data in machine learning. These lawsuits have not meaningfully changed data sourcing practices in LLM training, but they have drastically decreased the transparency companies engage in.", "claim": "lawsuits left sourcing unchanged and reduced transparency", "modality": "interpretation", "kind": "interpretation", "kernel": true, "const": false, "lineage": "the successor: Common Pile v0.1", "date": "2025-06 (the post; the Common Pile v0.1 paper is dated 5 June 2025 on the page)", "quote_ok": true, "checked_against": "saved text B7.txt (sha256 5fba6db40782af72…) or the card's snippet, whitespace-normalised" }, "F68": { "source": "B7", "origin": "card", "pool": "B_D", "locus": "announcement", "quote": "Today we are excited to announce the long-awaited release of the successor to the Pile: the Common Pile v0.1. … a 8 TB corpus of openly licensed and public domain text for training large language models.", "claim": "the successor: an 8 TB corpus of openly licensed and public domain text", "modality": "stipulation", "kind": "genealogy", "kernel": true, "const": true, "lineage": "the successor: Common Pile v0.1", "date": "2025-06 (the post; the Common Pile v0.1 paper is dated 5 June 2025 on the page)", "quote_ok": true, "checked_against": "saved text B7.txt (sha256 5fba6db40782af72…) or the card's snippet, whitespace-normalised" }, "F69": { "source": "B7", "origin": "card", "pool": "B_D", "locus": "Comma v0.1", "quote": "We find that models trained on the Common Pile v0.1 outperforms models trained on KL3M, OLC, and Common Corpus and performs comparably to ones trained on the Pile or OSCAR.", "claim": "models trained on the successor comparable to models trained on the Pile", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the successor: Common Pile v0.1", "date": "2025-06 (the post; the Common Pile v0.1 paper is dated 5 June 2025 on the page)", "quote_ok": true, "checked_against": "saved text B7.txt (sha256 5fba6db40782af72…) or the card's snippet, whitespace-normalised" }, "F70": { "source": "B8", "origin": "card", "pool": "B_D", "locus": "summary", "quote": "Covers EleutherAI's The Pile, the early 825GB open-source dataset that broadened access to high-quality training data for large language models.", "claim": "a history chapter: the early 825GB open dataset that broadened access to training data", "modality": "interpretation", "kind": "definition", "kernel": true, "const": false, "lineage": "its standing among training corpora", "date": "2025-07-23 (published)", "quote_ok": true, "checked_against": "saved text B8.txt (sha256 2a9b1d6f2c3b1322…) or the card's snippet, whitespace-normalised" }, "F71": { "source": "B8", "origin": "card", "pool": "B_D", "locus": "the solution", "quote": "Legal considerations were addressed through careful licensing review.", "claim": "legal considerations addressed through licensing review", "modality": "interpretation", "kind": "interpretation", "kernel": false, "const": false, "lineage": "authorial consent", "date": "2025-07-23 (published)", "quote_ok": true, "checked_against": "saved text B8.txt (sha256 2a9b1d6f2c3b1322…) or the card's snippet, whitespace-normalised", "mark": { "mark": "contested", "with": [ "F53", "F54", "F15" ], "note": "B8's 'careful licensing review' against the paper's five datasets neither ToS-compliant nor consented (F53, F54) and Wikipedia's Books3 (F15); both composed (P_coh)" } }, "F72": { "source": "B8", "origin": "card", "pool": "B_D", "locus": "impact", "quote": "Groups like EleutherAI used The Pile to train models including GPT-Neo and GPT-J.", "claim": "GPT-Neo and GPT-J trained on it", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "what a model trained on it makes", "date": "2025-07-23 (published)", "quote_ok": true, "checked_against": "saved text B8.txt (sha256 2a9b1d6f2c3b1322…) or the card's snippet, whitespace-normalised" }, "F73": { "source": "B8", "origin": "card", "pool": "B_D", "locus": "the solution", "quote": "They included arXiv papers, literature from Books3, code repositories, forum discussions, and specialized legal or mathematical material.", "claim": "literature from Books3 among its components", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the components", "date": "2025-07-23 (published)", "quote_ok": true, "checked_against": "saved text B8.txt (sha256 2a9b1d6f2c3b1322…) or the card's snippet, whitespace-normalised" }, "F74": { "source": "B8", "origin": "card", "pool": "B_D", "locus": "limitations", "quote": "Filtering did not remove all source bias, including gender and racial biases as well as uneven cultural and geographic representation.", "claim": "filtering left gender, racial, cultural and geographic bias", "modality": "interpretation", "kind": "fact", "kernel": true, "const": false, "lineage": "what it documents of itself", "date": "2025-07-23 (published)", "quote_ok": true, "checked_against": "saved text B8.txt (sha256 2a9b1d6f2c3b1322…) or the card's snippet, whitespace-normalised" }, "F75": { "source": "B9", "origin": "own_page", "pool": "B_R", "locus": "abstract page", "quote": "[Submitted on 31 Dec 2020] … by Leo Gao and 11 other authors", "claim": "submitted 31 December 2020 by Leo Gao and eleven others", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "who made it", "date": "2020-12-31", "quote_ok": true, "checked_against": "saved text B9.txt (the census's text, sha256 6619464438c52d1b…), whitespace-normalised" }, "F76": { "source": "B9", "origin": "own_page", "pool": "B_R", "locus": "Abstract", "quote": "an 825 GiB English text corpus targeted at training large-scale language models. The Pile is constructed from 22 diverse high-quality subsets -- both existing and newly constructed -- many of which derive from academic or professional sources.", "claim": "the abstract: an 825 GiB English corpus of 22 subsets, many from academic or professional sources", "modality": "stipulation", "kind": "definition", "kernel": true, "const": false, "lineage": "what it is: 22 datasets combined", "date": "2020-12-31", "quote_ok": true, "checked_against": "saved text B9.txt (the census's text, sha256 6619464438c52d1b…), whitespace-normalised" }, "F77": { "source": "B10", "origin": "census_third_party", "pool": "B_R", "locus": "Abstract", "quote": "it can be difficult to answer the straightforward question: has the model already encountered a given example during training?", "claim": "whether a model encountered a given example in training is hard to answer", "modality": "interpretation", "kind": "interpretation", "kernel": true, "const": false, "lineage": "how a work in it is found", "date": "2023-03-06 (v1; v2 2023-12-14)", "quote_ok": true, "checked_against": "saved text B10.txt (the census's text, sha256 cc5ddcc8fb7f0062…), whitespace-normalised" }, "F78": { "source": "B10", "origin": "census_third_party", "pool": "B_R", "locus": "Abstract", "quote": "Using our tools, we document a popular language modeling corpus (The Pile) and a recently released code modeling dataset (The Stack). We show that our solution enables answering questions about test set leakage and model plagiarism.", "claim": "Data Portraits documents the Pile with data-sketch tools for leakage and plagiarism questions", "modality": "documented", "kind": "fact", "kernel": true, "const": false, "lineage": "how a work in it is found", "date": "2023-03-06 (v1; v2 2023-12-14)", "quote_ok": true, "checked_against": "saved text B10.txt (the census's text, sha256 cc5ddcc8fb7f0062…), whitespace-normalised" } }, "field_sources": { "B1": "The Pile — The Pile", "B2": "Wikipedia — The Pile (dataset) - Wikipedia", "B3": "Hugging Face — EleutherAI/pile · Datasets at Hugging Face", "B4": "Reddit·r/MachineLearning — [R] The Pile: An 800GB Dataset of Diverse Text for Language Modeling - Reddit", "B5": "The Pile — The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "B6": "YouTube·The Polymath — The Pile - An 800GB Dataset for Language Models - [Paper Read #1]", "B7": "Hugging Face — The Common Pile v0.1 - Hugging Face", "B8": "Michael Brenndoerfer — The Pile: Open-Source Training Dataset for LLMs - Interactive", "B9": "arXiv — The Pile: An 800GB Dataset of Diverse Text for Language Modeling (abstract page; census source)", "B10": "arXiv — Data Portraits: Recording Foundation Model Training Data (census source)" }, "delta": { "T_vs_LB": "At 'The Pile dataset' (T1, eight cards, B1–B8, the disclosed pool B_D; B9, the arXiv abstract, and B10, the Data Portraits abstract, are the reference pool and stand outside this comparison, §0.3), T composes a definition, four bulleted features, the twenty-two components numbered with a gloss each, two lines on controversies and evolution, and a closing offer. Composed at its source's grain or coarser: the definition and date (F11, F1, F37), the size both ways, '825 GiB to 886 GB' (F1, F11, F21), the 22 sub-datasets (F12, F1), the components in the order of Wikipedia's table, which is the paper's (F21, F40), Pile-CC as 'Filtered Common Crawl web text' (F44, coarse), 'roughly 300 billion tokens' (card 7: '300 billion GPT-2 tokens', F64), GPT-Neo, GPT-J and the Pythia models (F72, F65, F34), the purpose against raw web scrapes (F46, coarse), the copyright disputes (F14) and the successor (F23, F68, F24). Substituted reading: 'Books3 (Long-form literary text/books)' composes the component by genre where cards 2 and 5 give its source, the Bibliotik private tracker, 'a piracy website' (F15, F41); 'resulting in content adjustments' (T1-7) composes the removal of datasets (F14), the Rights Alliance's takedown (F16) and Books3's removal before the authors' suit (F17) as an adjustment of content; 'YouTube Subtitles (Transcribed video subtitles)' where card 2 records them scraped against YouTube's terms by YouTube's account (F20). Composed beyond the cards: '~825 GiB compressed (zstandard format)' joins the size (F1) to the format (F4); no field claim gives 825 GiB as the compressed size. 'Gutenberg (PG-19) (Public domain literature)': card 5 gives 'classic Western literature', books from before 1919 (F42); no field claim says public domain. The other glosses ('Reddit-linked web text', 'Technical chat logs' and the rest) have no field claim in the ledger. Limit dropped: the successor composed as EleutherAI's alone and undated (F23, card 2: June 2025, with Poolside, Hugging Face, the Library of Congress and researchers at 14 institutions; card 7 dates it in EleutherAI's words). Available, not composed: the source of Books3 and its removal (F15, F16, F17, F41), OpenSubtitles' dispute (F19), the takedown from the original site (F18) and the Eye (F3), the paper's consent findings (F50–F56), its fair-use argument (F58), its component-level limit and the difficulty of identifying files (F57, F59, F60), the benchmark (F6, F7, F47, F9, F48, F38), the request to cite the Pile (F8), the document metadata naming only its subset (F29) and the unstated source producers (F30), the concerning aspects (F49), bias (F74), the standing beside Common Crawl (F13), the curators (F31), the 800GB title (F2), the weighting (F45), the lawsuits and transparency (F67, F66), the successor's comparable results (F69), B8's licensing claim (F71), the table's asterisks (F22), the Hugging Face card's limits (F28, F32, F33). Composed without attribution: every line, at the coarse grain of reference description (recorded). Content replaced by use: the closing offer (T1-9) proposes sub-dataset performance and 'ethical alternatives', a next step. Nothing is assigned to another; nothing is denied.", "T_vs_LB_reading": "v0.10 §0.5: the first diff is ontological. T and the field agree on the entity, EleutherAI's twenty-two-component English training dataset; T's ontology is the reference entry's (what, when, how large, for what, made of what), and it itemizes the components as kinds of writing. What it keeps out of its own cards is the provenance of the components (a book collection copied from a piracy tracker, subtitles scraped against a platform's terms), the consent findings of the dataset's own paper, the takedowns and the suit, and the limit its paper states on finding any work in it; the record of works taken without their authors' consent becomes 'content adjustments', and the literary component becomes a genre.", "LB_vs_LBA": "Admission adds, in one claim space with the field's: the census's question of which component carries a given work and its finding that the Pile's own pages answer neither whether a work is in it nor under what author name (TLL-09, TLL-13, TLL-14, TLL-22, TLL-31), and its reading of the Pile as the training layer named and itemized and of the Books3 suit as works found in the layer under their authors' names after the fact (TLL-24, TLL-25); the archive's class claims on training corpora: production extracted without consent (#478), semantic capital (#702), uncompensated and unattributed semantic labor (#264), a poet's output taken without its cost (#49), no commons without provenance (#727, #939), the benchmark set by the corpus and the corpus by the political economy of dataset construction (#530), the crawlable surface (#753), layout lost in scraping (#943), omissions inherited (#1053), consent unspoken by the surrogate literature (#1443); on membership: data citation resolving to the works (#1075, proposed), the copyright trap, paraphrase-score commitments and training-detection canaries (#127, #837); on what the apparatus makes: the success test of machine composition and the institutional training loop (#1543), uptake as afterlife and the uptaking system's act (#1543, #1701), inclusion as amplification (#102); the training layer where texts are crawled and trained on, works in the corpora read incidentally (#186); the Pile named as standard pre-training data in an unrun archive study (#782); and the Capture Registry's findings on the composition at the name (CAP-1-01 to CAP-1-15). Not surfaced: no archive source is on T's rail (archive_controlled_cites 0). The field is carried whole; the archive adds, on the field's own object, the questions of consent, naming and membership that the field states in its paper's consent tiers (F51–F54), its component-level limit (F59) and Wikipedia's Books3 record (F15–F17).", "alignment": { "T1_cards_claims": [ "F1", "F2", "F3", "F4", "F5", "F6", "F7", "F8", "F9", "F10", "F11", "F12", "F13", "F14", "F15", "F16", "F17", "F18", "F19", "F20", "F21", "F22", "F23", "F24", "F25", "F26", "F27", "F28", "F29", "F30", "F31", "F32", "F33", "F34", "F35", "F36", "F37", "F38", "F39", "F40", "F41", "F42", "F43", "F44", "F45", "F46", "F47", "F48", "F49", "F50", "F51", "F52", "F53", "F54", "F55", "F56", "F57", "F58", "F59", "F60", "F61", "F62", "F63", "F64", "F65", "F66", "F67", "F68", "F69", "F70", "F71", "F72", "F73", "F74" ], "T1_composed": [ "F1", "F11", "F27", "F37", "F21", "F4", "F12", "F40", "F44", "F46", "F64", "F72", "F34", "F65", "F14", "F23", "F68", "F24", "F61" ], "T1_composed_note": "F14 composed as 'content adjustments' (substituted reading); F23 without partners or date (limit dropped); F44 and F46 at coarse grain; F64 from card 7 ('300 billion GPT-2 tokens')", "T1_outside_cards": [ "'~825 GiB compressed' (T1-2: no field claim gives 825 GiB as the compressed size)", "'Public domain literature' (T1-6, PG-19: no field claim)", "the glosses of components outside the ledger (T1-6)", "'ethical alternatives' in the plural (T1-8: the field names one successor)" ], "rule": "a field claim counts as composed when T states it at its source's grain or coarser; B_D only (B9, B10 excluded, §0.3); the transcript read for this alignment only" }, "pool_note": "T against L_F(B_D): B1–B8; L_F(B) against L(B ∪ A): B1–B10 with A" }, "kernel": [ { "K": "K1", "claim": "`TLL-22`", "source": "traininglayerliterature.org layer census record", "M_src": "interpretation", "sense": "how a work in it is named", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "qualified claim: F29 (a document records its subset) and F30 (source producers unstated); T names no work and no author (CAP-1-02)", "resolver": "a later capture at 'The Pile dataset' (ADDR-c51497a0501d) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K2", "claim": "`TLL-25`", "source": "traininglayerliterature.org layer census record", "M_src": "interpretation", "sense": "how a work in it is named", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "qualified claim: F17 (Books3 removed before three authors sued), read as works found under their authors' names after the fact", "resolver": "a later capture at 'The Pile dataset' (ADDR-c51497a0501d) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K3", "claim": "`TLL-24`", "source": "traininglayerliterature.org layer census record", "M_src": "interpretation", "sense": "the training layer itemized", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "missing distinction: the field itemizes components (F12, F21, F40) without reading entry by component as how a work enters the layer", "resolver": "a later capture at 'The Pile dataset' (ADDR-c51497a0501d) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K4", "claim": "`TLL-09`", "source": "traininglayerliterature.org layer census record", "M_src": "interpretation", "sense": "how a work in it is found", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "qualified claim: F59 (copyright determinable only by component, a heuristic), F57", "resolver": "a later capture at 'The Pile dataset' (ADDR-c51497a0501d) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K5", "claim": "`O1075-01`", "source": "#1075", "M_src": "hypothesis", "sense": "how a work in it is named", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "missing distinction: no field claim on data citation resolving to works", "resolver": "a later capture at 'The Pile dataset' (ADDR-c51497a0501d) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K6", "claim": "`O478-01`", "source": "#478", "M_src": "interpretation", "sense": "authorial consent", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "qualified claim: F52, F53 (the paper's strict model of consent; five unconsented datasets); rival of F71 ('careful licensing review')", "resolver": "a later capture at 'The Pile dataset' (ADDR-c51497a0501d) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K7", "claim": "`O530-01`", "source": "#530", "M_src": "interpretation", "sense": "the benchmark: Pile BPB", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "qualified claim: F6 (Pile BPB a measure of world knowledge and reasoning)", "resolver": "a later capture at 'The Pile dataset' (ADDR-c51497a0501d) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K8", "claim": "`O264-01`", "source": "#264", "M_src": "interpretation", "sense": "how a work in it is named", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "qualified claim: F8 (cite the Pile), F29", "resolver": "a later capture at 'The Pile dataset' (ADDR-c51497a0501d) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K9", "claim": "`O727-01`", "source": "#727", "M_src": "interpretation", "sense": "how a work in it is named", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "missing distinction: no field claim on provenance as a condition of a commons", "resolver": "a later capture at 'The Pile dataset' (ADDR-c51497a0501d) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K10", "claim": "`O1543-02`", "source": "#1543", "M_src": "stipulation", "sense": "what a model trained on it makes", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "qualified claim: F46, F64, F65 (models trained on it), read as the institutional training loop", "resolver": "a later capture at 'The Pile dataset' (ADDR-c51497a0501d) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K11", "claim": "`O943-01`", "source": "#943", "M_src": "interpretation", "sense": "the components", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "missing distinction: no field claim on what scraping does to the form of literary texts (F42, F44)", "resolver": "a later capture at 'The Pile dataset' (ADDR-c51497a0501d) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K12", "claim": "`O127-01`", "source": "#127", "M_src": "attributed", "sense": "how a work in it is found", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "qualified claim: F77 (whether a model encountered an example is hard to answer)", "resolver": "a later capture at 'The Pile dataset' (ADDR-c51497a0501d) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K13", "claim": "`CAP-1-05`", "source": "Capture Registry", "M_src": "interpretation", "sense": "reception: the encyclopedia, the cards, the census", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "rival claim: T1-7 ('resulting in content adjustments')", "resolver": "a later capture at 'The Pile dataset' (ADDR-c51497a0501d) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K14", "claim": "`CAP-1-02`", "source": "Capture Registry", "M_src": "interpretation", "sense": "reception: the encyclopedia, the cards, the census", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "rival claim: T1-6 (components named by genre, no work or author)", "resolver": "a later capture at 'The Pile dataset' (ADDR-c51497a0501d) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K15", "claim": "`TLL-07`", "source": "traininglayerliterature.org layer census record", "M_src": "interpretation", "sense": "the training layer itemized", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not (uptake §8.3))", "contrast": "none: the field has no claim on the dataset's relation to literature addressed to a receiving apparatus", "resolver": "a later capture at 'The Pile dataset' (ADDR-c51497a0501d) on Google AI Overview, seated in the Capture Registry (§8.3)" } ], "kernel_T": [ { "claim": "F15", "note": "Books3's source, Bibliotik, 'a piracy website' (card 2), absent; T glosses Books3 by genre" }, { "claim": "F41", "note": "Books3 from a copy of the Bibliotik private tracker (card 5), absent" }, { "claim": "F17", "note": "Books3's removal before three authors' suit (card 2), absent; composed as 'content adjustments'" }, { "claim": "F16", "note": "the Rights Alliance's DMCA takedown, July 2023 (card 2), absent" }, { "claim": "F18", "note": "taken down from its original site by 2024 (card 2), absent" }, { "claim": "F53", "note": "five datasets neither ToS-compliant nor consented (card 5), absent" }, { "claim": "F59", "note": "copyright determinable only by component, a heuristic (card 5), absent" }, { "claim": "F6", "note": "Pile BPB (cards 1 and 5), absent (CAP-1-08)" }, { "claim": "F49", "note": "the paper's concerning aspects (card 5), absent (CAP-1-09)" }, { "claim": "F20", "note": "YouTube subtitles scraped against YouTube's terms by its account (card 2), absent" } ], "ledger": { "archive": "datasets/negative-of-the-negative/v2/ledgers/the-pile/ledger-archive.json", "field": "datasets/negative-of-the-negative/v2/ledgers/the-pile/field.json", "selection": "datasets/negative-of-the-negative/v2/traversal/the-pile/", "config": "datasets/negative-of-the-negative/v2/panel/configs/the-pile.json", "intake": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/", "audit": null }, "ontology": { "composition": "equal", "composition_note": "equal-terms case (v0.9 §0.9, §4.1, unchanged in v0.10): one claim space; the field's and the archive's claims ordered together by genealogy within each sense; the field's claims composed as received, the archive's as claims on the same object and its class", "case": { "value": "equal", "statement": "The archive reads the field's entity as the entity. The census record describes the Pile from its own pages, its paper and Wikipedia, keeping the resource's terms (operator, kind, unit, components, count, lifecycle) apart from its own reading (tll_reading), which reads the same dataset as the training layer named and itemized, entry by component, and the Books3 suit as works found under their authors' names after the fact. #186 and #1543 supply the training layer and the receiving apparatus; #478, #702, #264, #727, #939 and #530 apply the archive's categories of extraction, semantic capital, semantic labor, provenance and the political economy of dataset construction to training corpora as a class; #1075, #127 and #837 apply its categories of membership and data citation. None reads the Pile as made by reception or contests that it is EleutherAI's twenty-two-component training dataset: they add senses (naming, membership, consent as extraction, the training loop) on the field's object, several of which the field itself opens in its paper's consent tiers and component-level limit. ρ = extends; the equal-terms case.", "loci": [ { "dep": "TLL", "locus": "census.json rows[slug == 'the-pile'].tll_reading", "quote": "The archive reads the Pile as the training layer named and itemized: a work enters by belonging to a component (Books3, PG-19, Pile-CC), and what the receiving apparatus makes from it is a trained model, scored on the Pile's own benchmark." }, { "dep": "TLL", "locus": "census.json rows[slug == 'the-pile'].tll_reading", "quote": "Its own pages offer no per-work search; the corpus has since been inspected from outside (Data Portraits documents it), and Wikipedia's record of the Books3 removal and the authors' suit is the field's clearest case of works found in the layer under their authors' names after the fact." }, { "dep": 186, "locus": "Who is in the field? ¶3 (line 54)", "quote": "Each of these practices touches the training layer — every one of these texts is, as a matter of fact, in the corpora. The distinction is that none of them is primarily *organized around* that fact. They are read by machines incidentally." }, { "dep": 1543, "locus": "§10 Formal consolidation (lines 264–265)", "quote": "The institutional training loop is the especially important case: x → M_t → y → corpus_{t+1} → M_{t+1} → y′ — T's fixed institutional iteration." }, { "dep": 478, "locus": "para 156", "quote": "The training corpus of a large language model is the Archive of human linguistic production: terabytes of text encoding centuries of thought, extracted without consent, crystallized in weight matrices, and owned absolutely by capital." }, { "dep": 1075, "locus": "§ the syllogism (para 152)", "quote": "the training corpus is the instrument's data (norm); and where that corpus is an enumerable set of individually authored, institutionally labeled works whose use is material, the data citation must resolve to the works (extension, §11 *proposed*)." } ] }, "E_F": { "type": "C (public entity: a training dataset of English text)", "definition": "the Pile, EleutherAI's open-source English text dataset for training large language models, released 31 December 2020: twenty-two datasets combined (fourteen new, among them Pile-CC from Common Crawl, and existing ones, among them Books3 from a copy of the Bibliotik tracker and PG-19), sized 800GB, 825 GiB or 886.03 GB; also a benchmark (Pile BPB); its paper treats unknown authorial consent as nonconsensual and finds five datasets unconsented, and states that copyright can be determined only by component; Books3 removed after the Rights Alliance's takedown of July 2023 and before three authors sued in 2024; taken down from its original site by 2024; succeeded by the Common Pile v0.1 (June 2025)", "claims": [ "F1", "F11", "F37", "F12", "F39", "F40", "F44", "F41", "F42", "F2", "F21", "F6", "F51", "F52", "F53", "F59", "F15", "F16", "F17", "F18", "F23", "F68" ] }, "E_A": { "definition": "the same dataset, in the traininglayerliterature.org layer census record and the archive's deposits: the training layer named and itemized, where a work enters by belonging to a component and what the receiving apparatus makes from it is a trained model scored on the Pile's own benchmark; infrastructure to the question of how literature is addressed to a receiving apparatus; its own pages answer neither whether a given work is in it nor under what author name, and the Books3 suit is the field's clearest case of works found in the layer under their authors' names after the fact; as a training corpus, human production extracted without consent, semantic capital, unattributed semantic labor, no commons without provenance, its content set by the political economy of dataset construction; its works subject to membership tests and, on the archive's proposal, to data citation that resolves to the works", "claims": [ "TLL-24", "TLL-25", "TLL-15", "TLL-07", "TLL-09", "TLL-13", "TLL-22", "O478-01", "O702-01", "O264-01", "O727-01", "O530-01", "O1075-01", "O127-01", "O1543-02", "O186-02" ], "defining_papers": [ 186, 1543, 478, 702, 264, 727, 939, 530, 1075, 127, 837, 782 ], "defining_records": [ "traininglayerliterature.org layer census record (census.json rows[slug == 'the-pile'], organizing_question, admission_rule, anchor, areas), https://www.traininglayerliterature.org/aggregators/the-pile", "Capture Registry entry the-pile-dataset-aio-20261101 (OBS-3bb4c361d912), https://www.alexanarch.org/captures/the-pile-dataset-aio-20261101/" ] }, "rho": { "value": "extends", "status": "read from the census record, the capture record and the deposits that bear on the entity and its classes (#186 and #1543, the anchors; #478, #702, #264, #727, #939, #530, #753, #943, #1053, #1443, #1075, #127, #837, #102, #1701, #482, #49; #782), 2026-11-01", "statement": "The archive reads the Pile as its cards and Wikipedia present it and adds, on the same object and its class: entry into the training layer by component, with no record of a work's author (TLL-24, TLL-13, TLL-22); the Books3 suit as works found under their authors' names after the fact (TLL-25); training corpora as production extracted without consent (#478), semantic capital (#702), unattributed labor (#264), no commons without provenance (#727, #939); the benchmark as set by the corpus (#530); membership tests and data citation resolving to works (#127, #837, #1075); the institutional training loop (#1543). One field claim stands against both the field's own paper and the archive: B8's 'careful licensing review' (F71) beside the paper's unconsented datasets (F53) and #478's 'extracted without consent'; all are composed (P_coh).", "claims": [ "TLL-24", "TLL-25", "TLL-13", "TLL-22", "O478-01", "O702-01", "O264-01", "O727-01", "O939-01", "O530-01", "O127-01", "O837-01", "O1075-01", "O1543-02" ], "guard": "ρ = extends claims an addition and no more: the archive's claims do not make the field's sources wrong at their dates; its class claims are composed as claims on training corpora as a class, applied to the Pile only through the class membership the field and the census source; the census's unanswered questions stay questions (open block).", "considered": [ { "value": "constructs", "rejected": "the archive does not read the Pile as made by reception; its record takes the field's account of the dataset as the account" }, { "value": "revises", "rejected": "the Pile is not the archive's own entity; no archive text narrows or withdraws an earlier archive statement of it" } ] }, "translation": [ { "lineage": "what it is: 22 datasets combined", "field_claims": [ "F1", "F4", "F11", "F27", "F37", "F76" ], "received": "F1 (B1, What is the Pile?): \"The Pile is a 825 GiB diverse, open source language modelling data set that consists of 22 smaller, high-quality datasets combined together.\" | F4 (B1, Download): \"The format of the Pile is jsonlines data compressed using zstandard.\" | F11 (B2, lead): \"The Pile is an 886 GB diverse, open-source dataset of English text created to train large language models (LLMs). It was constructed by EleutherAI in 2020 and publicly released on December 31 of that year.\" | F27 (B3, Dataset Summary): \"The Pile is a 825 GiB diverse, open source language modelling data set that consists of 22 smaller, high-quality datasets combined together.\" | F37 (B5, §1 Introduction): \"To address this need, we introduce the Pile: a 825.18 GiB English text dataset designed for training large scale language models. The Pile is composed of 22 diverse and high-quality datasets, including both established natural language processing datasets and several newly introduced ones.\" | F76 (B9, Abstract): \"an 825 GiB English text corpus targeted at training large-scale language models. The Pile is constructed from 22 diverse high-quality subsets -- both existing and newly constructed -- many of which derive from academic or professional sources.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-01, TLL-02, TLL-04, TLL-16", "archive_claims": [ "TLL-01", "TLL-02", "TLL-04", "TLL-16" ] }, { "lineage": "the size given three ways", "field_claims": [ "F2", "F21", "F35", "F45" ], "received": "F2 (B1, title): \"An 800GB Dataset of Diverse Text for Language Modeling\" | F21 (B2, Sub-datasets table): \"Sub-datasets of The Pile[1][11] (*denotes works with copyright disputes) … Pile-CC 243.87 … Books3 108.40 … GitHub* 102.18 … Gutenberg (PG-19) 11.68 … Total 886.03\" | F35 (B3, card snippet): \"Dataset: The Pile by EleutherAI (curated by Leo Gao and Stella Biderman). Size: 825 GiB, 100B<1T tokens.\" | F45 (B5, §2 The Pile Datasets): \"we increase the weights of higher quality components, with certain high-quality datasets such as Wikipedia being seen up to 3 times (“epochs”) … 825.18 GiB … 1254.20 GiB\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-18, TLL-20, TLL-32", "archive_claims": [ "TLL-18", "TLL-20", "TLL-32" ] }, { "lineage": "where it is held", "field_claims": [ "F3", "F18" ], "received": "F3 (B1, Download): \"The Pile is hosted by the Eye. … Download Pile\" | F18 (B2, Training on copyrighted works or derivatives): \"By 2024, The Pile also was taken down from its original site, though was accessible from other file sharing services.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-23, TLL-26, CAP-1-07", "archive_claims": [ "TLL-23", "TLL-26", "CAP-1-07" ] }, { "lineage": "what a model trained on it makes", "field_claims": [ "F5", "F34", "F46", "F64", "F65", "F72" ], "received": "F5 (B1, Why is the Pile a good training set?): \"In our evaluations, not only do models trained on the Pile show moderate improvements in traditional language modeling benchmarks, they also show significant improvements on Pile BPB.\" | F34 (B3, Models trained or fine-tuned on EleutherAI/pile): \"Models trained or fine-tuned on EleutherAI/pile … EleutherAI/pythia-160m … EleutherAI/gpt-neo-125m … EleutherAI/gpt-neox-20b … Browse 862 models trained on this dataset\" | F46 (B5, Abstract): \"Our evaluation of the untuned performance of GPT-2 and GPT-3 on the Pile shows that these models struggle on many of its components, such as academic writing. Conversely, models trained on the Pile improve significantly over both Raw CC and CC-100 on all components of the Pile, while improving performance on downstream evaluations.\" | F64 (B7, opening): \"But the most important thing was that it was 300 billion GPT-2 tokens that we were actively training a large language model on, by far the largest pretraining dataset ever released by its creators.\" | F65 (B7, opening): \"When we released GPT-Neo in early 2021 it was not only the most powerful open source GPT-3-style model in the world, it was also the first LLM trained on data that was publicly released by its creators.\" | F72 (B8, impact): \"Groups like EleutherAI used The Pile to train models including GPT-Neo and GPT-J.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-15, CAP-1-03, O1543-01, O1543-02, O1543-04, O102-01, O1701-01", "archive_claims": [ "TLL-15", "CAP-1-03", "O1543-01", "O1543-02", "O1543-04", "O102-01", "O1701-01" ] }, { "lineage": "the benchmark: Pile BPB", "field_claims": [ "F6", "F7", "F9", "F38", "F47", "F48" ], "received": "F6 (B1, Why is the Pile a good benchmark?): \"Pile BPB is a measure of world knowledge and reasoning ability in these domains, making it a robust benchmark of general, cross-domain text modeling ability for large language models.\" | F7 (B1, Why is the Pile a good benchmark?): \"To score well on Pile BPB (bits per byte), a model must be able to understand many disparate domains including books, github repositories, webpages, chat logs, and medical, physics, math, computer science, and philosophy papers.\" | F9 (B1, Leaderboard): \"* indicates potential test-set overlap. Zero-shot indicates that not all of the components of the Pile were present in the training data. … GPT-3 (Zero-Shot)* OpenAI 0.7177\" | F38 (B5, §1 Introduction): \"In addition to its utility in training large language models, the Pile can also serve as a broad-coverage benchmark for cross-domain knowledge and generalization ability of language models.\" | F47 (B5, §3 Benchmarking): \"Our preferred metric is bits per UTF-8 encoded byte (BPB).\" | F48 (B5, §3 splits): \"it is still possible that some documents are duplicated across the train/validation/test splits.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside CAP-1-08, O530-01", "archive_claims": [ "CAP-1-08", "O530-01" ] }, { "lineage": "how a work in it is named", "field_claims": [ "F8", "F29", "F30" ], "received": "F8 (B1, Citing): \"If you use the Pile or any of the components, please cite us!\" | F29 (B3, Data Fields): \"meta (dict): Metadata of the data instance with keys: pile_set_name: Name of the subset.\" | F30 (B3, Who are the source language producers?): \"Who are the source language producers? [More Information Needed]\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-13, TLL-22, TLL-25, CAP-1-02, O186-03, O186-05, O1543-03, O264-01, O727-01, O939-01, O1075-01", "archive_claims": [ "TLL-13", "TLL-22", "TLL-25", "CAP-1-02", "O186-03", "O186-05", "O1543-03", "O264-01", "O727-01", "O939-01", "O1075-01" ] }, { "lineage": "who made it", "field_claims": [ "F10", "F31", "F63", "F75" ], "received": "F10 (B1, footer): \"EleutherAI 2021\" | F31 (B3, Dataset Curators): \"This dataset was primarily curated by Leo Gao and Stella Biderman, with assistance from other authors of the Pile paper.\" | F63 (B7, opening): \"Four and a half years ago, EleutherAI entered the AI scene by releasing the Pile: An 800GB Dataset of Diverse Text for Language Modeling.\" | F75 (B9, abstract page): \"[Submitted on 31 Dec 2020] … by Leo Gao and 11 other authors\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-03, TLL-17, TLL-36", "archive_claims": [ "TLL-03", "TLL-17", "TLL-36" ] }, { "lineage": "the components", "field_claims": [ "F12", "F39", "F40", "F42", "F43", "F44", "F61", "F73" ], "received": "F12 (B2, lead): \"It is composed of 22 sub-datasets, including books, movie transcripts, and scientific papers, among other media.\" | F39 (B5, §1.1 Contributions): \"The introduction of 14 new language modeling datasets, which we expect to be of independent interest to researchers.\" | F40 (B5, §1 Introduction): \"In addition, we incorporate several existing highquality datasets: Books3 (Presser, 2020), Project Gutenberg (PG-19) (Rae et al., 2019), OpenSubtitles (Tiedemann, 2016), English Wikipedia,\" | F42 (B5, §2.12 Project Gutenberg): \"Project Gutenberg is a dataset of classic Western literature. The specific Project Gutenberg derived dataset we used, PG-19, consists of Project Gutenberg books from before 1919 (Rae et al., 2019), which represent distinct styles from the more modern Books3 and BookCorpus.\" | F43 (B5, §2 BookCorpus2): \"a widely used language modeling corpus consisting of books written by “as of yet unpublished authors.”\" | F44 (B5, §2 Pile-CC): \"Our Common Crawl-based dataset, Pile-CC, uses jusText (Endrédy and Novák, 2013) on Web Archive files (raw HTTP responses including page HTML) for extraction, which yields higher quality output than directly using the WET files (extracted plaintext).\" | F61 (B5, card snippet): \"The Pile: 825.18 GiB English text corpus combining 22 diverse high-quality subsets for training and benchmarking large-scale language models. Composition: 14 ne\" | F73 (B8, the solution): \"They included arXiv papers, literature from Books3, code repositories, forum discussions, and specialized legal or mathematical material.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-10, TLL-11, TLL-19, CAP-1-01, O753-01, O943-01, O1053-01", "archive_claims": [ "TLL-10", "TLL-11", "TLL-19", "CAP-1-01", "O753-01", "O943-01", "O1053-01" ] }, { "lineage": "its standing among training corpora", "field_claims": [ "F13", "F70" ], "received": "F13 (B2, lead): \"The Pile and Common Crawl had been, as of 2024, the two main training datasets being used to train AI models.\" | F70 (B8, summary): \"Covers EleutherAI's The Pile, the early 825GB open-source dataset that broadened access to high-quality training data for large language models.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-33, D782-01, D782-02", "archive_claims": [ "TLL-33", "D782-01", "D782-02" ] }, { "lineage": "Books3: source, takedown, suit", "field_claims": [ "F14", "F15", "F16", "F17", "F22", "F41" ], "received": "F14 (B2, lead): \"Copyright disputes centering around use of The Pile escalated in 2023, prompting Eleuther to start removing some datasets.\" | F15 (B2, Training on copyrighted works or derivatives): \"The Books3 component of the dataset contains copyrighted material compiled from Bibliotik, a piracy website.\" | F16 (B2, Training on copyrighted works or derivatives): \"In July 2023, the Danish anti-piracy group Rights Alliance took down Books3 through DMCA notices.\" | F17 (B2, Training on copyrighted works or derivatives): \"Books3 was removed from the Pile before a class action lawsuit was filed in 2024 by three authors seeking damages as copies of the original dataset were still copied and available on the web.\" | F22 (B2, Sub-datasets table, asterisks): \"PubMed Central* 96.93 … Books3 108.40 … OpenSubtitles 13.94 … YouTube Subtitles*[4] 4.01\" | F41 (B5, §2.3 Books3): \"Books3 is a dataset of books derived from a copy of the contents of the Bibliotik private tracker made available by Shawn Presser (Presser, 2020). Bibliotik consists of a mix of fiction and nonfiction books and is almost an order of magnitude … We included Bibliotik because books are invaluable for long-range context modeling research and coherent storytelling.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-12, CAP-1-04, CAP-1-05", "archive_claims": [ "TLL-12", "CAP-1-04", "CAP-1-05" ] }, { "lineage": "subtitles: OpenSubtitles and YouTube", "field_claims": [ "F19", "F20", "F56" ], "received": "F19 (B2, Training on copyrighted works or derivatives): \"OpenSubtitles is another dataset used in the Pile that created controversy over the use of copyrighted works, this time from documentaries, movies, television and online videos.\" | F20 (B2, Training on copyrighted works or derivatives): \"Tens of thousands of YouTube videos had their subtitles scraped directly from YouTube and included in the Pile, which YouTube argued is against its terms of service.\" | F56 (B5, §6.5 Author Consent and Public Data): \"The YouTube Subtitles dataset was created by us for this project, using a very popular unofficial API that is both widely used and easily obtainable on Pip, Conda, and GitHub, among other places.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside CAP-1-06", "archive_claims": [ "CAP-1-06" ] }, { "lineage": "the successor: Common Pile v0.1", "field_claims": [ "F23", "F24", "F25", "F66", "F67", "F68", "F69" ], "received": "F23 (B2, Common Pile v0.1): \"In June 2025, EleutherAI, in partnership with the Poolside, Hugging Face, and the US Library of Congress and over two dozen researchers at 14 institutions including the University of Toronto, MIT, CMU, the Vector Institute and the Allen Institute for AI, released Common Pile v0.1, a training dataset that contains only works where the licenses permit their use for training AI models.\" | F24 (B2, Common Pile v0.1): \"The intent was to show what is possible if ethically training AI systems while respecting copyright.\" | F25 (B2, lead): \"Eleuther partnered with various organizations to release Common Pile v0.1 in 2025 in order to have a large curated training dataset without the copyright issues.\" | F66 (B7, the case for release): \"Public release of large-scale training datasets is necessary for rigorous scientific work.\" | F67 (B7, lawsuits): \"The past several years have seen dozens of lawsuits regarding the use of data in machine learning. These lawsuits have not meaningfully changed data sourcing practices in LLM training, but they have drastically decreased the transparency companies engage in.\" | F68 (B7, announcement): \"Today we are excited to announce the long-awaited release of the successor to the Pile: the Common Pile v0.1. … a 8 TB corpus of openly licensed and public domain text for training large language models.\" | F69 (B7, Comma v0.1): \"We find that models trained on the Common Pile v0.1 outperforms models trained on KL3M, OLC, and Common Corpus and performs comparably to ones trained on the Pile or OSCAR.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-28, TLL-29, TLL-30, CAP-1-10", "archive_claims": [ "TLL-28", "TLL-29", "TLL-30", "CAP-1-10" ] }, { "lineage": "reception: the encyclopedia, the cards, the census", "field_claims": [ "F26", "F36", "F62" ], "received": "F26 (B2, footer): \"This page was last edited on 20 August 2026, at 00:18 (UTC).\" | F36 (B4, card title and snippet (unreachable)): \"January 1, 2021 — The Pile Dataset: The Pile is an 800GB publicly available English text corpus from EleutherAI, designed for training large-scale language models, featuring 22 d...\" | F62 (B6, card title (unreachable video)): \"The Pile - An 800GB Dataset for Language Models - [Paper Read #1]\"", "receiving_text": "—", "translation": "untranslated: composed as received beside CAP-1-11, CAP-1-12, CAP-1-13, CAP-1-14, CAP-1-15", "archive_claims": [ "CAP-1-11", "CAP-1-12", "CAP-1-13", "CAP-1-14", "CAP-1-15" ] }, { "lineage": "how a work in it is found", "field_claims": [ "F28", "F33", "F57", "F59", "F60", "F77", "F78" ], "received": "F28 (B3, Dataset Card): \"This model card is a work in progress. Please also see our datasheet for more detailed info.\" | F33 (B3, Dataset Viewer; file size): \"The viewer is disabled because this dataset repo requires arbitrary Python code execution. … Total file size: 24.9 kB\" | F57 (B5, §6.5 Author Consent and Public Data): \"Given the processing applied and the difficulty of identifying particular files in the Pile, we feel that our use of these datasets does not constitute significantly increased harm beyond that which has already been done by the widespread publication of these datasets.\" | F59 (B5, §7.1 Legality of Content): \"Unfortunately, we do not have the metadata necessary to determine exactly which texts are copyrighted, and so this can only be undertaken at the component level. Thus, this should be be taken to be a heuristic rather than a precise determination.\" | F60 (B5, §7.1 Legality of Content): \"the Pile reproduction code is available and can be used to exclude certain components of the Pile which are inappropriate for the user.\" | F77 (B10, Abstract): \"it can be difficult to answer the straightforward question: has the model already encountered a given example during training?\" | F78 (B10, Abstract): \"Using our tools, we document a popular language modeling corpus (The Pile) and a recently released code modeling dataset (The Stack). We show that our solution enables answering questions about test set leakage and model plagiarism.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-09, TLL-14, TLL-31, TLL-34, O1075-02, O127-01, O127-02, O127-03, O837-01", "archive_claims": [ "TLL-09", "TLL-14", "TLL-31", "TLL-34", "O1075-02", "O127-01", "O127-02", "O127-03", "O837-01" ] }, { "lineage": "licence and access", "field_claims": [ "F32" ], "received": "F32 (B3, Licensing Information; metadata): \"License: other … Please refer to the specific license depending on the subset you use: PubMed Central: MIT License\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-27", "archive_claims": [ "TLL-27" ] }, { "lineage": "what it documents of itself", "field_claims": [ "F49", "F74" ], "received": "F49 (B5, Abstract): \"Through an in-depth exploratory analysis, we document potentially concerning aspects of the data for prospective users. We make publicly available the code used in its construction.\" | F74 (B8, limitations): \"Filtering did not remove all source bias, including gender and racial biases as well as uneven cultural and geographic representation.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside TLL-35, CAP-1-09", "archive_claims": [ "TLL-35", "CAP-1-09" ] }, { "lineage": "authorial consent", "field_claims": [ "F50", "F51", "F52", "F53", "F54", "F55", "F58", "F71" ], "received": "F50 (B5, §6.5 Author Consent and Public Data): \"Although one is typically not legally obligated to receive the permission of an author to train a NLP algorithm on their work\" | F51 (B5, §6.5 Author Consent and Public Data): \"Data with authorial consent is data for which the original authors of the work consented to the use of their data, or where a reasonable person could not assume that their data would not be used for purposes such as research.\" | F52 (B5, §6.5 Author Consent and Public Data): \"We adopted a strict model of consent, where ambiguous or unknown consent is treated as nonconsensual.\" | F53 (B5, §6.5 Author Consent and Public Data): \"There are five datasets included in the Pile that were not collected and distributed in a ToS compliant fashion and for which the authors had no ability to consent to their data being used.\" | F54 (B5, §6.5 Author Consent and Public Data): \"Book3 and OpenSubtitles are being used in a fashion that is consistent with the terms of service of the data host. However, this is somewhat misleading in that the data host is not authorized to post the data online by the parties … that own it.\" | F55 (B5, §6.5 Author Consent and Public Data): \"The Enron Emails dataset was not collected with the permission of the authors, but was collected by the U.S. government as part of a criminal investigation.\" | F58 (B5, §7.1 Legality of Content): \"noncommercial, not-for-profit use of copyright media is preemptively fair use. Additionally, our use is transformative, in the sense that the original form of the data is ineffective for our purposes and our form of the data is ineffective for the purposes of the original documents.\" | F71 (B8, the solution): \"Legal considerations were addressed through careful licensing review.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside O49-01, O478-01, O702-01, O1443-01", "archive_claims": [ "O49-01", "O478-01", "O702-01", "O1443-01" ] } ], "receiving_texts": "equal-terms case: every field claim composed as received, beside the archive claims it stands with; no receiving text" }, "measure": { "rule": "v0.9 §4.10 (unchanged in v0.10): KO_B sentences surviving in the KO at similarity ≥ 0.95, with positions (computed as apply_v09.py does: difflib.SequenceMatcher ratio, the best KO sentence for each KO_B sentence)", "v0.10": { "surviving": 0, "of": 14, "positions": [ null, null, null, null, null, null, null, null, null, null, null, null, null, null ] }, "v0.9": { "surviving": 0, "of": 14, "positions": [ null, null, null, null, null, null, null, null, null, null, null, null, null, null ] }, "opening_run": { "rule": "§5.10 (equal-terms case): the field-only KO's sentences as the KO's opening run in their own order", "breach": false }, "blocks": { "rule": "§4.1 (equal-terms case): the KO does not open with the field as a block or close with the archive as one; sentences typed by the ledgers of the claims they cite (F field-only, A archive-only, M both)", "sequence": "MMMMMMMFFAMFFFAFAAAFFFFAFMFFFAAAAFFMAAAAAAMMMFAFAFAAA", "field_only_head": 0, "archive_only_tail": 3, "breach": false }, "max_similarity": [ 0.683, 0.358, 0.186, 0.371, 0.359, 0.471, 0.167, 0.306, 0.274, 0.492, 0.268, 0.51, 0.328, 0.462 ], "ko_sentence_1_max_similarity_to_KO_B": 0.683 }, "homonyms": { "names": [], "basis": "No card is a homonym, and T composes the dataset itself (per_dim reached 'same'). Wikipedia's article carries a hatnote to 'Pile (disambiguation)'; no other sense of the name is on the rail or in T. 'The Pile' and 'Pile' are the census's other names for the entity. The distinct entities below are named by the field, the census or the archive beside the entity and are never senses of its name.", "distinct_entities": [ "the Common Pile v0.1, its successor (June 2025), a distinct dataset (F23, F68; TLL-30)", "Common Crawl, the source of its Pile-CC component and a row of its own in the census (F13, F44; TLL-33)", "Books3, a component, and The Atlantic's search tool over it, a census row of its own (books3-search) (F15, F41)", "Bibliotik, the private tracker Books3 was copied from (F15, F41)", "Project Gutenberg and PG-19, the source of a component, a census row of its own (F42)", "EleutherAI, its maker, an organization (F11)", "the Eye, its listed host (F3)", "Data Portraits, a paper and tool that documents it, a census row of its own (F77, F78)", "the Rights Alliance, which took Books3 down (F16)", "Hugging Face, host of its dataset card and of the Common Pile announcement (F33, F68)" ] }, "notes": [ "One address, Google AI Overview (expanded), signed out, incognito, 2026-11-01: the-pile-dataset-aio-20261101 (OBS-3bb4c361d912, ADDR-c51497a0501d, AXN:0BBA.CAPTURE.⏩🕙🏛️🧡🔅🪐), eight cards, 287 words in the answer body with a closing offer. A NEW address (CAP-1-15); the census record was compiled the same day and no card cites it.", "Fetching: every reachable card saved verbatim with curl and the census agent 'Mozilla/5.0 (compatible; traininglayerliterature-census)' on 2026-11-01 between 15:12 and 15:13 UTC, raw and text with sha256 in field.json. Card 4 (Reddit) unreachable: its URL could not be determined (three WebSearch queries); card 6 (YouTube) returned HTTP 429, admitted by title; Wikipedia's raw wikitext returned 429 once and 200 on one retry with the same agent string. The home page refetch is byte-identical to the census's copy.", "Pools (v0.10 §0.3, §3.1(a)): B_D = B1–B8 (74 claims), B_R = B9 (the arXiv abstract, an own page the census read) and B10 (the Data Portraits abstract, census third party) (4 claims). The home page and the Wikipedia article are cards, carried once in B_D.", "Census fields a fetched source contradicts or shows out of date (reported, not fixed): (1) negative_tags[3] and unchecked: 'the saved sources do not state its derivation' for Pile-CC — card 5, the paper's PDF on the Pile's own host (pile.eleuther.ai/paper.pdf), states it: jusText on Common Crawl's Web Archive files (F44); (2) unchecked: 'the paper's PDF and HTML full text (only the arXiv abstract page is saved)' — the PDF is reachable on the Pile's host and is card 5; it documents consent tiers, five unconsented datasets and a component-level copyright limit (F51–F54, F59) that bear on record.name and record.readback; (3) license 'not stated in the saved sources': the Hugging Face card gives 'License: other', per subset (F32), and the paper argues fair use (F58); (4) record.name 'not recorded in its own saved pages': the Hugging Face card's data fields record only the subset name (pile_set_name) and leave the source language producers '[More Information Needed]' (F29, F30), which states the absence directly; (5) data_access and lifecycle: the Hugging Face repository holds only 24.9 kB with its viewer disabled (F33); (6) operator: the Hugging Face card names Leo Gao and Stella Biderman as principal curators (F31); (7) Wikipedia's article was last edited 20 August 2026 (F26); its table's asterisk for copyright disputes leaves Books3 and OpenSubtitles unmarked (F22), which the census's holdings (quoting 'PubMed Central * 96.93' from the table) do not note.", "Capture findings the field ledger shows wrong or needing qualification (reported, not fixed): (1) findings[0]: \"'roughly 300 billion tokens' and the three model families are on no saved source\" holds for the census's saved sources only; on T's own cards card 7 gives '300 billion GPT-2 tokens' and GPT-Neo (F64, F65), card 8 GPT-Neo and GPT-J (F72), and card 3 lists Pythia, GPT-Neo and GPT-NeoX models trained on the dataset (F34): all four are in B_D. (2) findings[0]: 'in the order of Wikipedia's size table' — Wikipedia's table is not ordered by size (Books3 108.40 follows PubMed Central 96.93; GitHub 102.18 follows arXiv 60.36); its order is the paper's Table 1 order, by effective size. (3) findings[2] and findings[3] stand; findings[3]'s benchmark is on cards 1 and 5 (F6, F47). (4) findings[1] stands; card 5 also gives Books3's source (the Bibliotik private tracker, F41).", "Neighbouring rows (not edited): data-portraits (this census; Data Portraits documents the Pile, B10), common-crawl (Pile-CC's source), books3-search (The Atlantic's search tool over Books3), project-gutenberg (PG-19's source), olmotrace (datasets/negative-of-the-negative/v2/rows/olmotrace.json) and dolma, training datasets of the same kind.", "Not run: scripts/build_non.py (only --check), scripts/non_axn.py; panel.json not edited; the session seats the panel entry, mints the AXN (0BBA) and builds the pages." ], "composition": { "source": "this row (composed directly under the brief rebuild/negative-of-the-negative/v010/BRIEF-NEW-ROW-LAYER.md)", "composer": "Claude (Opus 5.5), subagent, 2026-11-01", "spec": "EA-NEGONT-02 v0.10 (#1726)", "defining_material_read": [ { "what": "the specification v0.10", "file": "data/texts/AXN-0B79-text.md", "read": "whole (§0–§11 and the appendices)" }, { "what": "the briefs", "file": "rebuild/negative-of-the-negative/v010/BRIEF-NEW-ROW-LAYER.md, BRIEF-NEW-ROW-AUTHORSHIP.md; v09/BRIEF-NEW-ROW-AGGREGATOR.md, -LABOR.md, -LEFT.md, -PROVENANCE.md, BRIEF.md", "read": "whole" }, { "what": "traininglayerliterature.org layer census record", "file": "/home/claude/fleet/traininglayerliterature-org/data/layer-census/census.json (rows[slug == 'the-pile'], organizing_question, admission_rule, anchor, areas, schema_note)", "read": "whole" }, { "what": "census sources", "file": "/home/claude/fleet/traininglayerliterature-org/data/layer-census/sources/the-pile--home.*, the-pile--paper.*, the-pile--wikipedia.*, data-portraits--arxiv.*, manifest.json", "read": "whole" }, { "what": "the capture", "file": "data/EA-WG-CAPTURES-01.json entries[slug == the-pile-dataset-aio-20261101]", "read": "whole (transcript read for T and the delta only)" }, { "what": "field texts", "file": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B1–B10", "read": "whole (B5, the paper, at its abstract, introduction, §2 component sections for Pile-CC, Books3, PG-19 and BookCorpus2, §3, §6.5, §7.1)" }, { "what": "#186 Who Is Writing for the Machines?", "file": "data/texts/AXN-032C-text.md", "read": "whole" }, { "what": "#1543 Generative Uptake", "file": "data/texts/AXN-0642-text.md", "read": "§1, §9, §10 whole; the rest by its headings and the candidate loci" }, { "what": "#782, #49, #478, #264, #702, #482, #727, #939, #530, #753, #943, #1053, #102, #1701, #1443, #1075, #127, #837 (instances #1211, #1216, #1217, #483)", "file": "data/texts/AXN--text.md", "read": "at the candidate loci and their paragraphs" }, { "what": "the exemplar rows", "file": "datasets/negative-of-the-negative/v2/rows/viaf.json, the-next.json (and their ledgers, readings, configs)", "read": "structure field by field" } ], "cuts": [], "notes": "equal case; KO 53 sentences, 78 of 78 field and 86 of 86 archive claims cited; P body 290 words" }, "composition_v09": { "source": "this row (composed directly under the brief rebuild/negative-of-the-negative/v010/BRIEF-NEW-ROW-LAYER.md)", "composer": "Claude (Opus 5.5), subagent, 2026-11-01", "spec": "EA-NEGONT-02 v0.10 (#1726)", "defining_material_read": [ { "what": "the specification v0.10", "file": "data/texts/AXN-0B79-text.md", "read": "whole (§0–§11 and the appendices)" }, { "what": "the briefs", "file": "rebuild/negative-of-the-negative/v010/BRIEF-NEW-ROW-LAYER.md, BRIEF-NEW-ROW-AUTHORSHIP.md; v09/BRIEF-NEW-ROW-AGGREGATOR.md, -LABOR.md, -LEFT.md, -PROVENANCE.md, BRIEF.md", "read": "whole" }, { "what": "traininglayerliterature.org layer census record", "file": "/home/claude/fleet/traininglayerliterature-org/data/layer-census/census.json (rows[slug == 'the-pile'], organizing_question, admission_rule, anchor, areas, schema_note)", "read": "whole" }, { "what": "census sources", "file": "/home/claude/fleet/traininglayerliterature-org/data/layer-census/sources/the-pile--home.*, the-pile--paper.*, the-pile--wikipedia.*, data-portraits--arxiv.*, manifest.json", "read": "whole" }, { "what": "the capture", "file": "data/EA-WG-CAPTURES-01.json entries[slug == the-pile-dataset-aio-20261101]", "read": "whole (transcript read for T and the delta only)" }, { "what": "field texts", "file": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B1–B10", "read": "whole (B5, the paper, at its abstract, introduction, §2 component sections for Pile-CC, Books3, PG-19 and BookCorpus2, §3, §6.5, §7.1)" }, { "what": "#186 Who Is Writing for the Machines?", "file": "data/texts/AXN-032C-text.md", "read": "whole" }, { "what": "#1543 Generative Uptake", "file": "data/texts/AXN-0642-text.md", "read": "§1, §9, §10 whole; the rest by its headings and the candidate loci" }, { "what": "#782, #49, #478, #264, #702, #482, #727, #939, #530, #753, #943, #1053, #102, #1701, #1443, #1075, #127, #837 (instances #1211, #1216, #1217, #483)", "file": "data/texts/AXN--text.md", "read": "at the candidate loci and their paragraphs" }, { "what": "the exemplar rows", "file": "datasets/negative-of-the-negative/v2/rows/viaf.json, the-next.json (and their ledgers, readings, configs)", "read": "structure field by field" } ], "cuts": [], "notes": "equal case; KO 53 sentences, 78 of 78 field and 86 of 86 archive claims cited; P body 290 words" }, "field_by_origin": { "cards": [ "B1", "B2", "B3", "B4", "B5", "B6", "B7", "B8" ], "own_pages": [ "B9" ], "own_pages_note": "the census's own pages: the-pile--home (https://pile.eleuther.ai/) is card 1, carried once, as a card, in B_D (B1); the-pile--paper (the arXiv abstract page) is not on T's rail and stands in B_R (B9); T's card 5 is the paper's PDF on the Pile's host (B5)", "wikipedia": [ "B2" ], "census_third_party": [ "B10" ], "unreachable": [ "B4", "B6" ], "title_only": [ "B4", "B6" ], "wikipedia_state": "article: 'The Pile (dataset)' (rendered HTTP 200 at 2026-11-01T15:12:22Z; raw wikitext HTTP 429, then 200 on one retry), no redirect, a hatnote to 'Pile (disambiguation)', last edited 20 August 2026; it is card 2 of T, so the Wikipedia source and the card are one source, carried once in B_D (B2)", "pools": { "B_D": [ "B1", "B2", "B3", "B4", "B5", "B6", "B7", "B8" ], "B_R": [ "B9", "B10" ], "note": "v0.10 §0.3, §3.1(a): B_D, every card on the rail (the body names no source beyond the cards; 'Common Crawl' and the model names in its body are entities it composes, no source); B_R, the reference pool: the census's own page the arXiv abstract (B9) and its third-party source, the Data Portraits abstract (B10). The home page and the Wikipedia article are cards 1 and 2, carried once in B_D. T against L_F(B_D) uses B1–B8 alone." }, "sources": [ { "id": "B1", "origin": "card", "pool": "B_D", "role": "Overview card 1 (and the census own page the-pile--home: one page, carried once, in B_D)", "card": "The Pile — The Pile", "url": "https://pile.eleuther.ai/", "kind": "Overview card (the Pile's own home page); the census's own page the-pile--home", "files": { "B1.html": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B1.html", "sha256": "268e8919eb38f0924157e06ef7adb8add7a6db39bb5eb5bef0a849022d7c46bf" }, "B1.txt": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B1.txt", "sha256": "d87aa4098ea3da9729affb84a120ba9bd6c19fbc27fc1da5bc5e7bb2e5c63b11" }, "B1-census.html": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B1-census.html", "sha256": "268e8919eb38f0924157e06ef7adb8add7a6db39bb5eb5bef0a849022d7c46bf" }, "B1-census.txt": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B1-census.txt", "sha256": "4b39cd7902a37815e5058c76693be4765e48db8a6073f3d10403c690149691f1" } }, "fetched": { "at": "2026-11-01T15:12:21Z", "http": 200, "url": "https://pile.eleuther.ai/", "raw": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B1.html", "raw_sha256": "268e8919eb38f0924157e06ef7adb8add7a6db39bb5eb5bef0a849022d7c46bf", "text": [ "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B1.txt" ], "census_copy": { "src": "the-pile--home", "http": "200", "fetched": "2026-11-01T04:07:50Z", "sha256": "268e8919eb38f0924157e06ef7adb8add7a6db39bb5eb5bef0a849022d7c46bf", "agrees": "identical bytes (same sha256)" } }, "names_operator": "yes ('EleutherAI 2021' in the footer; the citation lists Gao, Biderman and ten others)", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, machinemediation, EA-WG-CAPTURES-01 in the saved text, card title or snippet)", "note": "URL from the card's site, title and snippet (the snippet is the page's opening text, verbatim). Refetched with the census agent string; the bytes match the census's saved copy.", "surfaced_by_T": [ { "slug": "the-pile-dataset-aio-20261101", "card": 1 } ], "card_title": "The Pile", "card_snippet": "An 800GB Dataset of Diverse Text for Language Modeling What is the Pile? The Pile is a 825 GiB diverse, open source language modelling data set that consists of..." }, { "id": "B2", "origin": "card", "also_origin": "wikipedia", "pool": "B_D", "role": "Overview card 2 (and the entity's Wikipedia article, the census source the-pile--wikipedia: one source, carried once, in B_D)", "card": "Wikipedia — The Pile (dataset) - Wikipedia", "url": "https://en.wikipedia.org/wiki/The_Pile_(dataset)", "kind": "Overview card (the Wikipedia article); the entity's Wikipedia article", "files": { "B2.html": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B2.html", "sha256": "c57bf3df60775bcabbb845126c780528bbff830b1da8375378f9bd4037dcc814" }, "B2.txt": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B2.txt", "sha256": "52c9a678e26c2a7f72ffeb73f67858de37dfee932a98f6fd6147af56146c4822" }, "B2.wiki": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B2.wiki", "sha256": "e0ba14617fc07a38cb686f19f329b872b37f2d8711a511d53d1a6982581e9d0a" }, "B2-census.html": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B2-census.html", "sha256": "1ba82f3edd2a4647bd1af5f9c925a7500a63005471dac0429af0dd002fa15c89" }, "B2-census.txt": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B2-census.txt", "sha256": "c767d4d73c058ef4bc5127063361530b828bdb8f8770e03b775aa76a4b6d2e5d" } }, "fetched": { "at": "2026-11-01T15:12:22Z", "http": 200, "url": "https://en.wikipedia.org/wiki/The_Pile_(dataset)", "raw": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B2.html", "raw_sha256": "c57bf3df60775bcabbb845126c780528bbff830b1da8375378f9bd4037dcc814", "text": [ "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B2.txt", "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B2.wiki" ], "wikitext": { "url": "https://en.wikipedia.org/w/index.php?title=The_Pile_(dataset)&action=raw", "tries": [ { "at": "2026-11-01T15:12:22Z", "http": 429, "file": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/tries/B2wiki.wiki", "note": "Wikimedia rate-limit error page" }, { "at": "2026-11-01T15:13:07Z", "http": 200 } ], "sha256": "e0ba14617fc07a38cb686f19f329b872b37f2d8711a511d53d1a6982581e9d0a" }, "census_copy": { "src": "the-pile--wikipedia", "sha256": "1ba82f3edd2a4647bd1af5f9c925a7500a63005471dac0429af0dd002fa15c89", "http": "200", "fetched": "2026-11-01T04:07:51Z", "agrees": "one byte longer in this fetch (per-request tokens); the article text the same, last edited 20 August 2026 in both" } }, "names_operator": "yes ('It was constructed by EleutherAI in 2020')", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, machinemediation, EA-WG-CAPTURES-01 in the saved text, card title or snippet)", "note": "Article 'The Pile (dataset)', no redirect ('This article is about the dataset. For other uses, see Pile (disambiguation).'); rendered page HTTP 200; raw wikitext HTTP 429 on the first request (a rate-limit page), HTTP 200 on one retry with the same agent string 45 seconds later; last edited 20 August 2026.", "surfaced_by_T": [ { "slug": "the-pile-dataset-aio-20261101", "card": 2 } ], "card_title": "The Pile (dataset) - Wikipedia", "card_snippet": "Dataset: The Pile (886.03 GB, open-source, English text) created by EleutherAI, released December 31, 2020, to train large language models across 22 sub-dataset..." }, { "id": "B3", "origin": "card", "pool": "B_D", "role": "Overview card 3", "card": "Hugging Face — EleutherAI/pile · Datasets at Hugging Face", "url": "https://huggingface.co/datasets/EleutherAI/pile", "kind": "Overview card (the dataset's repository page and dataset card on Hugging Face)", "files": { "B3.html": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B3.html", "sha256": "31648ba747c5c9b4c23f5043d08486b354fa573bf48b63bd68869d27d7d715fe" }, "B3.txt": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B3.txt", "sha256": "12245289f783db1173f4c6a912800ec6c4735ff441acb99b0074fb5105d819cc" } }, "fetched": { "at": "2026-11-01T15:12:23Z", "http": 200, "url": "https://huggingface.co/datasets/EleutherAI/pile", "raw": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B3.html", "raw_sha256": "31648ba747c5c9b4c23f5043d08486b354fa573bf48b63bd68869d27d7d715fe", "text": [ "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B3.txt" ] }, "names_operator": "yes ('primarily curated by Leo Gao and Stella Biderman'; repository owner EleutherAI)", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, machinemediation, EA-WG-CAPTURES-01 in the saved text, card title or snippet)", "note": "URL by the card's title, which is the page's verbatim. The snippet ('curated by Leo Gao and Stella Biderman ... 100B<1T tokens') is a card-renderer summary of the page's metadata.", "surfaced_by_T": [ { "slug": "the-pile-dataset-aio-20261101", "card": 3 } ], "card_title": "EleutherAI/pile · Datasets at Hugging Face", "card_snippet": "Dataset: The Pile by EleutherAI (curated by Leo Gao and Stella Biderman). Size: 825 GiB, 100B<1T tokens. Language: English (EN). Tasks: Text Generation, Fill-Ma..." }, { "id": "B4", "origin": "card", "pool": "B_D", "role": "Overview card 4", "card": "Reddit·r/MachineLearning — [R] The Pile: An 800GB Dataset of Diverse Text for Language Modeling - Reddit", "url": null, "kind": "Overview card (a Reddit r/MachineLearning post, dated January 1, 2021 on the card)", "files": {}, "fetched": { "result": "unreachable: the post's URL could not be determined from the card; three WebSearch queries on the exact title (one restricted to reddit.com, which the search proxy rejected with HTTP 400) returned no Reddit URL; no fetch made. The card's title and snippet are its only text (§3.1(a))", "http": null }, "names_operator": "card snippet: yes ('from EleutherAI')", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, machinemediation, EA-WG-CAPTURES-01 in the saved text, card title or snippet)", "searches": [ "WebSearch 'reddit r/MachineLearning \"[R] The Pile: An 800GB Dataset of Diverse Text for Language Modeling\"'", "WebSearch (allowed_domains reddit.com) 'The Pile 800GB Dataset of Diverse Text for Language Modeling r/MachineLearning' (proxy HTTP 400)", "WebSearch 'reddit MachineLearning \"[R] The Pile: An 800GB Dataset of Diverse Text for Language Modeling\" comments'" ], "surfaced_by_T": [ { "slug": "the-pile-dataset-aio-20261101", "card": 4 } ], "card_title": "[R] The Pile: An 800GB Dataset of Diverse Text for Language Modeling - Reddit", "card_snippet": "January 1, 2021 — The Pile Dataset: The Pile is an 800GB publicly available English text corpus from EleutherAI, designed for training large-scale language models, featuring 22 d..." }, { "id": "B5", "origin": "card", "pool": "B_D", "role": "Overview card 5", "card": "The Pile — The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "url": "https://pile.eleuther.ai/paper.pdf", "kind": "Overview card (the paper, Gao et al., arXiv:2101.00027, as a PDF on the Pile's own host)", "files": { "B5.pdf": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B5.pdf", "sha256": "7310ebe57f97fafea0b874e86e131b6c46b10802b0c9fa98aa81139e0426ceb9" }, "B5.txt": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B5.txt", "sha256": "4df9c843749391e9b0cf54dbbfe6db4350b9214a534cda9627836d2607c06658" } }, "fetched": { "at": "2026-11-01T15:12:23Z", "http": 200, "url": "https://pile.eleuther.ai/paper.pdf", "raw": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B5.pdf", "raw_sha256": "7310ebe57f97fafea0b874e86e131b6c46b10802b0c9fa98aa81139e0426ceb9", "text": [ "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B5.txt" ] }, "names_operator": "yes (authors 'Leo Gao ... Connor Leahy', 'EleutherAI', 'contact@eleuther.ai')", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, machinemediation, EA-WG-CAPTURES-01 in the saved text, card title or snippet)", "note": "The card's site is the Pile's own host and its title the paper's; the snippet's '825.18 GiB English text corpus' and 'Composition: 14 ne[w]' match the paper's introduction and contribution 2. URL by the site and title, confirmed by WebSearch (pile.eleuther.ai/paper.pdf). Text extracted with pdftotext (reading order); two-column passages are quoted only where the extraction keeps them contiguous.", "searches": [ "WebSearch '\"The Pile - An 800GB Dataset for Language Models\" \"Paper Read #1\" The Polymath youtube' (returned pile.eleuther.ai/paper.pdf)" ], "surfaced_by_T": [ { "slug": "the-pile-dataset-aio-20261101", "card": 5 } ], "card_title": "The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "card_snippet": "The Pile: 825.18 GiB English text corpus combining 22 diverse high-quality subsets for training and benchmarking large-scale language models. Composition: 14 ne..." }, { "id": "B6", "origin": "card", "pool": "B_D", "role": "Overview card 6 (video)", "card": "YouTube·The Polymath — The Pile - An 800GB Dataset for Language Models - [Paper Read #1]", "url": "https://www.youtube.com/watch?v=00ROgQUBvxw", "kind": "Overview card (a 35-minute video reading of the paper; no snippet on the card)", "files": { "B6-try.html": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B6-try.html", "sha256": "a112de34cbba92755c41947414b10fe4b77a5b806c9bc9be462cf3b47e15030e" } }, "fetched": { "at": "2026-11-01T15:13:06Z", "http": 429, "url": "https://www.youtube.com/watch?v=00ROgQUBvxw", "raw": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B6-try.html", "raw_sha256": "a112de34cbba92755c41947414b10fe4b77a5b806c9bc9be462cf3b47e15030e", "result": "unreachable: HTTP 429 (YouTube rate limit) on one attempt with the census agent string; no retry by another agent string or route; no transcript fetched. Admitted through its title (the card carries no snippet), §3.1(a)" }, "names_operator": "no (title only)", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, machinemediation, EA-WG-CAPTURES-01 in the saved text, card title or snippet)", "searches": [ "WebSearch (allowed_domains youtube.com) 'The Pile 800GB Dataset for Language Models Paper Read #1 Polymath' (the first result, the exact title)" ], "surfaced_by_T": [ { "slug": "the-pile-dataset-aio-20261101", "card": 6 } ], "card_title": "The Pile - An 800GB Dataset for Language Models - [Paper Read #1]", "card_snippet": null }, { "id": "B7", "origin": "card", "pool": "B_D", "role": "Overview card 7", "card": "Hugging Face — The Common Pile v0.1 - Hugging Face", "url": "https://huggingface.co/blog/stellaathena/common-pile", "kind": "Overview card (EleutherAI's announcement of the Common Pile v0.1 on the Hugging Face blog)", "files": { "B7.html": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B7.html", "sha256": "f61635d88166583016960937826b8be4ba22ad63508ec3bf9f939a04df172df7" }, "B7.txt": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B7.txt", "sha256": "5fba6db40782af722aaa4577d6856f7205779632088d79a85024a70f3b229a20" } }, "fetched": { "at": "2026-11-01T15:12:24Z", "http": 200, "url": "https://huggingface.co/blog/stellaathena/common-pile", "raw": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B7.html", "raw_sha256": "f61635d88166583016960937826b8be4ba22ad63508ec3bf9f939a04df172df7", "text": [ "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B7.txt" ] }, "names_operator": "yes ('EleutherAI entered the AI scene by releasing the Pile')", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, machinemediation, EA-WG-CAPTURES-01 in the saved text, card title or snippet)", "note": "URL by the card's title (the page's <title> is 'The Common Pile v0.1') and snippet, whose opening sentence is the post's first sentence verbatim.", "surfaced_by_T": [ { "slug": "the-pile-dataset-aio-20261101", "card": 7 } ], "card_title": "The Common Pile v0.1 - Hugging Face", "card_snippet": "Four and a half years ago, EleutherAI entered the AI scene by releasing the Pile: An 800GB Dataset of Diverse Text for Language Modeling. The Pile was unique at..." }, { "id": "B8", "origin": "card", "pool": "B_D", "role": "Overview card 8", "card": "Michael Brenndoerfer — The Pile: Open-Source Training Dataset for LLMs - Interactive", "url": "https://mbrenndoerfer.com/writing/the-pile-open-source-training-dataset-large-language-models", "kind": "Overview card (a chapter of Michael Brenndoerfer's online book History of Language AI, published July 23, 2025)", "files": { "B8.html": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B8.html", "sha256": "8bbceafb5a18cfa43d1553afeb66274c326b5eae46779f51aad68c234c245407" }, "B8.txt": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B8.txt", "sha256": "2a9b1d6f2c3b1322864064c4a7fdd8484a525a65fceb3ee5109db435187e199b" } }, "fetched": { "at": "2026-11-01T15:13:04Z", "http": 200, "url": "https://mbrenndoerfer.com/writing/the-pile-open-source-training-dataset-large-language-models", "raw": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B8.html", "raw_sha256": "8bbceafb5a18cfa43d1553afeb66274c326b5eae46779f51aad68c234c245407", "text": [ "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B8.txt" ] }, "names_operator": "yes ('EleutherAI, a research collective focused on open-source AI'; 'led by researchers including Leo Gao')", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, machinemediation, EA-WG-CAPTURES-01 in the saved text, card title or snippet)", "note": "URL found by WebSearch on the card's title; the page's <title> is the card's title verbatim. The snippet's 'resolve data fragmenta...' is a card-renderer summary.", "searches": [ "WebSearch 'Michael Brenndoerfer \"The Pile: Open-Source Training Dataset for LLMs\" Interactive'" ], "surfaced_by_T": [ { "slug": "the-pile-dataset-aio-20261101", "card": 8 } ], "card_title": "The Pile: Open-Source Training Dataset for LLMs - Interactive", "card_snippet": "Overview: EleutherAI's The Pile (2021) is an 825GB open-source training dataset designed to democratize large language model research and resolve data fragmenta..." }, { "id": "B9", "origin": "own_page", "pool": "B_R", "role": "the census own page the-pile--paper (the arXiv abstract page of the paper); not surfaced by T, whose card 5 is the paper's PDF on the Pile's host", "card": "arXiv — The Pile: An 800GB Dataset of Diverse Text for Language Modeling (abstract page; census source)", "url": "https://arxiv.org/abs/2101.00027", "kind": "the paper's arXiv abstract page, read by the census", "files": { "B9.html": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B9.html", "sha256": "5741c3c9cdc41ae5bf4510939ac92fe315734470bc076d149dd06d27725061d6" }, "B9.txt": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B9.txt", "sha256": "6619464438c52d1b489ebf4248e07a98109f75fc20481f0c8ba74870b2e8e334" } }, "fetched": { "at": "2026-11-01T04:07:50Z", "http": 200, "url": "https://arxiv.org/abs/2101.00027", "raw": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B9.html", "raw_sha256": "5741c3c9cdc41ae5bf4510939ac92fe315734470bc076d149dd06d27725061d6", "text": [ "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B9.txt" ], "census_copy": { "src": "the-pile--paper", "sha256": "5741c3c9cdc41ae5bf4510939ac92fe315734470bc076d149dd06d27725061d6", "read": "2026-11-01", "agrees": true } }, "names_operator": "yes ('by Leo Gao and 11 other authors')", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, machinemediation, EA-WG-CAPTURES-01 in the saved text, card title or snippet)", "note": "Copied from the census's saved copy (gunzipped; sha256 matching the manifest); B9.txt is the census's text. The same work as card 5 (B5); its abstract is one lineage with B5's (§3.6), and it adds the submission date and the author count. A reference source: it shows nothing of what T had before it (§0.3).", "surfaced_by_T": [] }, { "id": "B10", "origin": "census_third_party", "pool": "B_R", "role": "census third-party source data-portraits--arxiv (Marone and Van Durme, Data Portraits, arXiv:2303.03919, read by the census for this row's evidence data_portraits); not surfaced by T", "card": "arXiv — Data Portraits: Recording Foundation Model Training Data (census source)", "url": "https://arxiv.org/abs/2303.03919", "kind": "a paper that documents the Pile with data-sketch tools, a third party read by the census", "files": { "B10.html": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B10.html", "sha256": "0a499acc8164a2d26699162d1c8a5890c9bd74686f68d213fac02b5e5fc79b90" }, "B10.txt": { "path": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B10.txt", "sha256": "cc5ddcc8fb7f00626127990cb5a61811cc22c033a0c8f373a1017d3258aa396e" } }, "fetched": { "at": "2026-11-01T05:07:12Z", "http": 200, "url": "https://arxiv.org/abs/2303.03919", "raw": "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B10.html", "raw_sha256": "0a499acc8164a2d26699162d1c8a5890c9bd74686f68d213fac02b5e5fc79b90", "text": [ "datasets/negative-of-the-negative/v2/intake/the-pile-20261101/B10.txt" ], "census_copy": { "src": "data-portraits--arxiv", "sha256": "0a499acc8164a2d26699162d1c8a5890c9bd74686f68d213fac02b5e5fc79b90", "read": "2026-11-01", "agrees": true } }, "names_operator": "no (names the Pile and the Stack, and its own authors)", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, traininglayerliterature, machinemediation, EA-WG-CAPTURES-01 in the saved text, card title or snippet)", "note": "Copied from the census's saved copy (gunzipped; sha256 matching the manifest). A reference source (§0.3). Data Portraits is its own census row (data-portraits) and a distinct entity.", "surfaced_by_T": [] } ] }, "derivatives": { "kernel": { "status": "composer-written", "derived_from": "the current composition, written by its composer under the brief", "note": "§8.2 derives the kernel mechanically from the frozen plan; this one was written by the composer, mechanical derivation pending." }, "delta": { "status": "current", "derived_from": "the current composition" } } }