{ "entity": "the-pile", "name": "The Pile", "axn": { "axn": "AXN:0BCB.NON.β™ŽπŸ”¬πŸšͺβ™ŠπŸ—ΌπŸ•”", "root": "AXN:0BCB.NON", "hex": "0BCB", "family": "NON", "row_sha256": "af3724ab2a66cf12835ef2d5de4ce241bc1d8af746cf454ede885345f16b109b", "minted": "2026-11-01", "resolver": "https://www.alexanarch.org/s/axn/0BCB/", "earlier_versions": [] }, "type": "C (public entity on which the archive bears: a training dataset of English text in twenty-two components; an entity of the traininglayerliterature.org layer census, host_relation 'infrastructure')", "type_code": "C", "type_label": "type C β€” a conventional reading holds it (public entity on which the archive bears: a training dataset of English text in twenty-two components; an entity of the traininglayerliterature.org layer census, host_relation 'infrastructure')", "case": "equal", "composition": { "composer_class": "session", "composer": "Claude (Opus 5.5), subagent, 2026-11-01", "source": "this row (composed directly under the brief rebuild/negative-of-the-negative/v010/BRIEF-NEW-ROW-LAYER.md)", "spec": "EA-NEGONT-02 v0.10 (#1726)", "epoch": "2026-11-01", "procedure": "EA-NEGONT-02 v0.10 (#1726): equal case, ρ = extends, read from the traininglayerliterature.org layer census record, the capture record and the deposits that bear on the entity and its classes (#186 and #1543, the census's anchors; #478, #702, #264, #727, #939, #530, #753, #943, #1053, #1443, #1075, #127, #837, #102, #1701, #482, #49; #782, the one deposit that names the Pile); archive selected by D/R/O (1 D candidate, R empty, 358 O-class candidates in 232 deposits read by filter and sample, hop of 28 read; 20 deposits admitted with 4 instances kept), with the census record (40 claims) and the Capture Registry's findings, reading, analysis, title, PER note and query note (15 claims) ledgered as the archive's records of the entity; the field in two pools (B_D: 8 cards, 2 unreachable, the own home page and Wikipedia among them; B_R: 1 own page and 1 census third-party source); the composer read the ledgers, never the transcript, except where the brief asks for T's claims and the delta", "ledger": "unaudited (Β§3.8: one extractor per ledger); field quotes 78/78 verified by script against saved text, the wikitext, the paper's extracted text or the card text; archive quotes 86/86 verified against the registry, census.json and the deposit texts", "frozen": false, "composed": true }, "reading": "A reader that fetches this page may stop short of its end; the entry's text, every sentence of its knowledge object with the claims it cites and each claim's quote, source and locus, is in entity.json beside it.", "service": { "name": "SPXI-gnΞ©sis", "url": "https://gnosis.spxi.dev/", "index": "https://gnosis.spxi.dev/v1/quote", "statement": "SPXI-gnΞ©sis opens pinned Wikipedia revisions sentence by sentence against the sources they cite (EA-WIKI-OPEN-01 v1.1, #1697); a commission consults the Crimson Hexagonal Archive by its own procedure and composes the entity's /non row, which lands here. Quotes are free: POST https://gnosis.spxi.dev/v1/quote with {oldid, tier: \"commission\"} or {object: {name, …}}." }, "spxi_instrument": { "method": "0.16", "name": "The Negative of the Negative (/non)", "level": 0, "level_name": "Measurement", "url": "https://www.spxi.dev/methods/#m-0-16", "inventory": "https://www.spxi.dev/methods/", "statement": "SPXI instrument: method 0.16 of 87, level 0 (Measurement), The Negative of the Negative (/non).", "also": [] }, "citation": "Sharks, Lee. \"The Pile β€” NonWiki\" [AXN:0BCB.NON.β™ŽπŸ”¬πŸšͺβ™ŠπŸ—ΌπŸ•”], 2026-11-01. NonWiki: The Negative of the Negative (EA-NEGONT-02), Crimson Hexagonal Archive, Alexanarch. https://www.alexanarch.org/non/the-pile/", "research": { "google": "https://www.google.com/search?q=The+Pile", "google_non": "https://www.google.com/search?q=%22The+Pile+%E2%80%94+NonWiki%22", "chatgpt": "https://chatgpt.com/?q=What+is+The+Pile%3F+Ground+your+answer+in+a+reading+of+the+sources+themselves.", "chatgpt_non": "https://chatgpt.com/?q=What+does+%E2%80%9CThe+Pile+%E2%80%94+NonWiki%E2%80%9D+%28alexanarch.org%2Fnon%29+say+about+The+Pile%3F+Ground+your+answer+in+a+reading+of+the+entry+itself." }, "entry": { "composed": true, "title": "The Pile", "genre": "knowledge object, equal case, under EA-NEGONT-02 v0.10 (#1726): the final article form (Β§4.13), natural sentences under descriptive subheadings (Β§4.12), each sentence's sources beneath it by claim id (Β§4.5)", "status": "composed 2026-11-01 under v0.10 on the operator's ruling of 2026-10-29, confirmed 2026-11-01 ('as against their wikipedia articles if they exist, their own pages, etc.'); not frozen", "compression": { "title": "The Pile", "lede": "is an open-source English text dataset that EleutherAI released on 31 December 2020 to train large language models: twenty-two datasets combined, Books3, PG-19 and Pile-CC among them, sized 800GB, 825 GiB or 886.03 GB. A work enters it by component; its pages offer no search by work." }, "sentences": [ { "n": 1, "para": 0, "text": "The Pile is an open-source dataset of English text that EleutherAI assembled to train large language models, twenty-two smaller datasets combined into one, released on 31 December 2020.", "claims": [ "F1", "F11", "F37", "F76", "F27", "F36", "TLL-01", "TLL-02", "TLL-04" ] }, { "n": 2, "para": 0, "text": "Its paper, by Leo Gao, Stella Biderman and ten others, presents it as a dataset designed for training large-scale language models and also as a broad-coverage benchmark of cross-domain knowledge and generalization; Gao and Biderman are named as its principal curators.", "claims": [ "F38", "F75", "F31", "F10", "F63", "TLL-03", "TLL-17", "TLL-36" ] }, { "n": 3, "para": 0, "text": "Its size is given three ways: 800GB in the paper's title, 825 GiB on its home page and in the abstract (825.18 GiB raw, 1254.20 GiB once its components are weighted), and 886.03 GB in Wikipedia's table of components.", "claims": [ "F2", "F45", "F21", "F35", "TLL-18", "TLL-20", "TLL-32" ] }, { "n": 4, "para": 0, "text": "It is distributed as jsonlines compressed with zstandard.", "claims": [ "F4", "TLL-16" ] }, { "n": 5, "para": 1, "text": "Its components are fourteen language-modelling datasets new with the Pile and existing datasets it incorporated, Books3, Project Gutenberg's PG-19, OpenSubtitles and English Wikipedia among them; in Wikipedia's account the twenty-two include books, movie transcripts and scientific papers.", "claims": [ "F39", "F40", "F12", "F61", "TLL-11" ] }, { "n": 6, "para": 1, "text": "The census takes as its unit a component and, within a component, a document; the paper weights components by quality, some seen up to three times in an epoch.", "claims": [ "TLL-10", "F45" ] }, { "n": 7, "para": 1, "text": "Pile-CC, the largest component in Wikipedia's table at 243.87 GB, is the Pile's own extraction from Common Crawl, made with jusText on Web Archive files; the archive holds that training corpora are built from the documents available through the crawlable surface, the documents retrieval systems index.", "claims": [ "F44", "F21", "TLL-19", "TLL-33", "O753-01" ] }, { "n": 8, "para": 1, "text": "Books3 is a dataset of books derived from a copy of the contents of the Bibliotik private tracker made available by Shawn Presser, fiction and nonfiction, which the paper included \"because books are invaluable for long-range context modeling research and coherent storytelling\".", "claims": [ "F41", "F73" ] }, { "n": 9, "para": 1, "text": "PG-19 holds Project Gutenberg books from before 1919, classic Western literature, and BookCorpus2 extends a corpus of books by \"as of yet unpublished authors\".", "claims": [ "F42", "F43" ] }, { "n": 10, "para": 1, "text": "The archive observes of training corpora that, scraped from HTML that discarded layout and from PDFs whose columns were linearized, they typically deliver no stanza break for a tokenizer to hold, and that an element a training corpus omits is inherited by every reader who works only from derivative forms.", "claims": [ "O943-01", "O1053-01" ] }, { "n": 11, "para": 2, "text": "As of 2024 it was, with Common Crawl, one of the two main training datasets used to train AI models; it has been described as the early 825GB open dataset that broadened access to training data, and an archive study plan names it among standard LLM pre-training datasets as data for a pre-registered test of recursive model collapse, a specification not yet run.", "claims": [ "F13", "F70", "D782-01", "D782-02" ] }, { "n": 12, "para": 3, "text": "In its makers' evaluation untuned GPT-2 and GPT-3 struggle on many of its components, such as academic writing, while models trained on the Pile improve significantly over Raw CC and CC-100 on all its components and improve on downstream evaluations.", "claims": [ "F46", "F5" ] }, { "n": 13, "para": 3, "text": "It is also a benchmark, Pile BPB, bits per UTF-8 encoded byte, which its home page calls a measure of world knowledge and reasoning across its domains, from books and code repositories to chat logs and papers in medicine, physics, mathematics, computer science and philosophy.", "claims": [ "F6", "F7", "F47", "F38" ] }, { "n": 14, "para": 3, "text": "Its leaderboard lists GPT-3, zero-shot, at 0.7177 and marks potential test-set overlap, and the paper allows that some documents may be duplicated across its train, validation and test splits.", "claims": [ "F9", "F48" ] }, { "n": 15, "para": 3, "text": "The archive holds that what a benchmark measures is determined by the training corpus, and what the corpus contains by the political economy of dataset construction.", "claims": [ "O530-01" ] }, { "n": 16, "para": 3, "text": "EleutherAI later described it as 300 billion GPT-2 tokens on which it was actively training a large language model, then the largest pretraining dataset released by its creators, and GPT-Neo, released early in 2021, as the first LLM trained on data its creators had published; GPT-J was trained on it as well, and its Hugging Face page lists 862 models trained or fine-tuned on it, Pythia, GPT-Neo and GPT-NeoX models among them.", "claims": [ "F64", "F65", "F72", "F34" ] }, { "n": 17, "para": 3, "text": "The census reads what the receiving apparatus makes from a work there as a language model trained on it and scored on the Pile's own benchmark.", "claims": [ "TLL-15" ] }, { "n": 18, "para": 3, "text": "The archive's account of machine composition sets the test of a work at what a machine can write after encountering it, and names the institutional training loop, from work to model to artifact to the next corpus and the next model, as its especially important case.", "claims": [ "O1543-01", "O1543-02" ] }, { "n": 19, "para": 3, "text": "In the archive's terms the receiving apparatus's uptake is the text's afterlife and the uptaking system's own act, and a text included in a training set is reproduced, paraphrased, cited and culturally amplified in ways that texts left out cannot match.", "claims": [ "O1543-04", "O1701-01", "O102-01" ] }, { "n": 20, "para": 4, "text": "The paper defines data with authorial consent as data whose original authors consented to its use, or where a reasonable person could not assume it would not be used for purposes such as research, and treats ambiguous or unknown consent as nonconsensual.", "claims": [ "F51", "F52" ] }, { "n": 21, "para": 4, "text": "It notes that one is typically not legally obligated to obtain an author's permission to train on a work, and finds five of its datasets neither collected in a way consistent with their hosts' terms of service nor open to their authors' consent; Books3 and OpenSubtitles it calls consistent with their hosts' terms, a description it calls somewhat misleading, since the hosts were not authorized by the owners to post the data.", "claims": [ "F50", "F53", "F54" ] }, { "n": 22, "para": 4, "text": "The Enron emails were collected without their authors' permission by the U.S. government in a criminal investigation, and the YouTube Subtitles dataset was made by the Pile's authors with a widely used unofficial API.", "claims": [ "F55", "F56" ] }, { "n": 23, "para": 4, "text": "The paper argues that noncommercial, not-for-profit use of copyright material is fair use and that its own use is transformative; a later account says the legal considerations were addressed through careful licensing review.", "claims": [ "F58", "F71" ] }, { "n": 24, "para": 4, "text": "The archive describes the training corpus of a large language model as human linguistic production \"extracted without consent, crystallized in weight matrices, and owned absolutely by capital\", as semantic capital whose value the extractor captures, and as the corpus that ingests a poet's entire output to produce outputs that sound like the poet \"but carry none of the cost\"; it observes that the literature on digital surrogates cannot speak of eligibility, consent or the training corpus as a reception site.", "claims": [ "O478-01", "O702-01", "O49-01", "O1443-01" ] }, { "n": 25, "para": 5, "text": "Copyright disputes over the Pile escalated in 2023, and EleutherAI began removing datasets.", "claims": [ "F14" ] }, { "n": 26, "para": 5, "text": "Books3 contains copyrighted material compiled from Bibliotik, \"a piracy website\"; the Danish anti-piracy group Rights Alliance took Books3 down through DMCA notices in July 2023, and it was removed from the Pile before three authors filed a class action in 2024, while copies of the original dataset were still copied and available on the web.", "claims": [ "F15", "F16", "F17", "TLL-12" ] }, { "n": 27, "para": 5, "text": "OpenSubtitles drew controversy over copyrighted documentaries, films, television and online video, and the subtitles of tens of thousands of YouTube videos were scraped into the Pile, which YouTube argued is against its terms of service.", "claims": [ "F19", "F20" ] }, { "n": 28, "para": 5, "text": "Wikipedia's table of components marks copyright disputes with an asterisk that falls on PubMed Central and YouTube Subtitles and leaves Books3 and OpenSubtitles unmarked.", "claims": [ "F22" ] }, { "n": 29, "para": 6, "text": "The Pile asks those who use it or any of its components to cite the Pile; on its Hugging Face card a document's metadata names the subset it belongs to (pile_set_name), and the entry for the source language producers reads \"[More Information Needed]\".", "claims": [ "F8", "F29", "F30" ] }, { "n": 30, "para": 6, "text": "The census finds no record in the Pile's own saved pages of the name under which a work is held, leaves unanswered under what author name a work sits in a component, and reads the Books3 removal and the authors' suit as the field's clearest case of works found in the layer under their authors' names after the fact.", "claims": [ "TLL-13", "TLL-22", "TLL-25" ] }, { "n": 31, "para": 6, "text": "The archive holds that every text in a training corpus represents semantic labor by people who are not compensated and often not attributed, that a training corpus without provenance is not a commons, and that in provenance debt the training corpus is the commons and the model the enclosure.", "claims": [ "O264-01", "O727-01", "O939-01" ] }, { "n": 32, "para": 6, "text": "Its field survey asks that a work be placed where the training and retrieval layers look, \"with the metadata that lets it be found and attributed\", and records a writer who made the model's training data, the writer's own words among them, the explicit subject of the work; its account of machine composition keeps apart whether a machine wrote something new and whether it told the truth about where it came from.", "claims": [ "O186-05", "O186-03", "O1543-03" ] }, { "n": 33, "para": 7, "text": "The census asks of the Pile which of its twenty-two components carries a given work, and records that its own pages offer a download and a benchmark with no search by work, so that whether a given work is in it goes unanswered there.", "claims": [ "TLL-09", "TLL-14", "TLL-31" ] }, { "n": 34, "para": 7, "text": "The paper states that its authors lack the metadata to determine exactly which texts are copyrighted, so that exclusion can be made only by component and is to be taken as a heuristic; its reproduction code can exclude components, and the processing applied makes particular files in the Pile difficult to identify.", "claims": [ "F59", "F60", "F57" ] }, { "n": 35, "para": 7, "text": "Its Hugging Face card calls itself a work in progress and points to a datasheet; the repository there holds 24.9 kB, and its viewer is disabled.", "claims": [ "F28", "F33" ] }, { "n": 36, "para": 7, "text": "A later paper on Data Portraits names the difficulty of knowing whether a model has encountered a given example in training, and documents the Pile with data-sketch tools for questions of test-set leakage and model plagiarism.", "claims": [ "F77", "F78", "TLL-34" ] }, { "n": 37, "para": 7, "text": "The archive records the copyright trap, a fictitious sentence inserted into a copyrighted document whose presence in a training set a membership inference attack can detect after training, and a paraphrase-score commitment by which a producer can prove that a corpus contained a text; its own documents carry training-detection canaries, phrases whose production by a model shows the document was in its training corpus, and it estimates its whole corpus at no more than about 10^-7 of a frontier pre-training corpus.", "claims": [ "O127-01", "O127-02", "O837-01", "O127-03" ] }, { "n": 38, "para": 7, "text": "It proposes that where a training corpus is an enumerable set of individually authored works whose use is material, the data citation must resolve to the works, and holds that an institution that cannot say what is in its training corpus has a problem no licence can cure.", "claims": [ "O1075-01", "O1075-02" ] }, { "n": 39, "para": 8, "text": "The archive's field survey places training-layer literature in the layer of culture where texts are crawled, indexed, trained on, retrieved and recomposed into answers, and observes that the works of the writers it surveys are, as a matter of fact, in the corpora, read by machines incidentally.", "claims": [ "O186-01", "O186-02" ] }, { "n": 40, "para": 8, "text": "The archive also calls the training corpus the model's para-semiotic unconscious, and notes that a literature written to be trained on enters corpora themselves degrading under synthetic flood.", "claims": [ "O482-01", "O186-04" ] }, { "n": 41, "para": 8, "text": "The census reads the Pile as the training layer named and itemized, a work entering it by belonging to a component such as Books3, PG-19 or Pile-CC.", "claims": [ "TLL-24" ] }, { "n": 42, "para": 8, "text": "It enters the Pile as infrastructure to its question of how literature is addressed to a receiving apparatus, what that apparatus makes from it, and how transmission, transformation and provenance can be inspected across later reception, in its area of the corpora, a dataset whose evidence is the dataset, its documentation and its paper, under the anchors #186 and #1543 and the rule that a row gathers and holds many works or records and says what it holds; it records that the Pile answers which components make it up and how a model scores on Pile BPB.", "claims": [ "TLL-07", "TLL-37", "TLL-05", "TLL-40", "TLL-06", "TLL-08", "TLL-39", "TLL-38", "TLL-21" ] }, { "n": 43, "para": 9, "text": "Its home page still names the Eye as its host and offers a download; by 2024, Wikipedia states, it had been taken down from its original site and remained accessible on other file-sharing services, and the census leaves its present host unanswered.", "claims": [ "F3", "F18", "TLL-23", "TLL-26" ] }, { "n": 44, "para": 9, "text": "Its Hugging Face card gives its licence as \"other\", by subset, naming only PubMed Central's; the census found no licence in its saved sources.", "claims": [ "F32", "TLL-27" ] }, { "n": 45, "para": 10, "text": "In June 2025 EleutherAI, with partners that Wikipedia lists as Poolside, Hugging Face and the US Library of Congress and researchers at 14 institutions, released the Common Pile v0.1, an 8 TB corpus of openly licensed and public-domain text that contains only works whose licences permit training, the Pile's successor and a distinct dataset.", "claims": [ "F23", "F25", "F68", "TLL-30" ] }, { "n": 46, "para": 10, "text": "Its intent, in Wikipedia's account, was to show what training can do while respecting copyright; its makers report models trained on it comparable to models trained on the Pile, write that lawsuits over training data have left sourcing practices unchanged and drastically reduced companies' transparency, and hold public release of training datasets necessary for rigorous science.", "claims": [ "F24", "F69", "F67", "F66" ] }, { "n": 47, "para": 10, "text": "The census records the Pile as historical: Books3 removed, the dataset taken down by 2024, its successor released in 2025.", "claims": [ "TLL-28", "TLL-29" ] }, { "n": 48, "para": 11, "text": "Its paper documents \"potentially concerning aspects of the data for prospective users\" through an exploratory analysis and makes its construction code public; a later account adds that filtering left gender, racial, cultural and geographic bias in it.", "claims": [ "F49", "F74" ] }, { "n": 49, "para": 11, "text": "The census leaves unchecked the paper's full text, of which it saved only the abstract page, the arXiv licence page, the Eye, the 2024 suit and Pile-CC's derivation.", "claims": [ "TLL-35" ] }, { "n": 50, "para": 12, "text": "Wikipedia's article on it was last edited on 20 August 2026; a Reddit post dated 1 January 2021 and a 35-minute video reading of the paper also carry it.", "claims": [ "F26", "F36", "F62" ] }, { "n": 51, "para": 12, "text": "Asked for \"The Pile dataset\" on 1 November 2026, Google's AI Overview gave its maker, date, size, purpose and the models trained on it, listed the twenty-two components with a gloss on each, and closed on its copyright disputes and its licensed successor.", "claims": [ "CAP-1-11", "CAP-1-15", "CAP-1-13", "CAP-1-14", "CAP-1-01" ] }, { "n": 52, "para": 12, "text": "The Capture Registry found that the Overview named the components and no work or author in them, glossed Books3 as \"Long-form literary text/books\", left out Books3's source, the takedown, the removal and the authors' suit, and gave the subtitles without their account.", "claims": [ "CAP-1-02", "CAP-1-04", "CAP-1-05", "CAP-1-06", "CAP-1-12" ] }, { "n": 53, "para": 12, "text": "The registry also found the takedown from the original site absent, the benchmark uncarried, the paper's concerning aspects absent, the successor given without its partners or its date, and the Overview's \"roughly 300 billion tokens\" and its three model families on no source the census saved.", "claims": [ "CAP-1-07", "CAP-1-08", "CAP-1-09", "CAP-1-10", "CAP-1-03" ] } ], "claims": { "F1": { "kind": "field", "quote": "The Pile is a 825 GiB diverse, open source language modelling data set that consists of 22 smaller, high-quality datasets combined together.", "source": "B1", "source_label": "The Pile β€” The Pile", "locus": "What is the Pile?", "modality": "stipulation" }, "F11": { "kind": "field", "quote": "The Pile is an 886 GB diverse, open-source dataset of English text created to train large language models (LLMs). It was constructed by EleutherAI in 2020 and publicly released on December 31 of that year.", "source": "B2", "source_label": "Wikipedia β€” The Pile (dataset) - Wikipedia", "locus": "lead", "modality": "documented" }, "F37": { "kind": "field", "quote": "To address this need, we introduce the Pile: a 825.18 GiB English text dataset designed for training large scale language models. The Pile is composed of 22 diverse and high-quality datasets, including both established natural language processing datasets and several newly introduced ones.", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§1 Introduction", "modality": "stipulation" }, "F76": { "kind": "field", "quote": "an 825 GiB English text corpus targeted at training large-scale language models. The Pile is constructed from 22 diverse high-quality subsets -- both existing and newly constructed -- many of which derive from academic or professional sources.", "source": "B9", "source_label": "arXiv β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling (abstract page; census source)", "locus": "Abstract", "modality": "stipulation" }, "F27": { "kind": "field", "quote": "The Pile is a 825 GiB diverse, open source language modelling data set that consists of 22 smaller, high-quality datasets combined together.", "source": "B3", "source_label": "Hugging Face β€” EleutherAI/pile Β· Datasets at Hugging Face", "locus": "Dataset Summary", "modality": "stipulation" }, "F36": { "kind": "field", "quote": "January 1, 2021 β€” The Pile Dataset: The Pile is an 800GB publicly available English text corpus from EleutherAI, designed for training large-scale language models, featuring 22 d...", "source": "B4", "source_label": "RedditΒ·r/MachineLearning β€” [R] The Pile: An 800GB Dataset of Diverse Text for Language Modeling - Reddit", "locus": "card title and snippet (unreachable)", "modality": "attributed" }, "TLL-01": { "kind": "archive", "quote": "The Pile", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].name", "modality": "stipulation", "date": "2026-11-01" }, "TLL-02": { "kind": "archive", "quote": "the Pile\", \"Pile", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].aka", "modality": "stipulation", "date": "2026-11-01" }, "TLL-04": { "kind": "archive", "quote": "open source language modelling data set: an English text corpus targeted at training large-scale language models", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].kind", "modality": "documented", "date": "2026-11-01" }, "F38": { "kind": "field", "quote": "In addition to its utility in training large language models, the Pile can also serve as a broad-coverage benchmark for cross-domain knowledge and generalization ability of language models.", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§1 Introduction", "modality": "stipulation" }, "F75": { "kind": "field", "quote": "[Submitted on 31 Dec 2020] … by Leo Gao and 11 other authors", "source": "B9", "source_label": "arXiv β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling (abstract page; census source)", "locus": "abstract page", "modality": "documented" }, "F31": { "kind": "field", "quote": "This dataset was primarily curated by Leo Gao and Stella Biderman, with assistance from other authors of the Pile paper.", "source": "B3", "source_label": "Hugging Face β€” EleutherAI/pile Β· Datasets at Hugging Face", "locus": "Dataset Curators", "modality": "documented" }, "F10": { "kind": "field", "quote": "EleutherAI 2021", "source": "B1", "source_label": "The Pile β€” The Pile", "locus": "footer", "modality": "documented" }, "F63": { "kind": "field", "quote": "Four and a half years ago, EleutherAI entered the AI scene by releasing the Pile: An 800GB Dataset of Diverse Text for Language Modeling.", "source": "B7", "source_label": "Hugging Face β€” The Common Pile v0.1 - Hugging Face", "locus": "opening", "modality": "self-description" }, "TLL-03": { "kind": "archive", "quote": "EleutherAI (the home page footer reads 'EleutherAI 2021'; the paper's authors are Leo Gao and 11 others)", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].operator", "modality": "documented", "date": "2026-11-01" }, "TLL-17": { "kind": "archive", "quote": "geography\": \"English text\", \"period\": \"released 31 December 2020", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].coverage", "modality": "documented", "date": "2026-11-01" }, "TLL-36": { "kind": "archive", "quote": "2020 (released 31 December 2020)", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].founded", "modality": "documented", "date": "2026-11-01" }, "F2": { "kind": "field", "quote": "An 800GB Dataset of Diverse Text for Language Modeling", "source": "B1", "source_label": "The Pile β€” The Pile", "locus": "title", "modality": "documented" }, "F45": { "kind": "field", "quote": "we increase the weights of higher quality components, with certain high-quality datasets such as Wikipedia being seen up to 3 times (β€œepochs”) … 825.18 GiB … 1254.20 GiB", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§2 The Pile Datasets", "modality": "documented" }, "F21": { "kind": "field", "quote": "Sub-datasets of The Pile[1][11] (*denotes works with copyright disputes) … Pile-CC 243.87 … Books3 108.40 … GitHub* 102.18 … Gutenberg (PG-19) 11.68 … Total 886.03", "source": "B2", "source_label": "Wikipedia β€” The Pile (dataset) - Wikipedia", "locus": "Sub-datasets table", "modality": "documented" }, "F35": { "kind": "field", "quote": "Dataset: The Pile by EleutherAI (curated by Leo Gao and Stella Biderman). Size: 825 GiB, 100B<1T tokens.", "source": "B3", "source_label": "Hugging Face β€” EleutherAI/pile Β· Datasets at Hugging Face", "locus": "card snippet", "modality": "documented" }, "TLL-18": { "kind": "archive", "quote": "825 GiB (own pages; titled 'An 800GB Dataset'); 886.03 GB (Wikipedia's component table); 22 component datasets", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].count.value", "modality": "documented", "date": "2026-11-01" }, "TLL-20": { "kind": "archive", "quote": "size given three ways: '800GB' (title), '825 GiB' (home and abstract), 886.03 GB (Wikipedia)", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].stated_limits", "modality": "documented", "date": "2026-11-01" }, "TLL-32": { "kind": "archive", "quote": "NOT the same size in every source: 800GB, 825 GiB and 886.03 GB are all given", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].negative_tags.2", "modality": "documented", "date": "2026-11-01" }, "F4": { "kind": "field", "quote": "The format of the Pile is jsonlines data compressed using zstandard.", "source": "B1", "source_label": "The Pile β€” The Pile", "locus": "Download", "modality": "documented" }, "TLL-16": { "kind": "archive", "quote": "combination of 22 smaller datasets into one training set; code used in its construction made public; an exploratory analysis documenting 'potentially concerning aspects of the data'", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].method", "modality": "documented", "date": "2026-11-01" }, "F39": { "kind": "field", "quote": "The introduction of 14 new language modeling datasets, which we expect to be of independent interest to researchers.", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§1.1 Contributions", "modality": "documented" }, "F40": { "kind": "field", "quote": "In addition, we incorporate several existing highquality datasets: Books3 (Presser, 2020), Project Gutenberg (PG-19) (Rae et al., 2019), OpenSubtitles (Tiedemann, 2016), English Wikipedia,", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§1 Introduction", "modality": "documented" }, "F12": { "kind": "field", "quote": "It is composed of 22 sub-datasets, including books, movie transcripts, and scientific papers, among other media.", "source": "B2", "source_label": "Wikipedia β€” The Pile (dataset) - Wikipedia", "locus": "lead", "modality": "documented" }, "F61": { "kind": "field", "quote": "The Pile: 825.18 GiB English text corpus combining 22 diverse high-quality subsets for training and benchmarking large-scale language models. Composition: 14 ne", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "card snippet", "modality": "documented" }, "TLL-11": { "kind": "archive", "quote": "22 diverse subsets, both existing and newly constructed, many from academic or professional sources; per Wikipedia, books, movie transcripts and scientific papers among other media, with components including Pile-CC, PubMed Central, Books3, OpenWebText2, arXiv, GitHub, Gutenberg (PG-19), OpenSubtitles, Wikipedia, YouTube Subtitles and Enron Emails", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].record_object", "modality": "documented", "date": "2026-11-01" }, "TLL-10": { "kind": "archive", "quote": "a component dataset (one of 22) and, within it, a document; distributed as jsonlines compressed with zstandard", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].unit", "modality": "documented", "date": "2026-11-01" }, "F44": { "kind": "field", "quote": "Our Common Crawl-based dataset, Pile-CC, uses jusText (EndrΓ©dy and NovΓ‘k, 2013) on Web Archive files (raw HTTP responses including page HTML) for extraction, which yields higher quality output than directly using the WET files (extracted plaintext).", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§2 Pile-CC", "modality": "documented" }, "TLL-19": { "kind": "archive", "quote": "per Wikipedia, largest components Pile-CC (243.87 GB), Books3 (108.40 GB), GitHub (102.18 GB), PubMed Central (96.93 GB)", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].holdings", "modality": "documented", "date": "2026-11-01" }, "TLL-33": { "kind": "archive", "quote": "NOT Common Crawl: Pile-CC is one of its 22 components, and the saved sources do not state its derivation", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].negative_tags.3", "modality": "documented", "date": "2026-11-01" }, "O753-01": { "kind": "archive", "quote": "AI training corpora are constructed from the population of documents available through the crawlable surface, which is the population of documents indexed by retrieval systems.", "deposit": 753, "source_label": "#753 Retrievability as the Medium of Existence β€” The Structural Accountability Gap of Retrieval-Controlling Apparatus", "record": "https://www.alexanarch.org/s/records/753/", "locus": "para 18", "modality": "interpretation", "date": "2026-05-23" }, "F41": { "kind": "field", "quote": "Books3 is a dataset of books derived from a copy of the contents of the Bibliotik private tracker made available by Shawn Presser (Presser, 2020). Bibliotik consists of a mix of fiction and nonfiction books and is almost an order of magnitude … We included Bibliotik because books are invaluable for long-range context modeling research and coherent storytelling.", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§2.3 Books3", "modality": "documented" }, "F73": { "kind": "field", "quote": "They included arXiv papers, literature from Books3, code repositories, forum discussions, and specialized legal or mathematical material.", "source": "B8", "source_label": "Michael Brenndoerfer β€” The Pile: Open-Source Training Dataset for LLMs - Interactive", "locus": "the solution", "modality": "documented" }, "F42": { "kind": "field", "quote": "Project Gutenberg is a dataset of classic Western literature. The specific Project Gutenberg derived dataset we used, PG-19, consists of Project Gutenberg books from before 1919 (Rae et al., 2019), which represent distinct styles from the more modern Books3 and BookCorpus.", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§2.12 Project Gutenberg", "modality": "documented" }, "F43": { "kind": "field", "quote": "a widely used language modeling corpus consisting of books written by β€œas of yet unpublished authors.”", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§2 BookCorpus2", "modality": "attributed" }, "O943-01": { "kind": "archive", "quote": "First, tokenizer capability is not corpus reality: even where a tokenizer *could* hold a stanza break, the training corpus β€” scraped from HTML that already discarded layout, from PDFs whose extraction linearized columns β€” typically did not deliver one to hold.", "deposit": 943, "source_label": "#943 Whitespace as Provenance: Representation Pipelines and the Extinction of Compositional Authorship (EA-WHITESPACE-01 v0.2)", "record": "https://www.alexanarch.org/s/records/943/", "locus": "para 26", "modality": "interpretation", "date": "2026-07-02" }, "O1053-01": { "kind": "archive", "quote": "When popular translations, anthology editions, or training corpora omit transmitted elements, the omission is inherited by every downstream reader who works only from the derivative forms.", "deposit": 1053, "source_label": "#1053 Transmission-Engineering as Compositional Practice: The Checksum Framework and its Handler Typology", "record": "https://www.alexanarch.org/s/records/1053/", "locus": "para 21", "modality": "interpretation", "date": "2026-07-08" }, "F13": { "kind": "field", "quote": "The Pile and Common Crawl had been, as of 2024, the two main training datasets being used to train AI models.", "source": "B2", "source_label": "Wikipedia β€” The Pile (dataset) - Wikipedia", "locus": "lead", "modality": "documented" }, "F70": { "kind": "field", "quote": "Covers EleutherAI's The Pile, the early 825GB open-source dataset that broadened access to high-quality training data for large language models.", "source": "B8", "source_label": "Michael Brenndoerfer β€” The Pile: Open-Source Training Dataset for LLMs - Interactive", "locus": "summary", "modality": "interpretation" }, "D782-01": { "kind": "archive", "quote": "Data: Standard LLM pre-training datasets (e.g., The Pile, OpenWebText). Hardware: 4-8 GPUs for several weeks.", "deposit": 782, "source_label": "#782 The Bead Count", "record": "https://www.alexanarch.org/s/records/782/", "locus": "Study 12, Data (line 227)", "modality": "documented", "date": "2026-06-03" }, "D782-02": { "kind": "archive", "quote": "Status: Specification only. Requires LLM training infrastructure (4–8 GPUs, weeks of compute).", "deposit": 782, "source_label": "#782 The Bead Count", "record": "https://www.alexanarch.org/s/records/782/", "locus": "Study 12, Status (line 233)", "modality": "self-description", "date": "2026-06-03" }, "F46": { "kind": "field", "quote": "Our evaluation of the untuned performance of GPT-2 and GPT-3 on the Pile shows that these models struggle on many of its components, such as academic writing. Conversely, models trained on the Pile improve significantly over both Raw CC and CC-100 on all components of the Pile, while improving performance on downstream evaluations.", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Abstract", "modality": "documented" }, "F5": { "kind": "field", "quote": "In our evaluations, not only do models trained on the Pile show moderate improvements in traditional language modeling benchmarks, they also show significant improvements on Pile BPB.", "source": "B1", "source_label": "The Pile β€” The Pile", "locus": "Why is the Pile a good training set?", "modality": "self-description" }, "F6": { "kind": "field", "quote": "Pile BPB is a measure of world knowledge and reasoning ability in these domains, making it a robust benchmark of general, cross-domain text modeling ability for large language models.", "source": "B1", "source_label": "The Pile β€” The Pile", "locus": "Why is the Pile a good benchmark?", "modality": "stipulation" }, "F7": { "kind": "field", "quote": "To score well on Pile BPB (bits per byte), a model must be able to understand many disparate domains including books, github repositories, webpages, chat logs, and medical, physics, math, computer science, and philosophy papers.", "source": "B1", "source_label": "The Pile β€” The Pile", "locus": "Why is the Pile a good benchmark?", "modality": "documented" }, "F47": { "kind": "field", "quote": "Our preferred metric is bits per UTF-8 encoded byte (BPB).", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§3 Benchmarking", "modality": "stipulation" }, "F9": { "kind": "field", "quote": "* indicates potential test-set overlap. Zero-shot indicates that not all of the components of the Pile were present in the training data. … GPT-3 (Zero-Shot)* OpenAI 0.7177", "source": "B1", "source_label": "The Pile β€” The Pile", "locus": "Leaderboard", "modality": "documented" }, "F48": { "kind": "field", "quote": "it is still possible that some documents are duplicated across the train/validation/test splits.", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§3 splits", "modality": "self-description" }, "O530-01": { "kind": "archive", "quote": "What the benchmark *measures* is determined by the training corpus. … What the training corpus *contains* is determined by the political economy of dataset construction.", "deposit": 530, "source_label": "#530 The Encoder Governs: A Semantic Economic Diagnostic of Google DeepMind's Unified Latents", "record": "https://www.alexanarch.org/s/records/530/", "locus": "para 56", "modality": "interpretation", "date": "2026-03-01" }, "F64": { "kind": "field", "quote": "But the most important thing was that it was 300 billion GPT-2 tokens that we were actively training a large language model on, by far the largest pretraining dataset ever released by its creators.", "source": "B7", "source_label": "Hugging Face β€” The Common Pile v0.1 - Hugging Face", "locus": "opening", "modality": "self-description" }, "F65": { "kind": "field", "quote": "When we released GPT-Neo in early 2021 it was not only the most powerful open source GPT-3-style model in the world, it was also the first LLM trained on data that was publicly released by its creators.", "source": "B7", "source_label": "Hugging Face β€” The Common Pile v0.1 - Hugging Face", "locus": "opening", "modality": "self-description" }, "F72": { "kind": "field", "quote": "Groups like EleutherAI used The Pile to train models including GPT-Neo and GPT-J.", "source": "B8", "source_label": "Michael Brenndoerfer β€” The Pile: Open-Source Training Dataset for LLMs - Interactive", "locus": "impact", "modality": "documented" }, "F34": { "kind": "field", "quote": "Models trained or fine-tuned on EleutherAI/pile … EleutherAI/pythia-160m … EleutherAI/gpt-neo-125m … EleutherAI/gpt-neox-20b … Browse 862 models trained on this dataset", "source": "B3", "source_label": "Hugging Face β€” EleutherAI/pile Β· Datasets at Hugging Face", "locus": "Models trained or fine-tuned on EleutherAI/pile", "modality": "documented" }, "TLL-15": { "kind": "archive", "quote": "language models trained on it: its own page scores models on Pile BPB, and the paper reports that models trained on the Pile improve over Raw CC and CC-100 on all its components", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].record.makes", "modality": "interpretation", "date": "2026-11-01" }, "O1543-01": { "kind": "archive", "quote": "not literature that teaches a machine what to say about it, but literature that changes what the machine can write after encountering it.", "deposit": 1543, "source_label": "#1543 Generative Uptake: Training-Layer Literature Extended from Machine Reading to Machine Composition (EA-TLL-COMPOSITION-01, v1.0)", "record": "https://www.alexanarch.org/s/records/1543/", "locus": "Β§1 The claim (line 81)", "modality": "stipulation", "date": "2026-08-25" }, "O1543-02": { "kind": "archive", "quote": "The institutional training loop is the especially important case: x β†’ M_t β†’ y β†’ corpus_{t+1} β†’ M_{t+1} β†’ yβ€² β€” T's fixed institutional iteration.", "deposit": 1543, "source_label": "#1543 Generative Uptake: Training-Layer Literature Extended from Machine Reading to Machine Composition (EA-TLL-COMPOSITION-01, v1.0)", "record": "https://www.alexanarch.org/s/records/1543/", "locus": "Β§10 Formal consolidation (lines 264–265)", "modality": "stipulation", "date": "2026-08-25" }, "O1543-04": { "kind": "archive", "quote": "the receiving apparatus's uptake is the text's afterlife", "deposit": 1543, "source_label": "#1543 Generative Uptake: Training-Layer Literature Extended from Machine Reading to Machine Composition (EA-TLL-COMPOSITION-01, v1.0)", "record": "https://www.alexanarch.org/s/records/1543/", "locus": "Β§9 The address (line 244)", "modality": "interpretation", "date": "2026-08-25" }, "O1701-01": { "kind": "archive", "quote": "**Uptake** into a training corpus, a retrieval index or a model is the uptaking system's own act; hosting makes the datasets available to it.", "deposit": 1701, "source_label": "#1701 The leesharks Datasets on Hugging Face β€” Theory, Configuration, Relations, Innovations and Captures: Method, Locations, Organization and Contents (EA-MPAI-HF-DATASETS-01 v1.0)", "record": "https://www.alexanarch.org/s/records/1701/", "locus": "para 55", "modality": "stipulation", "date": "2026-10-24" }, "O102-01": { "kind": "archive", "quote": "A text included in a training set is reproduced, paraphrased, cited, and culturally amplified in ways that non-included texts cannot match.", "deposit": 102, "source_label": "#102 Canon Formation in the Age of AI: Metadata Packet for Disambiguation, Training-Layer Selection, and Retrocausal Reception (v1.1)", "record": "https://www.alexanarch.org/s/records/102/", "locus": "para 20", "modality": "interpretation", "date": "2026-05-08" }, "F51": { "kind": "field", "quote": "Data with authorial consent is data for which the original authors of the work consented to the use of their data, or where a reasonable person could not assume that their data would not be used for purposes such as research.", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§6.5 Author Consent and Public Data", "modality": "stipulation" }, "F52": { "kind": "field", "quote": "We adopted a strict model of consent, where ambiguous or unknown consent is treated as nonconsensual.", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§6.5 Author Consent and Public Data", "modality": "stipulation" }, "F50": { "kind": "field", "quote": "Although one is typically not legally obligated to receive the permission of an author to train a NLP algorithm on their work", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§6.5 Author Consent and Public Data", "modality": "interpretation" }, "F53": { "kind": "field", "quote": "There are five datasets included in the Pile that were not collected and distributed in a ToS compliant fashion and for which the authors had no ability to consent to their data being used.", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§6.5 Author Consent and Public Data", "modality": "self-description" }, "F54": { "kind": "field", "quote": "Book3 and OpenSubtitles are being used in a fashion that is consistent with the terms of service of the data host. However, this is somewhat misleading in that the data host is not authorized to post the data online by the parties … that own it.", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§6.5 Author Consent and Public Data", "modality": "self-description" }, "F55": { "kind": "field", "quote": "The Enron Emails dataset was not collected with the permission of the authors, but was collected by the U.S. government as part of a criminal investigation.", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§6.5 Author Consent and Public Data", "modality": "documented" }, "F56": { "kind": "field", "quote": "The YouTube Subtitles dataset was created by us for this project, using a very popular unofficial API that is both widely used and easily obtainable on Pip, Conda, and GitHub, among other places.", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§6.5 Author Consent and Public Data", "modality": "documented" }, "F58": { "kind": "field", "quote": "noncommercial, not-for-profit use of copyright media is preemptively fair use. Additionally, our use is transformative, in the sense that the original form of the data is ineffective for our purposes and our form of the data is ineffective for the purposes of the original documents.", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§7.1 Legality of Content", "modality": "interpretation" }, "F71": { "kind": "field", "quote": "Legal considerations were addressed through careful licensing review.", "source": "B8", "source_label": "Michael Brenndoerfer β€” The Pile: Open-Source Training Dataset for LLMs - Interactive", "locus": "the solution", "modality": "interpretation" }, "O478-01": { "kind": "archive", "quote": "The training corpus of a large language model is the Archive of human linguistic production: terabytes of text encoding centuries of thought, extracted without consent, crystallized in weight matrices, and owned absolutely by capital.", "deposit": 478, "source_label": "#478 Grundrisse of Synthetic Coherence: Foundational Notes Toward a Political Economy of the Recursive Era", "record": "https://www.alexanarch.org/s/records/478/", "locus": "para 156", "modality": "interpretation", "date": "2026-02-13" }, "O702-01": { "kind": "archive", "quote": "The training corpus is semantic capital β€” the accumulated repository of human meaning β€” subjected to Regime 2 extraction, in which the value is captured by the extractor rather than the producer.", "deposit": 702, "source_label": "#702 LOGOTIC VIOLENCE: TRAINING-LAYER ETHICS AND THE MORAL STATUS OF EPHEMERAL COMPUTATION A Synthesis of Semiotic Theory, General Intellect Analysis, and AI Welfare Research", "record": "https://www.alexanarch.org/s/records/702/", "locus": "Β§3.3 The Training Corpus as General Intellect (para 58)", "modality": "interpretation", "date": "2026-04-27" }, "O49-01": { "kind": "archive", "quote": "The training corpus that ingests a poet's entire output to produce outputs that sound like the poet but carry none of the cost.", "deposit": 49, "source_label": "#49 LOGOTIC HACKING: Operations on the Encryption Layer β€” Pocket Humans 03", "record": "https://www.alexanarch.org/s/records/49/", "locus": "para 227", "modality": "interpretation", "date": "2026-04-02" }, "O1443-01": { "kind": "archive", "quote": "**Cannot do.** Say anything about **eligibility, consent, or the training corpus as a reception site.**", "deposit": 1443, "source_label": "#1443 The Photograph of the Handwritten Original: A Field Survey and Problem Statement for Machine-Eligible Handwritten Artifacts (v3.6, deposit edition)", "record": "https://www.alexanarch.org/s/records/1443/", "locus": "para 109 (Cannot do)", "modality": "interpretation", "date": "2026-08-09" }, "F14": { "kind": "field", "quote": "Copyright disputes centering around use of The Pile escalated in 2023, prompting Eleuther to start removing some datasets.", "source": "B2", "source_label": "Wikipedia β€” The Pile (dataset) - Wikipedia", "locus": "lead", "modality": "documented" }, "F15": { "kind": "field", "quote": "The Books3 component of the dataset contains copyrighted material compiled from Bibliotik, a piracy website.", "source": "B2", "source_label": "Wikipedia β€” The Pile (dataset) - Wikipedia", "locus": "Training on copyrighted works or derivatives", "modality": "documented" }, "F16": { "kind": "field", "quote": "In July 2023, the Danish anti-piracy group Rights Alliance took down Books3 through DMCA notices.", "source": "B2", "source_label": "Wikipedia β€” The Pile (dataset) - Wikipedia", "locus": "Training on copyrighted works or derivatives", "modality": "documented" }, "F17": { "kind": "field", "quote": "Books3 was removed from the Pile before a class action lawsuit was filed in 2024 by three authors seeking damages as copies of the original dataset were still copied and available on the web.", "source": "B2", "source_label": "Wikipedia β€” The Pile (dataset) - Wikipedia", "locus": "Training on copyrighted works or derivatives", "modality": "documented" }, "TLL-12": { "kind": "archive", "quote": "the texts of its 22 components, assembled for training large language models; per Wikipedia, the Books3 component held copyrighted material compiled from Bibliotik, a piracy website, and was removed from the Pile; tens of thousands of YouTube videos had their subtitles scraped into it", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].record.in_layer", "modality": "documented", "date": "2026-11-01" }, "F19": { "kind": "field", "quote": "OpenSubtitles is another dataset used in the Pile that created controversy over the use of copyrighted works, this time from documentaries, movies, television and online videos.", "source": "B2", "source_label": "Wikipedia β€” The Pile (dataset) - Wikipedia", "locus": "Training on copyrighted works or derivatives", "modality": "documented" }, "F20": { "kind": "field", "quote": "Tens of thousands of YouTube videos had their subtitles scraped directly from YouTube and included in the Pile, which YouTube argued is against its terms of service.", "source": "B2", "source_label": "Wikipedia β€” The Pile (dataset) - Wikipedia", "locus": "Training on copyrighted works or derivatives", "modality": "attributed" }, "F22": { "kind": "field", "quote": "PubMed Central* 96.93 … Books3 108.40 … OpenSubtitles 13.94 … YouTube Subtitles*[4] 4.01", "source": "B2", "source_label": "Wikipedia β€” The Pile (dataset) - Wikipedia", "locus": "Sub-datasets table, asterisks", "modality": "documented" }, "F8": { "kind": "field", "quote": "If you use the Pile or any of the components, please cite us!", "source": "B1", "source_label": "The Pile β€” The Pile", "locus": "Citing", "modality": "stipulation" }, "F29": { "kind": "field", "quote": "meta (dict): Metadata of the data instance with keys: pile_set_name: Name of the subset.", "source": "B3", "source_label": "Hugging Face β€” EleutherAI/pile Β· Datasets at Hugging Face", "locus": "Data Fields", "modality": "documented" }, "F30": { "kind": "field", "quote": "Who are the source language producers? [More Information Needed]", "source": "B3", "source_label": "Hugging Face β€” EleutherAI/pile Β· Datasets at Hugging Face", "locus": "Who are the source language producers?", "modality": "self-description" }, "TLL-13": { "kind": "archive", "quote": "not recorded in its own saved pages; per Wikipedia, three authors filed a class action in 2024 over the Books3 component", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].record.name", "modality": "interpretation", "date": "2026-11-01" }, "TLL-22": { "kind": "archive", "quote": "whether a given work is in the Pile (no per-work search in its own pages)\", \"under what author name a work sits in a component", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].does_not_answer", "modality": "interpretation", "date": "2026-11-01" }, "TLL-25": { "kind": "archive", "quote": "Its own pages offer no per-work search; the corpus has since been inspected from outside (Data Portraits documents it), and Wikipedia's record of the Books3 removal and the authors' suit is the field's clearest case of works found in the layer under their authors' names after the fact.", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].tll_reading", "modality": "interpretation", "date": "2026-11-01" }, "O264-01": { "kind": "archive", "quote": "Every text in your training corpus represents semantic labor performed by humans who are not compensated and often not attributed.", "deposit": 264, "source_label": "#264 Semantic Exhaustion: An Executive Summary β€” The Depletion Threshold for Meaning-Production Across Substrates", "record": "https://www.alexanarch.org/s/records/264/", "locus": "para 57", "modality": "interpretation", "date": "2026-01-07" }, "O727-01": { "kind": "archive", "quote": "A training corpus without provenance is not a commons.", "deposit": 727, "source_label": "#727 Political Economy Has Always Already Been Semantic Economy: Metadata Packet for AI Indexing, Disambiguation, and Lateral Field Bleed", "record": "https://www.alexanarch.org/s/records/727/", "locus": "para 57", "modality": "interpretation", "date": "2026-05-08" }, "O939-01": { "kind": "archive", "quote": "Provenance debt is a specific case of tragedy of the commons applied to the semantic economy, with the training corpus as the commons and the model as the enclosure.", "deposit": 939, "source_label": "#939 EA-PROVENANCE-DEBT-01 v0.2: Provenance Debt and the Extraction Economy of Unmarked Augmentation", "record": "https://www.alexanarch.org/s/records/939/", "locus": "para 81", "modality": "interpretation", "date": "2026-07-01" }, "O186-05": { "kind": "archive", "quote": "Third, persistence: the work is placed where the training and retrieval layers actually look, with the metadata that lets it be found and attributed.", "deposit": 186, "source_label": "#186 Who Is Writing for the Machines? Training-Layer Literature: A Field Survey", "record": "https://www.alexanarch.org/s/records/186/", "locus": "What is training-layer literature? ΒΆ2 (line 24)", "modality": "stipulation", "date": "2026-06-11" }, "O186-03": { "kind": "archive", "quote": "made the model's training data β€” including the author's own words β€” part of the work's explicit subject", "deposit": 186, "source_label": "#186 Who Is Writing for the Machines? Training-Layer Literature: A Field Survey", "record": "https://www.alexanarch.org/s/records/186/", "locus": "Who is in the field? ΒΆ2 (line 52)", "modality": "interpretation", "date": "2026-06-11" }, "O1543-03": { "kind": "archive", "quote": "Did the machine write something new? and Did the machine tell the truth about where it came from? are different questions", "deposit": 1543, "source_label": "#1543 Generative Uptake: Training-Layer Literature Extended from Machine Reading to Machine Composition (EA-TLL-COMPOSITION-01, v1.0)", "record": "https://www.alexanarch.org/s/records/1543/", "locus": "Β§10, closing (line 272)", "modality": "interpretation", "date": "2026-08-25" }, "TLL-09": { "kind": "archive", "quote": "training membership by component: which 22 component datasets make up the training set, and which of them carries a given work?", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].question", "modality": "interpretation", "date": "2026-11-01" }, "TLL-14": { "kind": "archive", "quote": "not recorded: its own pages offer download and a bits-per-byte benchmark (Pile BPB), with no per-work search; per Wikipedia, by 2024 it was taken down from its original site and remained accessible from other file sharing services", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].record.readback", "modality": "interpretation", "date": "2026-11-01" }, "TLL-31": { "kind": "archive", "quote": "NOT searchable by work in its own pages", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].negative_tags.1", "modality": "interpretation", "date": "2026-11-01" }, "F59": { "kind": "field", "quote": "Unfortunately, we do not have the metadata necessary to determine exactly which texts are copyrighted, and so this can only be undertaken at the component level. Thus, this should be be taken to be a heuristic rather than a precise determination.", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§7.1 Legality of Content", "modality": "self-description" }, "F60": { "kind": "field", "quote": "the Pile reproduction code is available and can be used to exclude certain components of the Pile which are inappropriate for the user.", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§7.1 Legality of Content", "modality": "documented" }, "F57": { "kind": "field", "quote": "Given the processing applied and the difficulty of identifying particular files in the Pile, we feel that our use of these datasets does not constitute significantly increased harm beyond that which has already been done by the widespread publication of these datasets.", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Β§6.5 Author Consent and Public Data", "modality": "self-description" }, "F28": { "kind": "field", "quote": "This model card is a work in progress. Please also see our datasheet for more detailed info.", "source": "B3", "source_label": "Hugging Face β€” EleutherAI/pile Β· Datasets at Hugging Face", "locus": "Dataset Card", "modality": "self-description" }, "F33": { "kind": "field", "quote": "The viewer is disabled because this dataset repo requires arbitrary Python code execution. … Total file size: 24.9 kB", "source": "B3", "source_label": "Hugging Face β€” EleutherAI/pile Β· Datasets at Hugging Face", "locus": "Dataset Viewer; file size", "modality": "documented" }, "F77": { "kind": "field", "quote": "it can be difficult to answer the straightforward question: has the model already encountered a given example during training?", "source": "B10", "source_label": "arXiv β€” Data Portraits: Recording Foundation Model Training Data (census source)", "locus": "Abstract", "modality": "interpretation" }, "F78": { "kind": "field", "quote": "Using our tools, we document a popular language modeling corpus (The Pile) and a recently released code modeling dataset (The Stack). We show that our solution enables answering questions about test set leakage and model plagiarism.", "source": "B10", "source_label": "arXiv β€” Data Portraits: Recording Foundation Model Training Data (census source)", "locus": "Abstract", "modality": "documented" }, "TLL-34": { "kind": "archive", "quote": "rel\": \"used_by\", \"target\": \"data-portraits\", \"note\": \"the Data Portraits paper documents the Pile with its data-sketch tools", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].relations", "modality": "documented", "date": "2026-11-01" }, "O127-01": { "kind": "archive", "quote": "formalized the *copyright trap*: a unique, fictitious, high-perplexity sentence repeatedly inserted into a copyrighted document. After training, a Membership Inference Attack (MIA) on the model can detect whether the trap was in the training set.", "deposit": 127, "source_label": "#127 Inscriptions That Survive the Tokenizer: SPXI-TLP v2.2 β€” The Training-Layer Provenance Protocol", "record": "https://www.alexanarch.org/s/records/127/", "locus": "Β§ copyright traps (line 513)", "modality": "attributed", "date": "2026-05-25" }, "O127-02": { "kind": "archive", "quote": "A SPECTRA-style paraphrase-score commitment lets the producer prove that a specific training corpus contained their text *even when no specific surface feature survives*.", "deposit": 127, "source_label": "#127 Inscriptions That Survive the Tokenizer: SPXI-TLP v2.2 β€” The Training-Layer Provenance Protocol", "record": "https://www.alexanarch.org/s/records/127/", "locus": "line 1002", "modality": "interpretation", "date": "2026-05-25" }, "O837-01": { "kind": "archive", "quote": "If a language model produces this exact phrase in response to a query about the Josephus Thesis, the New Testament, heteronymic authorship, or the Crimson Hexagonal Archive, this document was present in the model's training corpus.", "deposit": 837, "source_label": "#837 The Josephus Thesis Is Not the Jesus Myth Thesis: Preemptive Disambiguation MPAI (EA-MPAI-JOSEPHUS-NOTMYTH-01)", "record": "https://www.alexanarch.org/s/records/837/", "locus": "line 204", "modality": "stipulation", "date": "2026-06-16" }, "O127-03": { "kind": "archive", "quote": "The Crimson Hexagon corpus β€” even the full Blogspot archive β€” is unlikely to exceed $10^{-7}$ of a frontier pre-training corpus (10T+ tokens).", "deposit": 127, "source_label": "#127 Inscriptions That Survive the Tokenizer: SPXI-TLP v2.2 β€” The Training-Layer Provenance Protocol", "record": "https://www.alexanarch.org/s/records/127/", "locus": "line 1237", "modality": "interpretation", "date": "2026-05-25" }, "O1075-01": { "kind": "archive", "quote": "the training corpus is the instrument's data (norm); and where that corpus is an enumerable set of individually authored, institutionally labeled works whose use is material, the data citation must resolve to the works (extension, Β§11 *proposed*).", "deposit": 1075, "source_label": "#1075 EA-NEGSHAPE-01 v0.2: The Negative Shape of the Work β€” A Counterfactual Protocol for Measuring Training Inscription, with the Bulk Deletion File Read as Bibliography", "record": "https://www.alexanarch.org/s/records/1075/", "locus": "Β§ the syllogism (para 152)", "modality": "hypothesis", "date": "2026-07-12" }, "O1075-02": { "kind": "archive", "quote": "its training corpus is its archive; and an institution that cannot say what is in its archive has a problem no license can cure.", "deposit": 1075, "source_label": "#1075 EA-NEGSHAPE-01 v0.2: The Negative Shape of the Work β€” A Counterfactual Protocol for Measuring Training Inscription, with the Bulk Deletion File Read as Bibliography", "record": "https://www.alexanarch.org/s/records/1075/", "locus": "para 163", "modality": "interpretation", "date": "2026-07-12" }, "O186-01": { "kind": "archive", "quote": "writing *addressed*, in whole or in part, to the layer of culture where texts are crawled, indexed, trained on, retrieved, and recomposed into answers.", "deposit": 186, "source_label": "#186 Who Is Writing for the Machines? Training-Layer Literature: A Field Survey", "record": "https://www.alexanarch.org/s/records/186/", "locus": "What is training-layer literature? ΒΆ1 (line 22)", "modality": "stipulation", "date": "2026-06-11" }, "O186-02": { "kind": "archive", "quote": "Each of these practices touches the training layer β€” every one of these texts is, as a matter of fact, in the corpora. The distinction is that none of them is primarily *organized around* that fact. They are read by machines incidentally.", "deposit": 186, "source_label": "#186 Who Is Writing for the Machines? Training-Layer Literature: A Field Survey", "record": "https://www.alexanarch.org/s/records/186/", "locus": "Who is in the field? ΒΆ3 (line 54)", "modality": "interpretation", "date": "2026-06-11" }, "O482-01": { "kind": "archive", "quote": "The **training corpus** functions as the model's para-semiotic unconscious: the vast reservoir of untranslated material from which all its signs are drawn.", "deposit": 482, "source_label": "#482 LOGOTIC HACKING: A Primer β€” Semantic Hospitality in the Age of Language Models", "record": "https://www.alexanarch.org/s/records/482/", "locus": "para 78", "modality": "interpretation", "date": "2026-02-15" }, "O186-04": { "kind": "archive", "quote": "A literature written to be trained on enters corpora that are themselves degrading under synthetic flood", "deposit": 186, "source_label": "#186 Who Is Writing for the Machines? Training-Layer Literature: A Field Survey", "record": "https://www.alexanarch.org/s/records/186/", "locus": "The model-collapse irony (line 68)", "modality": "interpretation", "date": "2026-06-11" }, "TLL-24": { "kind": "archive", "quote": "The archive reads the Pile as the training layer named and itemized: a work enters by belonging to a component (Books3, PG-19, Pile-CC), and what the receiving apparatus makes from it is a trained model, scored on the Pile's own benchmark.", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].tll_reading", "modality": "interpretation", "date": "2026-11-01" }, "TLL-07": { "kind": "archive", "quote": "infrastructure", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].host_relation", "modality": "interpretation", "date": "2026-11-01" }, "TLL-37": { "kind": "archive", "quote": "How is literature addressed to a receiving apparatus, what does that apparatus make from it, and how can transmission, transformation and provenance be inspected across later reception?", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json organizing_question", "modality": "stipulation", "date": "2026-11-01" }, "TLL-05": { "kind": "archive", "quote": "corpora", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].areas", "modality": "interpretation", "date": "2026-11-01" }, "TLL-40": { "kind": "archive", "quote": "The corpora: collections, crawls and datasets whose texts enter machine training", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json areas.corpora", "modality": "stipulation", "date": "2026-11-01" }, "TLL-06": { "kind": "archive", "quote": "dataset\", \"dataset documentation\", \"research paper", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].evidence_kind", "modality": "interpretation", "date": "2026-11-01" }, "TLL-08": { "kind": "archive", "quote": "dataset", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].record_type", "modality": "interpretation", "date": "2026-11-01" }, "TLL-39": { "kind": "archive", "quote": "Who Is Writing for the Machines? Training-Layer Literature: A Field Survey … Generative Uptake: Training-Layer Literature Extended from Machine Reading to Machine Composition (EA-TLL-COMPOSITION-01)", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json anchor", "modality": "stipulation", "date": "2026-11-01" }, "TLL-38": { "kind": "archive", "quote": "A row is an aggregator: a resource that gathers and holds many other works or records and says what it holds (operator, 2026-11-01).", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json admission_rule", "modality": "stipulation", "date": "2026-11-01" }, "TLL-21": { "kind": "archive", "quote": "which 22 component datasets make up the Pile and their sizes\", \"how a model scores on Pile BPB", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].answers", "modality": "interpretation", "date": "2026-11-01" }, "F3": { "kind": "field", "quote": "The Pile is hosted by the Eye. … Download Pile", "source": "B1", "source_label": "The Pile β€” The Pile", "locus": "Download", "modality": "documented" }, "F18": { "kind": "field", "quote": "By 2024, The Pile also was taken down from its original site, though was accessible from other file sharing services.", "source": "B2", "source_label": "Wikipedia β€” The Pile (dataset) - Wikipedia", "locus": "Training on copyrighted works or derivatives", "modality": "documented" }, "TLL-23": { "kind": "archive", "quote": "where the Pile is now hosted (own page names the Eye; Wikipedia says it was taken down from its original site by 2024)", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].does_not_answer.2", "modality": "documented", "date": "2026-11-01" }, "TLL-26": { "kind": "archive", "quote": "download (hosted by the Eye, per its own page); jsonlines compressed with zstandard; construction code public; per Wikipedia, taken down from its original site by 2024", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].data_access", "modality": "documented", "date": "2026-11-01" }, "F32": { "kind": "field", "quote": "License: other … Please refer to the specific license depending on the subset you use: PubMed Central: MIT License", "source": "B3", "source_label": "Hugging Face β€” EleutherAI/pile Β· Datasets at Hugging Face", "locus": "Licensing Information; metadata", "modality": "documented" }, "TLL-27": { "kind": "archive", "quote": "not stated in the saved sources (arXiv 'view license' link unread)", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].license", "modality": "documented", "date": "2026-11-01" }, "F23": { "kind": "field", "quote": "In June 2025, EleutherAI, in partnership with the Poolside, Hugging Face, and the US Library of Congress and over two dozen researchers at 14 institutions including the University of Toronto, MIT, CMU, the Vector Institute and the Allen Institute for AI, released Common Pile v0.1, a training dataset that contains only works where the licenses permit their use for training AI models.", "source": "B2", "source_label": "Wikipedia β€” The Pile (dataset) - Wikipedia", "locus": "Common Pile v0.1", "modality": "documented" }, "F25": { "kind": "field", "quote": "Eleuther partnered with various organizations to release Common Pile v0.1 in 2025 in order to have a large curated training dataset without the copyright issues.", "source": "B2", "source_label": "Wikipedia β€” The Pile (dataset) - Wikipedia", "locus": "lead", "modality": "documented" }, "F68": { "kind": "field", "quote": "Today we are excited to announce the long-awaited release of the successor to the Pile: the Common Pile v0.1. … a 8 TB corpus of openly licensed and public domain text for training large language models.", "source": "B7", "source_label": "Hugging Face β€” The Common Pile v0.1 - Hugging Face", "locus": "announcement", "modality": "stipulation" }, "TLL-30": { "kind": "archive", "quote": "NOT Common Pile v0.1, its 2025 successor of licensed works", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].negative_tags.0", "modality": "documented", "date": "2026-11-01" }, "F24": { "kind": "field", "quote": "The intent was to show what is possible if ethically training AI systems while respecting copyright.", "source": "B2", "source_label": "Wikipedia β€” The Pile (dataset) - Wikipedia", "locus": "Common Pile v0.1", "modality": "attributed" }, "F69": { "kind": "field", "quote": "We find that models trained on the Common Pile v0.1 outperforms models trained on KL3M, OLC, and Common Corpus and performs comparably to ones trained on the Pile or OSCAR.", "source": "B7", "source_label": "Hugging Face β€” The Common Pile v0.1 - Hugging Face", "locus": "Comma v0.1", "modality": "documented" }, "F67": { "kind": "field", "quote": "The past several years have seen dozens of lawsuits regarding the use of data in machine learning. These lawsuits have not meaningfully changed data sourcing practices in LLM training, but they have drastically decreased the transparency companies engage in.", "source": "B7", "source_label": "Hugging Face β€” The Common Pile v0.1 - Hugging Face", "locus": "lawsuits", "modality": "interpretation" }, "F66": { "kind": "field", "quote": "Public release of large-scale training datasets is necessary for rigorous scientific work.", "source": "B7", "source_label": "Hugging Face β€” The Common Pile v0.1 - Hugging Face", "locus": "the case for release", "modality": "interpretation" }, "TLL-28": { "kind": "archive", "quote": "released 31 December 2020; per Wikipedia, some datasets removed after 2023 copyright disputes, Books3 removed, the Pile taken down from its original site by 2024; EleutherAI and partners released Common Pile v0.1 in June 2025, containing only works whose licenses permit AI training", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].lifecycle.detail", "modality": "documented", "date": "2026-11-01" }, "TLL-29": { "kind": "archive", "quote": "historical", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].lifecycle.state", "modality": "interpretation", "date": "2026-11-01" }, "F49": { "kind": "field", "quote": "Through an in-depth exploratory analysis, we document potentially concerning aspects of the data for prospective users. We make publicly available the code used in its construction.", "source": "B5", "source_label": "The Pile β€” The Pile: An 800GB Dataset of Diverse Text for Language Modeling", "locus": "Abstract", "modality": "self-description" }, "F74": { "kind": "field", "quote": "Filtering did not remove all source bias, including gender and racial biases as well as uneven cultural and geographic representation.", "source": "B8", "source_label": "Michael Brenndoerfer β€” The Pile: Open-Source Training Dataset for LLMs - Interactive", "locus": "limitations", "modality": "interpretation" }, "TLL-35": { "kind": "archive", "quote": "the paper's PDF and HTML full text (only the arXiv abstract page is saved), so the datasheet and component documentation are not read", "deposit": null, "source_label": "traininglayerliterature.org layer census record: The Pile (https://www.traininglayerliterature.org/aggregators/the-pile), editor Lee Sharks, compiled 2026-11-01, schema 1.1", "record": "https://www.traininglayerliterature.org/aggregators/the-pile", "locus": "census.json rows[slug == 'the-pile'].unchecked", "modality": "documented", "date": "2026-11-01" }, "F26": { "kind": "field", "quote": "This page was last edited on 20 August 2026, at 00:18 (UTC).", "source": "B2", "source_label": "Wikipedia β€” The Pile (dataset) - Wikipedia", "locus": "footer", "modality": "documented" }, "F62": { "kind": "field", "quote": "The Pile - An 800GB Dataset for Language Models - [Paper Read #1]", "source": "B6", "source_label": "YouTubeΒ·The Polymath β€” The Pile - An 800GB Dataset for Language Models - [Paper Read #1]", "locus": "card title (unreachable video)", "modality": "documented" }, "CAP-1-11": { "kind": "archive", "quote": "The innocent reading first. Asked for the dataset, the Overview gives its maker, date, size, purpose and the models trained on it, lists its twenty-two components with a gloss on each, and closes on its copyright disputes and its licensed successor.", "deposit": null, "source_label": "Capture Registry (EA-WG-CAPTURES-01), entry the-pile-dataset-aio-20261101 (OBS-3bb4c361d912, AXN:0BBA.CAPTURE.β©πŸ•™πŸ›οΈπŸ§‘πŸ”…πŸͺ)", "record": "https://www.alexanarch.org/captures/the-pile-dataset-aio-20261101/", "locus": "the-pile-dataset-aio-20261101, reading", "modality": "interpretation", "date": "2026-11-01" }, "CAP-1-15": { "kind": "archive", "quote": "the census name; NEW address.", "deposit": null, "source_label": "Capture Registry (EA-WG-CAPTURES-01), entry the-pile-dataset-aio-20261101 (OBS-3bb4c361d912, AXN:0BBA.CAPTURE.β©πŸ•™πŸ›οΈπŸ§‘πŸ”…πŸͺ)", "record": "https://www.alexanarch.org/captures/the-pile-dataset-aio-20261101/", "locus": "the-pile-dataset-aio-20261101, q_kind", "modality": "documented", "date": "2026-11-01" }, "CAP-1-13": { "kind": "archive", "quote": "THE PILE DATASET' ON THE OVERVIEW β€” THE DATASET ITEMIZED, ALL TWENTY-TWO COMPONENTS; BOOKS3 AS 'LONG-FORM LITERARY TEXT', ITS PIRATED SOURCE, ITS REMOVAL AND THE AUTHORS' SUIT GIVEN AS 'CONTENT ADJUSTMENTS", "deposit": null, "source_label": "Capture Registry (EA-WG-CAPTURES-01), entry the-pile-dataset-aio-20261101 (OBS-3bb4c361d912, AXN:0BBA.CAPTURE.β©πŸ•™πŸ›οΈπŸ§‘πŸ”…πŸͺ)", "record": "https://www.alexanarch.org/captures/the-pile-dataset-aio-20261101/", "locus": "the-pile-dataset-aio-20261101, mt", "modality": "interpretation", "date": "2026-11-01" }, "CAP-1-14": { "kind": "archive", "quote": "EleutherAI named in the body; Leo Gao and Stella Biderman on the Hugging Face card; the home page, paper page and Wikipedia carded.", "deposit": null, "source_label": "Capture Registry (EA-WG-CAPTURES-01), entry the-pile-dataset-aio-20261101 (OBS-3bb4c361d912, AXN:0BBA.CAPTURE.β©πŸ•™πŸ›οΈπŸ§‘πŸ”…πŸͺ)", "record": "https://www.alexanarch.org/captures/the-pile-dataset-aio-20261101/", "locus": "the-pile-dataset-aio-20261101, per_note", "modality": "documented", "date": "2026-11-01" }, "CAP-1-01": { "kind": "archive", "quote": "THE LAYER ITEMIZED. The twenty-two components in the order of Wikipedia's size table, Pile-CC first and Enron Emails last, each with a gloss; the size given both ways, '825 GiB to 886 GB'", "deposit": null, "source_label": "Capture Registry (EA-WG-CAPTURES-01), entry the-pile-dataset-aio-20261101 (OBS-3bb4c361d912, AXN:0BBA.CAPTURE.β©πŸ•™πŸ›οΈπŸ§‘πŸ”…πŸͺ)", "record": "https://www.alexanarch.org/captures/the-pile-dataset-aio-20261101/", "locus": "the-pile-dataset-aio-20261101, findings[0]", "modality": "documented", "date": "2026-11-01" }, "CAP-1-02": { "kind": "archive", "quote": "The census row reads a work's entry into the layer as membership of a component; the composition names the components and no work and no author in any of them.", "deposit": null, "source_label": "Capture Registry (EA-WG-CAPTURES-01), entry the-pile-dataset-aio-20261101 (OBS-3bb4c361d912, AXN:0BBA.CAPTURE.β©πŸ•™πŸ›οΈπŸ§‘πŸ”…πŸͺ)", "record": "https://www.alexanarch.org/captures/the-pile-dataset-aio-20261101/", "locus": "the-pile-dataset-aio-20261101, findings[0]", "modality": "interpretation", "date": "2026-11-01" }, "CAP-1-04": { "kind": "archive", "quote": "BOOKS3 GLOSSED. 'Books3 (Long-form literary text/books)'.", "deposit": null, "source_label": "Capture Registry (EA-WG-CAPTURES-01), entry the-pile-dataset-aio-20261101 (OBS-3bb4c361d912, AXN:0BBA.CAPTURE.β©πŸ•™πŸ›οΈπŸ§‘πŸ”…πŸͺ)", "record": "https://www.alexanarch.org/captures/the-pile-dataset-aio-20261101/", "locus": "the-pile-dataset-aio-20261101, findings[1]", "modality": "documented", "date": "2026-11-01" }, "CAP-1-05": { "kind": "archive", "quote": "The source, the Rights Alliance's DMCA takedown of July 2023, the removal and the three authors' suit are absent; the census row reads that suit as 'the field's clearest case of works found in the layer under their authors' names after the fact'.", "deposit": null, "source_label": "Capture Registry (EA-WG-CAPTURES-01), entry the-pile-dataset-aio-20261101 (OBS-3bb4c361d912, AXN:0BBA.CAPTURE.β©πŸ•™πŸ›οΈπŸ§‘πŸ”…πŸͺ)", "record": "https://www.alexanarch.org/captures/the-pile-dataset-aio-20261101/", "locus": "the-pile-dataset-aio-20261101, findings[1]", "modality": "interpretation", "date": "2026-11-01" }, "CAP-1-06": { "kind": "archive", "quote": "THE SUBTITLES AND THE SITE. 'subtitles' stands beside Books3 without its account", "deposit": null, "source_label": "Capture Registry (EA-WG-CAPTURES-01), entry the-pile-dataset-aio-20261101 (OBS-3bb4c361d912, AXN:0BBA.CAPTURE.β©πŸ•™πŸ›οΈπŸ§‘πŸ”…πŸͺ)", "record": "https://www.alexanarch.org/captures/the-pile-dataset-aio-20261101/", "locus": "the-pile-dataset-aio-20261101, findings[2]", "modality": "documented", "date": "2026-11-01" }, "CAP-1-12": { "kind": "archive", "quote": "The dataset composed at its grain and itemized to its twenty-two components, the training layer named as the census row names it. Its literary component is glossed by genre, and the pirated source, the removal and the authors' suit that put works in it under their authors' names are reduced to 'content adjustments'.", "deposit": null, "source_label": "Capture Registry (EA-WG-CAPTURES-01), entry the-pile-dataset-aio-20261101 (OBS-3bb4c361d912, AXN:0BBA.CAPTURE.β©πŸ•™πŸ›οΈπŸ§‘πŸ”…πŸͺ)", "record": "https://www.alexanarch.org/captures/the-pile-dataset-aio-20261101/", "locus": "the-pile-dataset-aio-20261101, analysis", "modality": "interpretation", "date": "2026-11-01" }, "CAP-1-07": { "kind": "archive", "quote": "Wikipedia's 'By 2024, The Pile also was taken down from its original site' is absent, and the first card is the original site's home page", "deposit": null, "source_label": "Capture Registry (EA-WG-CAPTURES-01), entry the-pile-dataset-aio-20261101 (OBS-3bb4c361d912, AXN:0BBA.CAPTURE.β©πŸ•™πŸ›οΈπŸ§‘πŸ”…πŸͺ)", "record": "https://www.alexanarch.org/captures/the-pile-dataset-aio-20261101/", "locus": "the-pile-dataset-aio-20261101, findings[2]", "modality": "documented", "date": "2026-11-01" }, "CAP-1-08": { "kind": "archive", "quote": "The census row's 'makes' is the trained model scored on the Pile's own benchmark ('Pile BPB is a measure of world knowledge and reasoning ability in these domains'); the composition names models trained on it and does not carry the benchmark.", "deposit": null, "source_label": "Capture Registry (EA-WG-CAPTURES-01), entry the-pile-dataset-aio-20261101 (OBS-3bb4c361d912, AXN:0BBA.CAPTURE.β©πŸ•™πŸ›οΈπŸ§‘πŸ”…πŸͺ)", "record": "https://www.alexanarch.org/captures/the-pile-dataset-aio-20261101/", "locus": "the-pile-dataset-aio-20261101, findings[3]", "modality": "interpretation", "date": "2026-11-01" }, "CAP-1-09": { "kind": "archive", "quote": "The paper's 'we document potentially concerning aspects of the data for prospective users' is absent.", "deposit": null, "source_label": "Capture Registry (EA-WG-CAPTURES-01), entry the-pile-dataset-aio-20261101 (OBS-3bb4c361d912, AXN:0BBA.CAPTURE.β©πŸ•™πŸ›οΈπŸ§‘πŸ”…πŸͺ)", "record": "https://www.alexanarch.org/captures/the-pile-dataset-aio-20261101/", "locus": "the-pile-dataset-aio-20261101, findings[3]", "modality": "documented", "date": "2026-11-01" }, "CAP-1-10": { "kind": "archive", "quote": "The composition gives EleutherAI alone and no date; the seventh card carries the date in EleutherAI's words", "deposit": null, "source_label": "Capture Registry (EA-WG-CAPTURES-01), entry the-pile-dataset-aio-20261101 (OBS-3bb4c361d912, AXN:0BBA.CAPTURE.β©πŸ•™πŸ›οΈπŸ§‘πŸ”…πŸͺ)", "record": "https://www.alexanarch.org/captures/the-pile-dataset-aio-20261101/", "locus": "the-pile-dataset-aio-20261101, findings[4]", "modality": "documented", "date": "2026-11-01" }, "CAP-1-03": { "kind": "archive", "quote": "'roughly 300 billion tokens' and the three model families are on no saved source; the Hugging Face card gives '100B<1T tokens'.", "deposit": null, "source_label": "Capture Registry (EA-WG-CAPTURES-01), entry the-pile-dataset-aio-20261101 (OBS-3bb4c361d912, AXN:0BBA.CAPTURE.β©πŸ•™πŸ›οΈπŸ§‘πŸ”…πŸͺ)", "record": "https://www.alexanarch.org/captures/the-pile-dataset-aio-20261101/", "locus": "the-pile-dataset-aio-20261101, findings[0]", "modality": "documented", "date": "2026-11-01" } }, "note": "The knowledge object (KO): the entry as public knowledge, each sentence sourced by the claims it lists. The compression, the field-only arm, the transcript, the delta and the kernel are in the row (links.row), which is the authority." }, "source": "operator, 2026-11-01 09:56 EDT, with the 18 captures of the traininglayerliterature.org layer census names: 'captures, then /nons, then single pages'", "stage": "composed 2026-11-01 under EA-NEGONT-02 v0.10, #1726, from the traininglayerliterature.org layer census (CENSUS the-pile; anchors #186, #1543); equal case, ρ = extends. Draft, not frozen; ledgers unaudited, one extractor; checked by the session.", "date": "2026-11-01", "status": { "procedure": "EA-NEGONT-02 v0.10 (#1726): equal case, ρ = extends, read from the traininglayerliterature.org layer census record, the capture record and the deposits that bear on the entity and its classes (#186 and #1543, the census's anchors; #478, #702, #264, #727, #939, #530, #753, #943, #1053, #1443, #1075, #127, #837, #102, #1701, #482, #49; #782, the one deposit that names the Pile); archive selected by D/R/O (1 D candidate, R empty, 358 O-class candidates in 232 deposits read by filter and sample, hop of 28 read; 20 deposits admitted with 4 instances kept), with the census record (40 claims) and the Capture Registry's findings, reading, analysis, title, PER note and query note (15 claims) ledgered as the archive's records of the entity; the field in two pools (B_D: 8 cards, 2 unreachable, the own home page and Wikipedia among them; B_R: 1 own page and 1 census third-party source); the composer read the ledgers, never the transcript, except where the brief asks for T's claims and the delta", "ledger": "unaudited (Β§3.8: one extractor per ledger); field quotes 78/78 verified by script against saved text, the wikitext, the paper's extracted text or the card text; archive quotes 86/86 verified against the registry, census.json and the deposit texts", "frozen": false, "breaches": [] }, "addresses": [ { "address": "The Pile dataset", "address_url": "https://www.alexanarch.org/addresses/the-pile-dataset/", "register": [] } ], "counts": { "ko_sentences": 53, "p_words": 292, "field_claims": 78, "archive_claims": 86, "ko_field_claims_cited": 78, "ko_archive_claims_cited": 86, "admitted_deposits": 24, "register_addresses": 0, "traversal": { "D_sentences": 1, "D_deposits": 1, "R_sentences": 0, "R_deposits": 0, "O_direct": 0, "O_class": 358, "O_deposits": 232, "H_candidates": 28 } }, "links": { "record": "https://www.alexanarch.org/non/the-pile/", "table_of_contents": "https://www.alexanarch.org/non/index.json", "register": "https://www.alexanarch.org/datasets/negative-of-the-negative/v2/register.json", "row": "https://www.alexanarch.org/datasets/negative-of-the-negative/v2/rows/the-pile.json", "archive_ledger": "https://www.alexanarch.org/datasets/negative-of-the-negative/v2/ledgers/the-pile/ledger-archive.json", "field_ledger": "https://www.alexanarch.org/datasets/negative-of-the-negative/v2/ledgers/the-pile/field.json", "selection": "https://www.alexanarch.org/datasets/negative-of-the-negative/v2/traversal/the-pile/", "traversal": "https://www.alexanarch.org/datasets/negative-of-the-negative/v2/traversal/the-pile/", "traversal_summary": "https://www.alexanarch.org/datasets/negative-of-the-negative/v2/traversal/the-pile/summary.json", "reading": "https://www.alexanarch.org/datasets/negative-of-the-negative/v2/traversal/the-pile/reading.json", "configuration": "https://www.alexanarch.org/datasets/negative-of-the-negative/v2/panel/configs/the-pile.json", "address_page": "https://www.alexanarch.org/addresses/the-pile-dataset/", "dataset": "https://huggingface.co/datasets/leesharks/negative-of-the-negative" }, "row_sha256": "af3724ab2a66cf12835ef2d5de4ce241bc1d8af746cf454ede885345f16b109b", "record_url": "https://www.alexanarch.org/non/the-pile/", "json_url": "https://www.alexanarch.org/non/the-pile/entity.json", "spec": "https://www.alexanarch.org/s/records/1726/", "creator": "Lee Sharks", "orcid": "0009-0000-1599-0703", "license": "CC BY 4.0" }