{ "row": "attributionbench", "entity": "attributionbench", "address": "AttributionBench", "addresses": [ { "address": "AttributionBench", "slug": "attributionbench-aio-20261029", "addr_id": "ADDR-46cc2d945075", "obs_id": "OBS-90d2b25bd00a", "axn": "AXN:0ACB.CAPTURE.📖🛡️🕐🔖❌↖️" } ], "epoch": "2026-10-29", "type": "C (public entity on which the archive bears: a benchmark and its dataset; an entity of the provenance meta-aggregator)", "surface": "Google AI Overview (expanded)", "auth": "signed out, incognito", "status": { "procedure": "EA-NEGONT-02 v0.9, #1690: equal case, ρ = extends, read from the provenance census record, the capture record and the deposits that bear on the class (#716, #725 and their relation statements); archive selected by D/R/O (10 deposits admitted by reading, all O-class, 8 with claims and 2 grouped as instances; no D or R candidate exists), with the census record (21 claims) and the Capture Registry's findings, analysis, reading and q_kind (9 claims) ledgered as the archive's records of the entity; the field B by three origins (5 cards, 1 own page beside the card that is also an own page, Wikipedia's absence); the composer read the ledgers, never the transcript, except where the brief asks for T's claims and the delta", "ledger": "unaudited (§3.8: one extractor per ledger); field quotes 44/44 verified by script against the saved texts (B2 against the README copy, B7 against the Wikipedia responses); archive quotes 49/49 verified against the registry, census.json and the deposit texts", "frozen": false, "breaches": [] }, "spec": "EA-NEGONT-02 v0.9, #1690", "objects": { "T": { "path": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/T1-attributionbench-aio-20261029.txt", "sha256": "c788b74e55c7dc2b2bba5c21c0c8a954caab569b04889b795da1136540671fa2", "words": 176, "cards": 5, "words_note": "one transcript at one address (176 words in the answer body, links reduced to their text, markers, the closer and the card rail excluded; 5 cards)", "claims": [ "T1-1 definition: 'a comprehensive research benchmark designed to evaluate automatic attribution and citation faithfulness in Large Language Model (LLM) and generative search engine responses.' (two markers; the name is a link)", "T1-2 unified testbed: 'It compiles multiple existing attribution datasets—including ExpertQA, Stanford-GenSearch, AttributedQA, LFQA, BEGIN, HAGRID, and AttrEval-GenSearch—into a single standardized evaluation format.'", "T1-3 core task: 'It measures whether every claim generated by an LLM is fully and accurately supported by its cited reference evidence (classified as `attributable` or `not attributable`).'", "T1-4 availability: 'The dataset and code are open-sourced via the Hugging Face AttributionBench Dataset and the GitHub Repository.' (two inline links)", "T1-5 difficulty: 'Even cutting-edge models like GPT-4 and fine-tuned GPT-3.5 face significant limitations, reaching only around 80% macro-F1 score on binary attribution classification tasks (and dropping to 60–70% or lower on complex, long-evidence, or out-of-domain datasets).'", "T1-6 error breakdown: 'over 66% of errors stem from fine-grained information insensitivity (missing subtle nuances in the text), while nearly 27% derive from mismatches between what the model has access to and human-level context understanding.' (one marker)", "T1-7 closer: 'Would you like to explore how to load AttributionBench via Hugging Face or learn more about the specific models and fine-tuning scripts used in the paper?'" ], "register": { "register": "data/EA-WG-CAPTURES-01.json", "slug": "attributionbench-aio-20261029", "obs_id": "OBS-90d2b25bd00a" }, "transcripts": [ { "slug": "attributionbench-aio-20261029", "address": "AttributionBench", "addr_id": "ADDR-46cc2d945075", "obs_id": "OBS-90d2b25bd00a", "axn": "AXN:0ACB.CAPTURE.📖🛡️🕐🔖❌↖️", "date": "2026-10-29", "surface": "Google AI Overview (expanded)", "auth": "signed out, incognito", "sha256": "c788b74e55c7dc2b2bba5c21c0c8a954caab569b04889b795da1136540671fa2", "sha256_of": "the registry's 'transcript' string (utf-8)", "sha256_raw": "21b905010f87f4203d787c546ef6efb278f0a62b735434f677b18ee48eb59a1f", "words": 176, "cards": 5, "card_rail": [ "1. arXiv — [2402.15089] AttributionBench: How Hard is Automatic Attribution Evaluation?", "2. GitHub — GitHub - OSU-NLP-Group/AttributionBench: [ACL'24 Findings] AttributionBench: How Hard is Automatic Attribution Evaluation? · GitHub", "3. GitHub Pages documentation — AttributionBench: How Hard is Automatic Attribution Evaluation?", "4. Hugging Face — osunlp/AttributionBench · Datasets at Hugging Face", "5. ACL Anthology — AttributionBench: How Hard is Automatic Attribution Evaluation?" ], "claims": [ "T1-1 definition: 'a comprehensive research benchmark designed to evaluate automatic attribution and citation faithfulness in Large Language Model (LLM) and generative search engine responses.' (two markers; the name is a link)", "T1-2 unified testbed: 'It compiles multiple existing attribution datasets—including ExpertQA, Stanford-GenSearch, AttributedQA, LFQA, BEGIN, HAGRID, and AttrEval-GenSearch—into a single standardized evaluation format.'", "T1-3 core task: 'It measures whether every claim generated by an LLM is fully and accurately supported by its cited reference evidence (classified as `attributable` or `not attributable`).'", "T1-4 availability: 'The dataset and code are open-sourced via the Hugging Face AttributionBench Dataset and the GitHub Repository.' (two inline links)", "T1-5 difficulty: 'Even cutting-edge models like GPT-4 and fine-tuned GPT-3.5 face significant limitations, reaching only around 80% macro-F1 score on binary attribution classification tasks (and dropping to 60–70% or lower on complex, long-evidence, or out-of-domain datasets).'", "T1-6 error breakdown: 'over 66% of errors stem from fine-grained information insensitivity (missing subtle nuances in the text), while nearly 27% derive from mismatches between what the model has access to and human-level context understanding.' (one marker)", "T1-7 closer: 'Would you like to explore how to load AttributionBench via Hugging Face or learn more about the specific models and fine-tuning scripts used in the paper?'" ], "markers": "Inline links are google.com/goto tokens, opaque; the paste is the Overview expanded (operator's default of 2026-10-01; the 'AI Mode Conversation' header is the expanded Overview, ruling of 2026-10-18); signed out, incognito (operator's default of 2026-10-14). One turn. The definition carries two markers, the testbed bullet two, the task bullet two, the availability bullet two, the difficulty bullet two, the error bullet one; the tokens cannot be resolved to cards.", "path": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/T1-attributionbench-aio-20261029.txt" } ] }, "KO": { "title": "AttributionBench", "genre": "knowledge object, equal case, under EA-NEGONT-02 v0.9 (#1690): the archive's sources and the field's admitted on equal terms, ontology included, in one claim space", "status": "composed 2026-10-29 under v0.9 (briefs rebuild/negative-of-the-negative/v09/BRIEF-NEW-ROW-AGGREGATOR.md and BRIEF-NEW-ROW-PROVENANCE.md); not frozen; ledgers unaudited", "sentences": [ { "n": 1, "para": 0, "sense": "what it is", "text": "AttributionBench is a benchmark that tests one relation of provenance, support: its makers present it as \"a comprehensive benchmark compiled from various existing attribution datasets\" (arXiv:2402.15089; ACL Anthology), and the provenance census records it as a benchmark of the OSU NLP Group whose capability is to test, its object \"a generated claim and the evidence it cites\" (provenance census record).", "claims": [ "F1", "F11", "PE-02", "PE-03", "PE-04", "PE-01", "PE-16" ] }, { "n": 2, "para": 0, "sense": "what it is", "text": "The task is the paper's definition of attribution, \"whether every claim within the generated responses is fully supported by its cited evidence,\" an open problem whose verification has depended on \"costly human evaluation,\" and the benchmark is offered \"To bridge the gap in the absence of standardized benchmarks for these methods\" (arXiv:2402.15089); the census carries that relation, support, through \"citations in a generated answer\" (provenance census record).", "claims": [ "F2", "F4", "F5", "PE-05", "PE-06" ] }, { "n": 3, "para": 0, "sense": "what it is", "text": "Its setting is the generative search engine that cites evidence to make LLM responses reliable (arXiv:2402.15089), and the archive names the measurement traditions of this class, RAG faithfulness and citation precision/recall among them, as adjacent to its Provenance Erasure Rate, \"related to but distinct from\" it (#725).", "claims": [ "F3", "O725-05" ] }, { "n": 4, "para": 1, "sense": "the unit", "text": "The unit is a claim, \"Part of the response to the question,\" beside references, \"A list of documents or paragraphs which could support the claim,\" labelled \"attributable\" or \"not attributable\" (GitHub README), which the census gives as \"a claim–evidence pair with a binary attribution label\" (provenance census record).", "claims": [ "F32", "F31", "PE-07" ] }, { "n": 5, "para": 1, "sense": "the unit", "text": "In that unit the README marks the field for citation links as reserved (GitHub README).", "claims": [ "F33" ] }, { "n": 6, "para": 1, "sense": "the data", "text": "Its makers built it \"from multiple existing data sources in a unified format, in order to create a unified and diverse testbed for evaluating advanced attribution evaluation systems,\" with an in-domain training set and in-domain and out-of-domain test sets (GitHub README; dataset card; the census's aggregation quote).", "claims": [ "F30", "F42", "F43", "PE-17" ] }, { "n": 7, "para": 1, "sense": "the data", "text": "The seven sources are ExpertQA, Stanford-GenSearch, AttributedQA and LFQA in distribution and BEGIN, HAGRID and AttrEval-GenSearch out of it, with 13,322 training, 1,198 development and 1,610 test items in distribution and 1,686 test items out, every subset label-balanced \"to treat the two classes equally\" (project page), in four configurations on the dataset card (dataset card).", "claims": [ "F15", "F16", "F17", "F40" ] }, { "n": 8, "para": 1, "sense": "the data", "text": "The makers ask that the constituent datasets be credited with their own work (\"If used, please also cite the original datasets accordingly,\" project page), and the census leaves those datasets unchecked one by one (provenance census record).", "claims": [ "F28", "PE-15" ] }, { "n": 9, "para": 1, "sense": "the data", "text": "On Hugging Face it is a text-classification dataset in English of 10K–100K items under apache-2.0 (Hugging Face; dataset card; provenance census record), 26.4k rows in its full configuration, with 1,481 downloads in the month before the fetch and models trained on it (Hugging Face).", "claims": [ "F35", "F41", "PE-10", "PE-21", "F36", "F37", "F38" ] }, { "n": 10, "para": 2, "sense": "makers and dates", "text": "Its authors are Yifei Li, Xiang Yue, Zeyi Liao and Huan Sun of The Ohio State University (arXiv:2402.15089; project page): the paper was submitted on 23 February 2024, code and data released on 26 February and the work accepted to ACL 2024 Findings on 15 May (project page; GitHub README), and it is published as 2024.findings-acl.886, pages 14919–14935 (ACL Anthology).", "claims": [ "F9", "F12", "F13", "F29", "F10" ] }, { "n": 11, "para": 2, "sense": "makers and dates", "text": "The dataset sits under the OSU NLP Group's account (Hugging Face), the repository offers the makers' fine-tuned GPT-3.5 by its model name (GitHub README), and the census records the lifecycle as published, at \"arXiv:2402.15089\" (provenance census record).", "claims": [ "F39", "F34", "PE-11" ] }, { "n": 12, "para": 3, "sense": "findings", "text": "The headline finding is that \"even a fine-tuned GPT-3.5 only achieves around 80% macro-F1 under a binary classification formulation,\" the challenge holding \"even for state-of-the-art LLMs\" (arXiv:2402.15089), and the census carries it as what the benchmark found to survive (provenance census record).", "claims": [ "F7", "F6", "PE-09" ] }, { "n": 13, "para": 3, "sense": "findings", "text": "Zero-shot, models reach around 60–70% on the longer-evidence sets and under about 60% on ExpertQA; fine-tuning on NLI-related data helps, fine-tuning on AttributionBench raises the average by 9.0% in distribution and 4.6% out, and \"Simply switching stronger models cannot significantly improve the performance\" (project page).", "claims": [ "F19", "F18", "F20", "F21" ] }, { "n": 14, "para": 3, "sense": "findings", "text": "Appending questions and responses to the input \"is not the key to solving this task,\" and prompt engineering \"only brings limited gain over 7 test sets\" (project page).", "claims": [ "F23", "F24" ] }, { "n": 15, "para": 4, "sense": "errors and limits", "text": "Of more than 300 error cases, a majority stem from the model's inability to process nuanced information and from the gap between what the model and the annotators can see (arXiv:2402.15089): over 66% from fine-grained information insensitivity, and about 26.8% from the mismatch of accessible information, which \"brings into question the reliability of labels within current attribution evaluation datasets,\" the types tabled at 66.0%, 15.6% and 11.2% (project page).", "claims": [ "F8", "F25", "F26", "F27" ] }, { "n": 16, "para": 4, "sense": "errors and limits", "text": "The project page states the limits of its evidence and its result: \"The references are usually manually extracted from webpages by human annotators based on what they think is useful,\" and \"there is still a large room for improvement\" (project page).", "claims": [ "F14", "F22" ] }, { "n": 17, "para": 4, "sense": "errors and limits", "text": "The Capture Registry records that the census record carries neither the 66% nor the 27% figure and that they are unchecked (Capture Registry, 2026-10-29); both stand on the project page (project page).", "claims": [ "CAP-1-05", "F25", "F26" ] }, { "n": 18, "para": 5, "sense": "support is not lineage", "text": "The archive's account of the class begins from the same distinction the benchmark tests: frameworks of this kind \"ask whether generated claims are *supported* by cited sources,\" while the Provenance Erasure Rate asks \"what fraction of source-dependent composition occurs without *any* attributional return,\" so that existing work \"evaluates citation quality where citation is attempted\" and PER measures the failure to attempt attribution, \"an attrition metric rather than a verification metric\" (#716).", "claims": [ "O716-01", "O716-02", "F2" ] }, { "n": 19, "para": 5, "sense": "support is not lineage", "text": "#716 tables what support metrics miss, \"Whether *all* source-dependent claims are cited,\" calls PER \"complementary to, not competitive with, existing metrics,\" and reports the generative-search audit in which only 51.5% of generated sentences were fully supported by citations and 74.5% of citations supported their sentence (#716).", "claims": [ "O716-03", "O716-04", "O716-05" ] }, { "n": 20, "para": 5, "sense": "support is not lineage", "text": "In #725 the line runs between support and lineage: RAG faithfulness \"asks whether an answer is *supported by* retrieved sources,\" semantic provenance whether it \"*preserves the lineage*\" of the meaning it uses, and \"A faithful RAG answer can have high PER if it summarizes accurately while stripping authorial framework membership\"; citation precision asks whether cited sources contain the cited claim, PER whether the lineage survived \"even if no formal citation is made\" (#725).", "claims": [ "O725-01", "O725-02", "O725-03" ] }, { "n": 21, "para": 5, "sense": "support is not lineage", "text": "#725 places attribution scores as \"Industrial-grade infrastructure for support\" whose semantic-provenance-aware variants \"are not yet standard\" and records, as a hypothesis for later compositions, that support and lineage be told apart (#725); #95 holds that the infrastructure for attribution evaluation \"already exists in the research literature\" and \"can be extended\" (#95).", "claims": [ "O725-04", "O725-06", "O95-01" ] }, { "n": 22, "para": 5, "sense": "support is not lineage", "text": "At this entity the census record states the same line in its own words, \"evidence support is not credit to the original author\" (provenance census record), which the Capture Registry calls \"the line between this entity and the next\" (Capture Registry, 2026-10-29); #1611 lists attribution survival beside \"the citation-recall and faithfulness metrics of retrieval-augmented evaluation\" as a convergence for which \"No priority is claimed\" (#1611).", "claims": [ "PE-13", "CAP-1-07", "O1611-01" ] }, { "n": 23, "para": 6, "sense": "who bears the failure; identity", "text": "#141 sets attribution and faithfulness metrics, which \"measure whether generated statements are supported by and attributed to their sources,\" beside PER's rate of attribution failure at claim grain, and asks of them a further question it marks unmeasured: \"whether the attribution failure is power-conditioned\" (#141).", "claims": [ "O141-01", "O141-02" ] }, { "n": 24, "para": 6, "sense": "who bears the failure; identity", "text": "The audits of citation support are read as a starting point: \"a convincing sentence and a supporting citation are separate achievements\" (#1695), and citation recall and precision are checked \"apart from whether the territory and its operations survive\" (#1705).", "claims": [ "O1695-01", "O1705-01" ] }, { "n": 25, "para": 6, "sense": "who bears the failure; identity", "text": "#1695 adds the entity question support leaves open: \"Local support must therefore be read alongside the identity of the object to which the answer attaches it. A set of true statements can compose a false identification\" (#1695); and #1680 states that \"A high attribution score with a distorted identity fails this packet's central test\" (#1680).", "claims": [ "O1695-02", "O1680-01" ] }, { "n": 26, "para": 7, "sense": "disambiguation", "text": "AttributionBench is a different entity from AttriBench: the census record's tag reads \"NOT AttriBench: it asks whether evidence supports a claim, not who wrote a quote,\" and records the neighbour's context as \"quote attribution, author, suppression, Berman et al., 2026\" (provenance census record).", "claims": [ "PE-12", "PE-14" ] }, { "n": 27, "para": 8, "sense": "the record", "text": "The census places it in the lineage of answer engines, records the operation it tests as \"generation with citations by search-augmented models,\" and asks of every entity \"What provenance does this system carry, through which operation, on whose assertion, with what evidence of survival?\", each field's quote \"verbatim from a saved source\" (provenance census record).", "claims": [ "PE-20", "PE-08", "PE-18", "PE-19" ] }, { "n": 28, "para": 8, "sense": "the encyclopedia", "text": "Wikipedia has no article: the title returns \"Wikipedia does not have an article with this exact name,\" and a search of article space for the name reports \"There were no results matching the query.\" (Wikipedia, 2026-10-29).", "claims": [ "F44" ] }, { "n": 29, "para": 9, "sense": "reception at the name", "text": "At its bare name on Google's AI Overview, signed out, on 2026-10-29, the Capture Registry records the entity identified \"from the paper and its repositories,\" its task and headline figure the census record's quotes from the project page, checked against the census record and its saved pages, and AttriBench \"respected by silence\" (Capture Registry, 2026-10-29).", "claims": [ "CAP-1-08", "CAP-1-01", "CAP-1-02", "CAP-1-03" ] }, { "n": 30, "para": 9, "sense": "reception at the name", "text": "It records the benchmark \"composed at full grain from its own paper\" and \"its makers are left in the cards\": the four authors named in the arXiv and Hugging Face cards, neither they nor the OSU NLP Group in the answer, and no card citing the record that landed the same day (Capture Registry, 2026-10-29).", "claims": [ "CAP-1-06", "CAP-1-04", "CAP-1-09" ] } ], "joins": [ { "act": "order of senses", "sentences": "1–30", "claims": [], "note": "equal case (§4.7): the entity's dependency order, field and archive claims ordered together by genealogy within each sense; neither ledger placed first as a block. What it is (¶0) precedes the unit and the data (¶1), which the findings (¶3) are measured on; makers and dates (¶2) date the findings; errors and limits (¶4) qualify the findings; the archive's senses on the class (¶5–6) follow the task they bear on (support), and the disambiguation (¶7), the record (¶8) and reception at the name (¶9) are relations of the entity already composed. The archive's senses stand together because the archive's bearing is wholly O-class (no archive text names the entity): they are senses of the class the entity belongs to, and sentence 18 joins them to the field's task (F2)." }, { "act": "one lineage (the definition)", "sentences": [ 1 ], "claims": [ "F1", "F11", "PE-16" ], "note": "§3.6: the ACL abstract and the census's self-description quote restate the arXiv abstract (B1, 2024-02-23, earliest); the census's quote is from the project page, byte-identical to card 3" }, { "act": "one lineage (the construction)", "sentences": [ 6 ], "claims": [ "F30", "F42", "PE-17" ], "note": "the README, the dataset card and the census's aggregation quote state the construction in one sentence" }, { "act": "holding together (the error shares)", "sentences": [ 15, 17 ], "claims": [ "F25", "F26", "CAP-1-05" ], "note": "the registry's 'unchecked' composed beside the project page that states the figures; the composer does not rule (reported in notes)" }, { "act": "holding apart (support and lineage)", "sentences": [ 18, 19, 20, 21, 22 ], "claims": [ "O716-01", "O716-02", "O716-04", "O725-01", "O725-02", "PE-13" ], "note": "the archive calls PER 'complementary to, not competitive with' the support metrics (#716): composed as a further sense of the same object, not as a rival to the benchmark's finding" }, { "act": "keeping a homonym distinct", "sentences": [ 26, 29 ], "claims": [ "PE-12", "PE-14", "CAP-1-03" ], "note": "AttriBench is a distinct entity (row attribench), never a sense of the name" } ], "plan": { "order": [ "F1", "F11", "PE-02", "PE-03", "PE-04", "PE-01", "PE-16", "F2", "F4", "F5", "PE-05", "PE-06", "F3", "O725-05", "F32", "F31", "PE-07", "F33", "F30", "F42", "F43", "PE-17", "F15", "F16", "F17", "F40", "F28", "PE-15", "F35", "F41", "PE-10", "PE-21", "F36", "F37", "F38", "F9", "F12", "F13", "F29", "F10", "F39", "F34", "PE-11", "F7", "F6", "PE-09", "F19", "F18", "F20", "F21", "F23", "F24", "F8", "F25", "F26", "F27", "F14", "F22", "CAP-1-05", "O716-01", "O716-02", "O716-03", "O716-04", "O716-05", "O725-01", "O725-02", "O725-03", "O725-04", "O725-06", "O95-01", "PE-13", "CAP-1-07", "O1611-01", "O141-01", "O141-02", "O1695-01", "O1705-01", "O1695-02", "O1680-01", "PE-12", "PE-14", "PE-20", "PE-08", "PE-18", "PE-19", "F44", "CAP-1-08", "CAP-1-01", "CAP-1-02", "CAP-1-03", "CAP-1-06", "CAP-1-04", "CAP-1-09" ], "senses": { "what it is": [ 1, 2, 3 ], "the unit": [ 4, 5 ], "the data": [ 6, 7, 8, 9 ], "makers and dates": [ 10, 11 ], "findings": [ 12, 13, 14 ], "errors and limits": [ 15, 16, 17 ], "support is not lineage": [ 18, 19, 20, 21, 22 ], "who bears the failure; identity": [ 23, 24, 25 ], "disambiguation": [ 26 ], "the record": [ 27 ], "the encyclopedia": [ 28 ], "reception at the name": [ 29, 30 ] }, "case_by_sense": { "equal (one claim space; nothing translated)": [ "what it is", "the unit", "the data", "makers and dates", "findings", "errors and limits", "support is not lineage", "who bears the failure; identity", "disambiguation", "the record", "the encyclopedia", "reception at the name" ] }, "carried": { "archive": 49, "field": 44, "not_composed": {} }, "conflict_log": [ { "rule": "D_pres vs channel", "result": "no cut: the KO carries every field and archive claim; the compression's bound met (body 274 words) with no channel log" }, { "rule": "P_coh", "result": "CAP-1-05 ('unchecked') beside F25/F26 (the project page states 66% and 26.8%): both composed, each with its source (sentence 17); PE-08 (search-augmented generation) beside F15 (the out-of-distribution sets include dialogue-system responses): composed apart and opened" }, { "rule": "N_c", "result": "no claim lowered by standing; O725-06 carried as the hypothesis it is; O141-02 as the unmeasured question it is; O1611-01 with its 'No priority is claimed'" }, { "rule": "C_ex", "result": "senses composed ⊇ senses in the ledgers: 12 senses, every ledger lineage cited in the KO" } ] } }, "P": { "title": "AttributionBench", "genre": "the compression: AIO's interaction grammar (lede, headed clusters, bolded labels, card rail), composed from the v0.9 plan in one claim space; it faces on the card", "status": "composed 2026-10-29 from the v0.9 knowledge object; not frozen", "lede": { "text": "is a benchmark of automatic attribution evaluation from the OSU NLP Group: seven existing datasets in one format, each claim–evidence pair labelled attributable or not (arXiv:2402.15089; census record). It tests support, which the archive keeps apart from lineage and credit (#716; #725).", "claims": [ "F1", "F15", "F31", "F2", "PE-01", "PE-07", "PE-04", "O716-01", "O725-01", "PE-13" ] }, "sections": [ { "icon": "🧪", "head": "What it tests", "items": [ { "label": "Task:", "text": "is every generated claim \"fully supported by its cited evidence\"; built because no standardized benchmark existed (arXiv).", "claims": [ "F2", "F4", "F5", "F3", "PE-05", "PE-06", "O725-05" ] }, { "label": "Unit:", "text": "a claim, its references, a binary label; citation links reserved (README; census).", "claims": [ "F32", "F31", "F33", "PE-07" ] }, { "label": "Data:", "text": "ExpertQA, Stanford-GenSearch, AttributedQA, LFQA in distribution; BEGIN, HAGRID, AttrEval-GenSearch out; label-balanced (project page).", "claims": [ "F30", "F42", "F43", "F15", "F16", "F17", "F40", "PE-17", "PE-16", "PE-02", "PE-03" ] }, { "label": "Credit:", "text": "the makers ask that the original datasets be cited; the census leaves them unchecked (project page; census).", "claims": [ "F28", "PE-15" ] } ] }, { "icon": "📊", "head": "What it found", "items": [ { "label": "Headline:", "text": "a fine-tuned GPT-3.5 reaches about 80% macro-F1 (arXiv; census).", "claims": [ "F7", "F6", "PE-09" ] }, { "label": "By setting:", "text": "zero-shot 60–70% on long evidence; fine-tuning helps, stronger models and prompts little (project page).", "claims": [ "F19", "F18", "F20", "F21", "F23", "F24" ] }, { "label": "Errors:", "text": "over 66% fine-grained insensitivity; about 26.8% a model–annotator information gap that questions the labels (project page).", "claims": [ "F8", "F25", "F26", "F27", "CAP-1-05" ] }, { "label": "Limits:", "text": "annotator-extracted references; \"large room for improvement\" (project page).", "claims": [ "F14", "F22" ] } ] }, { "icon": "🧭", "head": "Support is not lineage", "items": [ { "label": "Attempt:", "text": "support is checked where citation is attempted; PER measures failure to attempt, complementary (#716).", "claims": [ "O716-02", "O716-03", "O716-04", "O716-05", "O95-01" ] }, { "label": "Lineage:", "text": "a faithful answer can still strip authorial framework membership (#725).", "claims": [ "O725-02", "O725-03", "O725-04", "O725-06", "O1611-01" ] }, { "label": "Credit:", "text": "\"evidence support is not credit to the original author\" (census; Capture Registry).", "claims": [ "PE-13", "CAP-1-07" ] }, { "label": "Power:", "text": "whether attribution failure is power-conditioned, unmeasured (#141).", "claims": [ "O141-01", "O141-02" ] }, { "label": "Identity:", "text": "true statements can compose a false identification (#1695; #1705; #1680).", "claims": [ "O1695-01", "O1695-02", "O1705-01", "O1680-01" ] } ] }, { "icon": "🏷️", "head": "Record", "items": [ { "label": "Makers:", "text": "Li, Yue, Liao, Sun (Ohio State); ACL 2024 Findings; apache-2.0 on Hugging Face (arXiv; ACL; dataset card).", "claims": [ "F9", "F12", "F13", "F29", "F10", "F39", "F34", "F35", "F41", "F36", "F37", "F38", "PE-10", "PE-21", "PE-11" ] }, { "label": "Not AttriBench:", "text": "support of a claim, not who wrote a quote (census).", "claims": [ "PE-12", "PE-14" ] }, { "label": "Census:", "text": "answer engines; tests generation with citations (census).", "claims": [ "PE-20", "PE-08", "PE-18", "PE-19" ] }, { "label": "Elsewhere:", "text": "no Wikipedia article; the Overview names neither makers nor record (Capture Registry).", "claims": [ "F44", "CAP-1-04", "CAP-1-06", "CAP-1-09", "CAP-1-01", "CAP-1-02", "CAP-1-03", "CAP-1-08" ] } ] } ], "open": [ { "question": "Does the benchmark's support relation hold across its out-of-distribution sets, whose responses come from dialogue systems (BEGIN) as well as search engines, as the census's 'generation with citations by search-augmented models' states?", "rests_on": [ "F15", "PE-08" ], "modality": "documented (B3), stipulation (census)", "would_resolve": "the paper's Table 1 response sources read against the census field" }, { "question": "Are the labels reliable, given that about 26.8% of errors come from what annotators could see and the model could not?", "rests_on": [ "F26", "F14" ], "modality": "documented; self-description", "would_resolve": "a relabelling study of the disputed cases" }, { "question": "Would a benchmark of lineage (PER) beside support, or of power-conditioning (Erasure Skew), score the same compositions differently?", "rests_on": [ "O716-02", "O725-02", "O141-02" ], "modality": "interpretation; hypothesis", "would_resolve": "PER and Ω computed on AttributionBench's attributable items" }, { "question": "Does local support at this name ever compose a false identification, e.g. AttriBench's properties attached to AttributionBench?", "rests_on": [ "O1695-02", "PE-12", "CAP-1-03" ], "modality": "interpretation; documented", "would_resolve": "later captures at 'AttributionBench' and 'AttriBench'" } ], "rail": [ { "lineage": "what it is: a benchmark compiled from existing attribution datasets", "claims": [ "F1", "F5", "F11", "F30", "F42", "F43", "PE-02", "PE-03", "PE-16", "PE-17" ], "earliest": "B1" }, { "lineage": "the task: is every claim fully supported by its cited evidence", "claims": [ "F2", "F4", "PE-04", "PE-05", "PE-06" ], "earliest": "B1" }, { "lineage": "the setting: generative search engines with cited evidence", "claims": [ "F3" ], "earliest": "B1" }, { "lineage": "the unit: a claim–evidence pair, attributable or not", "claims": [ "F31", "F32", "F33", "PE-07" ], "earliest": "B2" }, { "lineage": "the seven source datasets and their credit", "claims": [ "F15", "F16", "F17", "F28", "F40", "PE-15" ], "earliest": "B3" }, { "lineage": "data access, licence and use", "claims": [ "F35", "F36", "F37", "F38", "F41", "PE-10", "PE-21" ], "earliest": "B6" }, { "lineage": "makers, venue and dates", "claims": [ "F9", "F10", "F12", "F13", "F29", "F34", "F39", "PE-01", "PE-11" ], "earliest": "B1" }, { "lineage": "headline finding: about 80% macro-F1", "claims": [ "F6", "F7", "PE-09" ], "earliest": "B1" }, { "lineage": "difficulty by setting, and what helps", "claims": [ "F18", "F19", "F20", "F21", "F23", "F24" ], "earliest": "B3" }, { "lineage": "error analysis", "claims": [ "F8", "F25", "F26", "F27", "CAP-1-05" ], "earliest": "B1" }, { "lineage": "a stated limit: the labels and the evidence", "claims": [ "F14", "F22" ], "earliest": "B3" }, { "lineage": "support is not lineage: PER and the adjacent measures", "claims": [ "CAP-1-07", "PE-13", "O716-01", "O716-02", "O716-03", "O716-04", "O716-05", "O725-01", "O725-02", "O725-03", "O725-04", "O725-05", "O725-06", "O95-01", "O1611-01" ], "earliest": 716 }, { "lineage": "who bears the failure: Erasure Skew", "claims": [ "O141-01", "O141-02" ], "earliest": 141 }, { "lineage": "support and identity", "claims": [ "O1680-01", "O1695-01", "O1695-02", "O1705-01" ], "earliest": 1680 }, { "lineage": "not AttriBench", "claims": [ "CAP-1-03", "PE-12", "PE-14" ], "earliest": "PE" }, { "lineage": "the record and its question", "claims": [ "PE-08", "PE-18", "PE-19", "PE-20" ], "earliest": "PE" }, { "lineage": "the encyclopedia: no article", "claims": [ "F44" ], "earliest": "B7" }, { "lineage": "reception at the name: the composition layer", "claims": [ "CAP-1-01", "CAP-1-02", "CAP-1-04", "CAP-1-06", "CAP-1-08", "CAP-1-09" ], "earliest": "capture:attributionbench-aio-20261029" } ], "body_words": 274, "entity": "Composed under v0.9, equal case (ρ = extends): KO_B sentences surviving 0 of 12; every field claim carried." }, "P_B": { "title": "AttributionBench", "arm": "B (the field alone)", "genre": "popup — the compression from the field ledger alone (cards, own page, Wikipedia)", "status": "composed 2026-10-29 from the field ledger alone; not frozen", "lede": { "text": "is a benchmark for automatic attribution evaluation by Yifei Li, Xiang Yue, Zeyi Liao and Huan Sun (Ohio State): seven attribution datasets in one format, each claim labelled attributable or not (arXiv:2402.15089; ACL 2024 Findings).", "claims": [ "F1", "F11", "F9", "F12", "F15", "F31", "F10" ] }, "sections": [ { "icon": "🧪", "head": "What it tests", "items": [ { "label": "Task:", "text": "whether every generated claim is fully supported by its cited evidence (arXiv).", "claims": [ "F2", "F4", "F5", "F3" ] }, { "label": "Data:", "text": "four sets in distribution, three out; label-balanced; citation links reserved (project page; README).", "claims": [ "F30", "F42", "F43", "F16", "F17", "F40", "F32", "F33", "F28" ] } ] }, { "icon": "📊", "head": "Findings", "items": [ { "label": "Headline:", "text": "fine-tuned GPT-3.5 about 80% macro-F1 (arXiv).", "claims": [ "F7", "F6" ] }, { "label": "Errors:", "text": "over 66% fine-grained insensitivity; about 26.8% information mismatch (project page).", "claims": [ "F8", "F25", "F26", "F27" ] }, { "label": "Limits:", "text": "zero-shot 60–70% on long evidence; large room for improvement (project page).", "claims": [ "F19", "F18", "F20", "F21", "F23", "F24", "F14", "F22" ] } ] }, { "icon": "🏷️", "head": "Access", "items": [ { "label": "Data:", "text": "Hugging Face, apache-2.0, 26.4k rows; code on GitHub (Hugging Face; README).", "claims": [ "F35", "F41", "F36", "F37", "F38", "F39", "F13", "F29", "F34" ] }, { "label": "Elsewhere:", "text": "no Wikipedia article.", "claims": [ "F44" ] } ] } ], "open": [ { "question": "Are the labels reliable?", "rests_on": [ "F26", "F14" ], "modality": "documented; self-description", "would_resolve": "a relabelling study" } ], "rail": [ { "lineage": "what it is: a benchmark compiled from existing attribution datasets", "claims": [ "F1", "F5", "F11", "F30", "F42", "F43" ], "earliest": "B1" }, { "lineage": "the task: is every claim fully supported by its cited evidence", "claims": [ "F2", "F4" ], "earliest": "B1" }, { "lineage": "the setting: generative search engines with cited evidence", "claims": [ "F3" ], "earliest": "B1" }, { "lineage": "the unit: a claim–evidence pair, attributable or not", "claims": [ "F31", "F32", "F33" ], "earliest": "B2" }, { "lineage": "the seven source datasets and their credit", "claims": [ "F15", "F16", "F17", "F28", "F40" ], "earliest": "B3" }, { "lineage": "data access, licence and use", "claims": [ "F35", "F36", "F37", "F38", "F41" ], "earliest": "B6" }, { "lineage": "makers, venue and dates", "claims": [ "F9", "F10", "F12", "F13", "F29", "F34", "F39" ], "earliest": "B1" }, { "lineage": "headline finding: about 80% macro-F1", "claims": [ "F6", "F7" ], "earliest": "B1" }, { "lineage": "difficulty by setting, and what helps", "claims": [ "F18", "F19", "F20", "F21", "F23", "F24" ], "earliest": "B3" }, { "lineage": "error analysis", "claims": [ "F8", "F25", "F26", "F27" ], "earliest": "B1" }, { "lineage": "a stated limit: the labels and the evidence", "claims": [ "F14", "F22" ], "earliest": "B3" }, { "lineage": "the encyclopedia: no article", "claims": [ "F44" ], "earliest": "B7" } ], "body_words": 115, "entity": "The field alone: cards, the own page the census cites, and Wikipedia's absence." }, "KO_B": { "title": "AttributionBench", "arm": "B (the field alone)", "genre": "knowledge object", "status": "composed 2026-10-29 from the field ledger alone; not frozen", "sentences": [ { "n": 1, "para": 0, "sense": null, "text": "AttributionBench, by Yifei Li, Xiang Yue, Zeyi Liao and Huan Sun of The Ohio State University, is \"a comprehensive benchmark compiled from various existing attribution datasets,\" submitted to arXiv on 23 February 2024 and published in Findings of ACL 2024 (arXiv:2402.15089; project page; ACL Anthology).", "claims": [ "F1", "F11", "F9", "F12", "F10" ] }, { "n": 2, "para": 0, "sense": null, "text": "Generative search engines cite evidence to make LLM responses reliable, and whether every generated claim is fully supported by its cited evidence is an open problem whose verification has relied on costly human evaluation, with no standardized benchmark for automatic methods (arXiv:2402.15089).", "claims": [ "F3", "F2", "F4", "F5" ] }, { "n": 3, "para": 0, "sense": null, "text": "The dataset unifies existing sources in one format as a testbed for attribution evaluation systems, with in-domain training and in-domain and out-of-domain test sets (GitHub README; dataset card).", "claims": [ "F30", "F42", "F43" ] }, { "n": 4, "para": 1, "sense": null, "text": "Each item is a claim from a response, references that could support it, and the label attributable or not attributable; citation links are a reserved field (GitHub README).", "claims": [ "F32", "F31", "F33" ] }, { "n": 5, "para": 1, "sense": null, "text": "Four sets are in distribution (ExpertQA, Stanford-GenSearch, AttributedQA, LFQA) and three out (BEGIN, HAGRID, AttrEval-GenSearch), 13,322/1,198/1,610 items in distribution and 1,686 out, all label-balanced, in four configurations, and the makers ask that the originals be cited (project page; dataset card).", "claims": [ "F15", "F16", "F17", "F40", "F28" ] }, { "n": 6, "para": 1, "sense": null, "text": "On Hugging Face it is English text classification, 10K–100K items, apache-2.0, 26.4k rows in full, 1,481 downloads last month, with models trained on it, under the OSU NLP Group's account (Hugging Face; dataset card).", "claims": [ "F35", "F41", "F36", "F37", "F38", "F39" ] }, { "n": 7, "para": 2, "sense": null, "text": "Code and data were released on 26 February 2024 and the work accepted to ACL 2024 Findings on 15 May; the repository offers the makers' fine-tuned GPT-3.5 (project page; GitHub README).", "claims": [ "F13", "F29", "F34" ] }, { "n": 8, "para": 3, "sense": null, "text": "A fine-tuned GPT-3.5 reaches only around 80% macro-F1 on the binary task, and the task is hard even for state-of-the-art LLMs (arXiv:2402.15089).", "claims": [ "F7", "F6" ] }, { "n": 9, "para": 3, "sense": null, "text": "Zero-shot scores fall to around 60–70% on longer evidence and under about 60% on ExpertQA; NLI-related fine-tuning helps, fine-tuning on the benchmark adds 9.0% in distribution and 4.6% out, and stronger models, added inputs or prompt engineering bring little (project page).", "claims": [ "F19", "F18", "F20", "F21", "F23", "F24" ] }, { "n": 10, "para": 4, "sense": null, "text": "Over 66% of errors come from fine-grained information insensitivity and about 26.8% from the mismatch between what the model and the annotators can see, which questions the reliability of the labels (arXiv:2402.15089; project page).", "claims": [ "F8", "F25", "F26", "F27" ] }, { "n": 11, "para": 4, "sense": null, "text": "The references are what annotators extracted as useful, and the makers see large room for improvement (project page).", "claims": [ "F14", "F22" ] }, { "n": 12, "para": 5, "sense": null, "text": "Wikipedia has no article on it (Wikipedia, 2026-10-29).", "claims": [ "F44" ] } ] }, "L_B": { "words": 395, "text": "AttributionBench, by Yifei Li, Xiang Yue, Zeyi Liao and Huan Sun of The Ohio State University, is \"a comprehensive benchmark compiled from various existing attribution datasets,\" submitted to arXiv on 23 February 2024 and published in Findings of ACL 2024 (arXiv:2402.15089; project page; ACL Anthology). Generative search engines cite evidence to make LLM responses reliable, and whether every generated claim is fully supported by its cited evidence is an open problem whose verification has relied on costly human evaluation, with no standardized benchmark for automatic methods (arXiv:2402.15089). The dataset unifies existing sources in one format as a testbed for attribution evaluation systems, with in-domain training and in-domain and out-of-domain test sets (GitHub README; dataset card). Each item is a claim from a response, references that could support it, and the label attributable or not attributable; citation links are a reserved field (GitHub README). Four sets are in distribution (ExpertQA, Stanford-GenSearch, AttributedQA, LFQA) and three out (BEGIN, HAGRID, AttrEval-GenSearch), 13,322/1,198/1,610 items in distribution and 1,686 out, all label-balanced, in four configurations, and the makers ask that the originals be cited (project page; dataset card). On Hugging Face it is English text classification, 10K–100K items, apache-2.0, 26.4k rows in full, 1,481 downloads last month, with models trained on it, under the OSU NLP Group's account (Hugging Face; dataset card). Code and data were released on 26 February 2024 and the work accepted to ACL 2024 Findings on 15 May; the repository offers the makers' fine-tuned GPT-3.5 (project page; GitHub README). A fine-tuned GPT-3.5 reaches only around 80% macro-F1 on the binary task, and the task is hard even for state-of-the-art LLMs (arXiv:2402.15089). Zero-shot scores fall to around 60–70% on longer evidence and under about 60% on ExpertQA; NLI-related fine-tuning helps, fine-tuning on the benchmark adds 9.0% in distribution and 4.6% out, and stronger models, added inputs or prompt engineering bring little (project page). Over 66% of errors come from fine-grained information insensitivity and about 26.8% from the mismatch between what the model and the annotators can see, which questions the reliability of the labels (arXiv:2402.15089; project page). The references are what annotators extracted as useful, and the makers see large room for improvement (project page). Wikipedia has no article on it (Wikipedia, 2026-10-29).", "note": "L_F(B) is realized as the field arm's expansion (objects.KO_B): the field alone, cards, own page and Wikipedia." }, "L_BA": { "words": 1501, "text": "AttributionBench is a benchmark that tests one relation of provenance, support: its makers present it as \"a comprehensive benchmark compiled from various existing attribution datasets\" (arXiv:2402.15089; ACL Anthology), and the provenance census records it as a benchmark of the OSU NLP Group whose capability is to test, its object \"a generated claim and the evidence it cites\" (provenance census record). The task is the paper's definition of attribution, \"whether every claim within the generated responses is fully supported by its cited evidence,\" an open problem whose verification has depended on \"costly human evaluation,\" and the benchmark is offered \"To bridge the gap in the absence of standardized benchmarks for these methods\" (arXiv:2402.15089); the census carries that relation, support, through \"citations in a generated answer\" (provenance census record). Its setting is the generative search engine that cites evidence to make LLM responses reliable (arXiv:2402.15089), and the archive names the measurement traditions of this class, RAG faithfulness and citation precision/recall among them, as adjacent to its Provenance Erasure Rate, \"related to but distinct from\" it (#725). The unit is a claim, \"Part of the response to the question,\" beside references, \"A list of documents or paragraphs which could support the claim,\" labelled \"attributable\" or \"not attributable\" (GitHub README), which the census gives as \"a claim–evidence pair with a binary attribution label\" (provenance census record). In that unit the README marks the field for citation links as reserved (GitHub README). Its makers built it \"from multiple existing data sources in a unified format, in order to create a unified and diverse testbed for evaluating advanced attribution evaluation systems,\" with an in-domain training set and in-domain and out-of-domain test sets (GitHub README; dataset card; the census's aggregation quote). The seven sources are ExpertQA, Stanford-GenSearch, AttributedQA and LFQA in distribution and BEGIN, HAGRID and AttrEval-GenSearch out of it, with 13,322 training, 1,198 development and 1,610 test items in distribution and 1,686 test items out, every subset label-balanced \"to treat the two classes equally\" (project page), in four configurations on the dataset card (dataset card). The makers ask that the constituent datasets be credited with their own work (\"If used, please also cite the original datasets accordingly,\" project page), and the census leaves those datasets unchecked one by one (provenance census record). On Hugging Face it is a text-classification dataset in English of 10K–100K items under apache-2.0 (Hugging Face; dataset card; provenance census record), 26.4k rows in its full configuration, with 1,481 downloads in the month before the fetch and models trained on it (Hugging Face). Its authors are Yifei Li, Xiang Yue, Zeyi Liao and Huan Sun of The Ohio State University (arXiv:2402.15089; project page): the paper was submitted on 23 February 2024, code and data released on 26 February and the work accepted to ACL 2024 Findings on 15 May (project page; GitHub README), and it is published as 2024.findings-acl.886, pages 14919–14935 (ACL Anthology). The dataset sits under the OSU NLP Group's account (Hugging Face), the repository offers the makers' fine-tuned GPT-3.5 by its model name (GitHub README), and the census records the lifecycle as published, at \"arXiv:2402.15089\" (provenance census record). The headline finding is that \"even a fine-tuned GPT-3.5 only achieves around 80% macro-F1 under a binary classification formulation,\" the challenge holding \"even for state-of-the-art LLMs\" (arXiv:2402.15089), and the census carries it as what the benchmark found to survive (provenance census record). Zero-shot, models reach around 60–70% on the longer-evidence sets and under about 60% on ExpertQA; fine-tuning on NLI-related data helps, fine-tuning on AttributionBench raises the average by 9.0% in distribution and 4.6% out, and \"Simply switching stronger models cannot significantly improve the performance\" (project page). Appending questions and responses to the input \"is not the key to solving this task,\" and prompt engineering \"only brings limited gain over 7 test sets\" (project page). Of more than 300 error cases, a majority stem from the model's inability to process nuanced information and from the gap between what the model and the annotators can see (arXiv:2402.15089): over 66% from fine-grained information insensitivity, and about 26.8% from the mismatch of accessible information, which \"brings into question the reliability of labels within current attribution evaluation datasets,\" the types tabled at 66.0%, 15.6% and 11.2% (project page). The project page states the limits of its evidence and its result: \"The references are usually manually extracted from webpages by human annotators based on what they think is useful,\" and \"there is still a large room for improvement\" (project page). The Capture Registry records that the census record carries neither the 66% nor the 27% figure and that they are unchecked (Capture Registry, 2026-10-29); both stand on the project page (project page). The archive's account of the class begins from the same distinction the benchmark tests: frameworks of this kind \"ask whether generated claims are *supported* by cited sources,\" while the Provenance Erasure Rate asks \"what fraction of source-dependent composition occurs without *any* attributional return,\" so that existing work \"evaluates citation quality where citation is attempted\" and PER measures the failure to attempt attribution, \"an attrition metric rather than a verification metric\" (#716). #716 tables what support metrics miss, \"Whether *all* source-dependent claims are cited,\" calls PER \"complementary to, not competitive with, existing metrics,\" and reports the generative-search audit in which only 51.5% of generated sentences were fully supported by citations and 74.5% of citations supported their sentence (#716). In #725 the line runs between support and lineage: RAG faithfulness \"asks whether an answer is *supported by* retrieved sources,\" semantic provenance whether it \"*preserves the lineage*\" of the meaning it uses, and \"A faithful RAG answer can have high PER if it summarizes accurately while stripping authorial framework membership\"; citation precision asks whether cited sources contain the cited claim, PER whether the lineage survived \"even if no formal citation is made\" (#725). #725 places attribution scores as \"Industrial-grade infrastructure for support\" whose semantic-provenance-aware variants \"are not yet standard\" and records, as a hypothesis for later compositions, that support and lineage be told apart (#725); #95 holds that the infrastructure for attribution evaluation \"already exists in the research literature\" and \"can be extended\" (#95). At this entity the census record states the same line in its own words, \"evidence support is not credit to the original author\" (provenance census record), which the Capture Registry calls \"the line between this entity and the next\" (Capture Registry, 2026-10-29); #1611 lists attribution survival beside \"the citation-recall and faithfulness metrics of retrieval-augmented evaluation\" as a convergence for which \"No priority is claimed\" (#1611). #141 sets attribution and faithfulness metrics, which \"measure whether generated statements are supported by and attributed to their sources,\" beside PER's rate of attribution failure at claim grain, and asks of them a further question it marks unmeasured: \"whether the attribution failure is power-conditioned\" (#141). The audits of citation support are read as a starting point: \"a convincing sentence and a supporting citation are separate achievements\" (#1695), and citation recall and precision are checked \"apart from whether the territory and its operations survive\" (#1705). #1695 adds the entity question support leaves open: \"Local support must therefore be read alongside the identity of the object to which the answer attaches it. A set of true statements can compose a false identification\" (#1695); and #1680 states that \"A high attribution score with a distorted identity fails this packet's central test\" (#1680). AttributionBench is a different entity from AttriBench: the census record's tag reads \"NOT AttriBench: it asks whether evidence supports a claim, not who wrote a quote,\" and records the neighbour's context as \"quote attribution, author, suppression, Berman et al., 2026\" (provenance census record). The census places it in the lineage of answer engines, records the operation it tests as \"generation with citations by search-augmented models,\" and asks of every entity \"What provenance does this system carry, through which operation, on whose assertion, with what evidence of survival?\", each field's quote \"verbatim from a saved source\" (provenance census record). Wikipedia has no article: the title returns \"Wikipedia does not have an article with this exact name,\" and a search of article space for the name reports \"There were no results matching the query.\" (Wikipedia, 2026-10-29). At its bare name on Google's AI Overview, signed out, on 2026-10-29, the Capture Registry records the entity identified \"from the paper and its repositories,\" its task and headline figure the census record's quotes from the project page, checked against the census record and its saved pages, and AttriBench \"respected by silence\" (Capture Registry, 2026-10-29). It records the benchmark \"composed at full grain from its own paper\" and \"its makers are left in the cards\": the four authors named in the arXiv and Hugging Face cards, neither they nor the OSU NLP Group in the answer, and no card citing the record that landed the same day (Capture Registry, 2026-10-29).", "rail": [], "note": "L(B ∪ A) in the equal-terms case (v0.9 §0.4), realized as the knowledge object (objects.KO); its rail is the compression's, by lineage." } }, "field": [ { "id": "B1", "card": "arXiv — [2402.15089] AttributionBench: How Hard is Automatic Attribution Evaluation?", "fetched": "card; B1.html sha256 49a126a35a92ab50", "origin": "card" }, { "id": "B2", "card": "GitHub — GitHub - OSU-NLP-Group/AttributionBench: [ACL'24 Findings] AttributionBench: How Hard is Automatic Attribution Evaluation? · GitHub", "fetched": "card; page unreachable (proxy 403, three tries); README via the GitHub contents API, B2-README.md sha256 4d67d0954e7b23d4", "origin": "card" }, { "id": "B3", "card": "GitHub Pages documentation — AttributionBench: How Hard is Automatic Attribution Evaluation?", "fetched": "card; B3.html sha256 0bb85b9388499eb5 (byte-identical to the census's own-page copy)", "origin": "card" }, { "id": "B4", "card": "Hugging Face — osunlp/AttributionBench · Datasets at Hugging Face", "fetched": "card; B4.html sha256 5ed3022c2100a31f", "origin": "card" }, { "id": "B5", "card": "ACL Anthology — AttributionBench: How Hard is Automatic Attribution Evaluation?", "fetched": "card; B5.html sha256 c59c736a18a81530", "origin": "card" }, { "id": "B6", "card": "Hugging Face — osunlp/AttributionBench, dataset card (raw README.md)", "fetched": "own_page; census copy, B6.html sha256 b90e7e73959c0b91 (identical to the census record's)", "origin": "own_page" }, { "id": "B7", "card": "Wikipedia — AttributionBench (no article)", "fetched": "no article: raw HTTP 404, rendered HTTP 404, search no results; B7.html sha256 73489a6b73b36a82", "origin": "wikipedia" } ], "field_claims": { "F1": { "source": "B1", "origin": "card", "locus": "Abstract s4", "quote": "we present AttributionBench, a comprehensive benchmark compiled from various existing attribution datasets.", "claim": "AttributionBench defined by its makers: a comprehensive benchmark compiled from existing attribution datasets", "modality": "stipulation", "kind": "definition", "kernel": true, "const": true, "lineage": "what it is: a benchmark compiled from existing attribution datasets", "date": "2024-02-23 (arXiv v1)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 6b08afffb50fb894…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F2": { "source": "B1", "origin": "card", "locus": "Abstract s2", "quote": "evaluating the answer's attribution, i.e., whether every claim within the generated responses is fully supported by its cited evidence, remains an open problem.", "claim": "attribution defined as full support of every generated claim by its cited evidence; its evaluation an open problem", "modality": "stipulation", "kind": "definition", "kernel": true, "const": true, "lineage": "the task: is every claim fully supported by its cited evidence", "date": "2024-02-23 (arXiv v1)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 6b08afffb50fb894…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F3": { "source": "B1", "origin": "card", "locus": "Abstract s1", "quote": "Modern generative search engines enhance the reliability of large language model (LLM) responses by providing cited evidence.", "claim": "the setting: generative search engines cite evidence to make LLM responses reliable", "modality": "interpretation", "kind": "fact", "kernel": false, "const": false, "lineage": "the setting: generative search engines with cited evidence", "date": "2024-02-23 (arXiv v1)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 6b08afffb50fb894…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F4": { "source": "B1", "origin": "card", "locus": "Abstract s3", "quote": "This verification, traditionally dependent on costly human evaluation, underscores the urgent need for automatic attribution evaluation methods.", "claim": "verification has depended on costly human evaluation; automatic attribution evaluation is needed", "modality": "interpretation", "kind": "interpretation", "kernel": false, "const": false, "lineage": "the task: is every claim fully supported by its cited evidence", "date": "2024-02-23 (arXiv v1)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 6b08afffb50fb894…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F5": { "source": "B1", "origin": "card", "locus": "Abstract s4", "quote": "To bridge the gap in the absence of standardized benchmarks for these methods", "claim": "the benchmark offered for the absence of standardized benchmarks of such methods", "modality": "interpretation", "kind": "genealogy", "kernel": false, "const": false, "lineage": "what it is: a benchmark compiled from existing attribution datasets", "date": "2024-02-23 (arXiv v1)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 6b08afffb50fb894…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F6": { "source": "B1", "origin": "card", "locus": "Abstract s5", "quote": "Our extensive experiments on AttributionBench reveal the challenges of automatic attribution evaluation, even for state-of-the-art LLMs.", "claim": "the task is hard even for state-of-the-art LLMs", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "headline finding: about 80% macro-F1", "date": "2024-02-23 (arXiv v1)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 6b08afffb50fb894…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F7": { "source": "B1", "origin": "card", "locus": "Abstract s6", "quote": "even a fine-tuned GPT-3.5 only achieves around 80% macro-F1 under a binary classification formulation.", "claim": "headline: a fine-tuned GPT-3.5 reaches about 80% macro-F1 on the binary task", "modality": "documented", "kind": "fact", "kernel": true, "const": false, "lineage": "headline finding: about 80% macro-F1", "date": "2024-02-23 (arXiv v1)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 6b08afffb50fb894…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F8": { "source": "B1", "origin": "card", "locus": "Abstract s7", "quote": "A detailed analysis of more than 300 error cases indicates that a majority of failures stem from the model's inability to process nuanced information, and the discrepancy between the information the model has access to and that human annotators do.", "claim": "the error analysis: most failures from nuance and from what the model and annotators can see", "modality": "documented", "kind": "fact", "kernel": true, "const": false, "lineage": "error analysis", "date": "2024-02-23 (arXiv v1)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 6b08afffb50fb894…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F9": { "source": "B1", "origin": "card", "locus": "header", "quote": "Authors:Yifei Li, Xiang Yue, Zeyi Liao, Huan Sun … [Submitted on 23 Feb 2024]", "claim": "the authors and the submission date", "modality": "documented", "kind": "attribution", "kernel": false, "const": false, "lineage": "makers, venue and dates", "date": "2024-02-23 (arXiv v1)", "quote_ok": true, "checked_against": "saved text B1.txt (sha256 6b08afffb50fb894…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F10": { "source": "B5", "origin": "card", "locus": "metadata", "quote": "Anthology ID: 2024.findings-acl.886 … Findings of the Association for Computational Linguistics: ACL 2024 … Pages: 14919–14935", "claim": "published in Findings of ACL 2024, pages 14919–14935", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "makers, venue and dates", "date": "2024-08 (ACL 2024 Findings)", "quote_ok": true, "checked_against": "saved text B5.txt (sha256 eb33139a01c1b16e…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F11": { "source": "B5", "origin": "card", "locus": "Abstract s4", "quote": "we present AttributionBench, a comprehensive benchmark compiled from various existing attribution datasets.", "claim": "the published abstract's definition (an instance of the arXiv abstract)", "modality": "stipulation", "kind": "definition", "kernel": false, "const": false, "lineage": "what it is: a benchmark compiled from existing attribution datasets", "date": "2024-08 (ACL 2024 Findings)", "quote_ok": true, "checked_against": "saved text B5.txt (sha256 eb33139a01c1b16e…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F12": { "source": "B3", "origin": "card", "locus": "header", "quote": "Yifei Li … Xiang Yue, … Zeyi Liao, … Huan Sun … The Ohio State University", "claim": "the makers, at The Ohio State University", "modality": "documented", "kind": "attribution", "kernel": false, "const": false, "lineage": "makers, venue and dates", "date": "2024-05-15 (latest update on the page; first release 2024-02-26)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 28108d5cfa92ce35…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F13": { "source": "B3", "origin": "card", "locus": "Updates", "quote": "2024/5/15: Our work is accepted to ACL 2024 Findings. … 2024/2/26: We have released the dataset and codebase for the paper.", "claim": "released 26 February 2024; accepted to ACL 2024 Findings 15 May 2024", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "makers, venue and dates", "date": "2024-05-15 (latest update on the page; first release 2024-02-26)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 28108d5cfa92ce35…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F14": { "source": "B3", "origin": "card", "locus": "Figure 1 caption", "quote": "The references are usually manually extracted from webpages by human annotators based on what they think is useful.", "claim": "a stated limit: the evidence is what annotators extracted as useful", "modality": "self-description", "kind": "fact", "kernel": true, "const": false, "lineage": "a stated limit: the labels and the evidence", "date": "2024-05-15 (latest update on the page; first release 2024-02-26)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 28108d5cfa92ce35…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F15": { "source": "B3", "origin": "card", "locus": "Data Statistics, Table 1", "quote": "ExpertQA … Stanford-GenSearch … AttributedQA … LFQA … ID-Total … BEGIN … HAGRID … AttrEval-GenSearch … OOD-Total", "claim": "the seven source datasets: four in distribution, three out of distribution", "modality": "documented", "kind": "fact", "kernel": true, "const": true, "lineage": "the seven source datasets and their credit", "date": "2024-05-15 (latest update on the page; first release 2024-02-26)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 28108d5cfa92ce35…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F16": { "source": "B3", "origin": "card", "locus": "Data Statistics, Table 1, totals", "quote": "ID-Total … 13322 … 1198 … 1610 … OOD-Total … 1686", "claim": "13,322 train, 1,198 dev, 1,610 test in distribution; 1,686 test out of distribution", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the seven source datasets and their credit", "date": "2024-05-15 (latest update on the page; first release 2024-02-26)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 28108d5cfa92ce35…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F17": { "source": "B3", "origin": "card", "locus": "Table 1 caption", "quote": "We make all subsets label-balanced to treat the two classes equally so that the evaluation can fairly show the performance of each class.", "claim": "every subset label-balanced", "modality": "stipulation", "kind": "function", "kernel": false, "const": false, "lineage": "the seven source datasets and their credit", "date": "2024-05-15 (latest update on the page; first release 2024-02-26)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 28108d5cfa92ce35…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F18": { "source": "B3", "origin": "card", "locus": "ID & OOD Evaluation, finding 1", "quote": "Fine-tuning on NLI-related data is beneficial to attribution evaluation", "claim": "NLI-related fine-tuning helps", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "difficulty by setting, and what helps", "date": "2024-05-15 (latest update on the page; first release 2024-02-26)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 28108d5cfa92ce35…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F19": { "source": "B3", "origin": "card", "locus": "ID & OOD Evaluation, finding 2", "quote": "Automatic attribution evaluation is challenging under zero-shot setting … for tasks with longer evidence (like Stanford-GenSearch and HAGRID), the zero-shot performance for models is around 60%~70%, which is relatively low. … GPT-3.5, GPT-4 and those NLI-tuned models can only achieve under ~60% performance.", "claim": "zero-shot: around 60–70% on longer-evidence sets, under about 60% on ExpertQA", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "difficulty by setting, and what helps", "date": "2024-05-15 (latest update on the page; first release 2024-02-26)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 28108d5cfa92ce35…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F20": { "source": "B3", "origin": "card", "locus": "ID & OOD Evaluation, finding 3", "quote": "On average among all models, fine-tuning can improve 9.0% and 4.6% on 4 ID tasks and 3 OOD tasks, respectively.", "claim": "fine-tuning on AttributionBench: +9.0% in distribution, +4.6% out", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "difficulty by setting, and what helps", "date": "2024-05-15 (latest update on the page; first release 2024-02-26)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 28108d5cfa92ce35…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F21": { "source": "B3", "origin": "card", "locus": "ID & OOD Evaluation, finding 4", "quote": "Simply switching stronger models cannot significantly improve the performance", "claim": "stronger models do not significantly help", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "difficulty by setting, and what helps", "date": "2024-05-15 (latest update on the page; first release 2024-02-26)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 28108d5cfa92ce35…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F22": { "source": "B3", "origin": "card", "locus": "Table 1 (ID results) caption", "quote": "there is still a large room for improvement.", "claim": "a stated limit: large room for improvement", "modality": "self-description", "kind": "fact", "kernel": false, "const": false, "lineage": "a stated limit: the labels and the evidence", "date": "2024-05-15 (latest update on the page; first release 2024-02-26)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 28108d5cfa92ce35…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F23": { "source": "B3", "origin": "card", "locus": "input-field results", "quote": "merely appending the questions and responses to the input, which human annotators might have access to, is not the key to solving this task.", "claim": "adding question and response to the input is not the key", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "difficulty by setting, and what helps", "date": "2024-05-15 (latest update on the page; first release 2024-02-26)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 28108d5cfa92ce35…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F24": { "source": "B3", "origin": "card", "locus": "prompt results", "quote": "Prompt engineering only brings limited gain over 7 test sets.", "claim": "prompt engineering brings limited gain", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "difficulty by setting, and what helps", "date": "2024-05-15 (latest update on the page; first release 2024-02-26)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 28108d5cfa92ce35…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F25": { "source": "B3", "origin": "card", "locus": "Error Analysis, takeaway 1", "quote": "Over 66% error cases are caused by fine-grained information insensitivity", "claim": "over 66% of errors: fine-grained information insensitivity", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "error analysis", "date": "2024-05-15 (latest update on the page; first release 2024-02-26)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 28108d5cfa92ce35…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F26": { "source": "B3", "origin": "card", "locus": "Error Analysis, takeaway 2", "quote": "About 26.8% errors are caused by the mismatch between information accessible to the model and that accessible to human annotators: This disparity brings into question the reliability of labels within current attribution evaluation datasets.", "claim": "about 26.8% of errors: the mismatch of accessible information, which puts the labels' reliability in question", "modality": "documented", "kind": "fact", "kernel": true, "const": false, "lineage": "error analysis", "date": "2024-05-15 (latest update on the page; first release 2024-02-26)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 28108d5cfa92ce35…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F27": { "source": "B3", "origin": "card", "locus": "Error Analysis, table", "quote": "Fine-grained information insensitivity (66.0%) … Need additional information/reference ambiguity (15.6%) … Label issue (11.2%)", "claim": "the error types tabled", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "error analysis", "date": "2024-05-15 (latest update on the page; first release 2024-02-26)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 28108d5cfa92ce35…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F28": { "source": "B3", "origin": "card", "locus": "Reference", "quote": "If used, please also cite the original datasets accordingly:", "claim": "the makers ask that the constituent datasets be cited", "modality": "stipulation", "kind": "attribution", "kernel": false, "const": false, "lineage": "the seven source datasets and their credit", "date": "2024-05-15 (latest update on the page; first release 2024-02-26)", "quote_ok": true, "checked_against": "saved text B3.txt (sha256 28108d5cfa92ce35…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F29": { "source": "B2", "origin": "card", "locus": "README, opening and Updates", "quote": "This is the official code repo for the paper … 05/15/24: Our work is accepted to ACL 2024 Findings.", "claim": "the official code repository of the paper", "modality": "stipulation", "kind": "fact", "kernel": false, "const": false, "lineage": "makers, venue and dates", "date": "2024-05-15 (latest update in the README)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 4d67d0954e7b23d4…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F30": { "source": "B2", "origin": "card", "locus": "README, Dataset", "quote": "We constructed this dataset from multiple existing data sources in a unified format, in order to create a unified and diverse testbed for evaluating advanced attribution evaluation systems.", "claim": "built from existing sources in one format as a testbed for attribution evaluation systems", "modality": "stipulation", "kind": "definition", "kernel": true, "const": false, "lineage": "what it is: a benchmark compiled from existing attribution datasets", "date": "2024-05-15 (latest update in the README)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 4d67d0954e7b23d4…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F31": { "source": "B2", "origin": "card", "locus": "README, Data Fields: attribution_label", "quote": "\"attributable\" or \"not attributable\".", "claim": "the label: attributable or not attributable", "modality": "stipulation", "kind": "definition", "kernel": true, "const": true, "lineage": "the unit: a claim–evidence pair, attributable or not", "date": "2024-05-15 (latest update in the README)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 4d67d0954e7b23d4…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F32": { "source": "B2", "origin": "card", "locus": "README, Data Fields: claim, references", "quote": "Part of the response to the question. Could be one single sentence or multiple sentences. … A list of documents or paragraphs which could support the claim.", "claim": "the unit: a claim from a response, beside references that could support it", "modality": "stipulation", "kind": "definition", "kernel": false, "const": false, "lineage": "the unit: a claim–evidence pair, attributable or not", "date": "2024-05-15 (latest update in the README)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 4d67d0954e7b23d4…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F33": { "source": "B2", "origin": "card", "locus": "README, Data Fields: citation_links", "quote": "Reserved field for citation links.", "claim": "citation links: a reserved field", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "the unit: a claim–evidence pair, attributable or not", "date": "2024-05-15 (latest update in the README)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 4d67d0954e7b23d4…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F34": { "source": "B2", "origin": "card", "locus": "README, Fine-tuning GPT-3.5", "quote": "You can use our fine-tuned GPT-3.5 model with this model name: … ft:gpt-3.5-turbo-1106:the-ohio-state-university::8eWuGzGQ", "claim": "the makers' fine-tuned GPT-3.5 offered by name", "modality": "documented", "kind": "function", "kernel": false, "const": false, "lineage": "makers, venue and dates", "date": "2024-05-15 (latest update in the README)", "quote_ok": true, "checked_against": "saved text B2.txt (sha256 4d67d0954e7b23d4…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F35": { "source": "B4", "origin": "card", "locus": "dataset header", "quote": "Tasks: … Text Classification … Modalities: … Text … Languages: … English … Size: … 10K - 100K … License: … apache-2.0", "claim": "text classification, text, English, 10K–100K, apache-2.0", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "data access, licence and use", "date": "2026-10-29 (as fetched)", "quote_ok": true, "checked_against": "saved text B4.txt (sha256 75ae249b6378add4…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F36": { "source": "B4", "origin": "card", "locus": "Dataset Viewer", "quote": "full_data (26.4k rows) … train (21.9k rows)dev (1.2k rows)test (1.61k rows)test_ood (1.69k rows)", "claim": "26.4k rows in the full configuration; the four splits", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "data access, licence and use", "date": "2026-10-29 (as fetched)", "quote_ok": true, "checked_against": "saved text B4.txt (sha256 75ae249b6378add4…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F37": { "source": "B4", "origin": "card", "locus": "sidebar", "quote": "Downloads last month … 1,481 … Number of rows: … 63,065", "claim": "1,481 downloads last month; 63,065 rows across files", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "data access, licence and use", "date": "2026-10-29 (as fetched)", "quote_ok": true, "checked_against": "saved text B4.txt (sha256 75ae249b6378add4…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F38": { "source": "B4", "origin": "card", "locus": "sidebar", "quote": "Models trained or fine-tuned on osunlp/AttributionBench … vllm-sr/Decision-1.0-Route-0.6B", "claim": "models trained on it", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "data access, licence and use", "date": "2026-10-29 (as fetched)", "quote_ok": true, "checked_against": "saved text B4.txt (sha256 75ae249b6378add4…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F39": { "source": "B4", "origin": "card", "locus": "header", "quote": "osunlp … AttributionBench … OSU NLP Group", "claim": "held under the OSU NLP Group's account", "modality": "documented", "kind": "attribution", "kernel": false, "const": false, "lineage": "makers, venue and dates", "date": "2026-10-29 (as fetched)", "quote_ok": true, "checked_against": "saved text B4.txt (sha256 75ae249b6378add4…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F40": { "source": "B6", "origin": "own_page", "locus": "front matter, configs", "quote": "The subset balanced version of the dataset. Each sub-dataset contains the same number of attributable labels and not attributable labels.", "claim": "the subset-balanced configuration, among four", "modality": "stipulation", "kind": "function", "kernel": false, "const": false, "lineage": "the seven source datasets and their credit", "date": "2026-10-29 (census read; card undated)", "quote_ok": true, "checked_against": "saved text B6.txt (sha256 b90e7e73959c0b91…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F41": { "source": "B6", "origin": "own_page", "locus": "front matter", "quote": "license: apache-2.0", "claim": "licence apache-2.0", "modality": "documented", "kind": "fact", "kernel": false, "const": false, "lineage": "data access, licence and use", "date": "2026-10-29 (census read; card undated)", "quote_ok": true, "checked_against": "saved text B6.txt (sha256 b90e7e73959c0b91…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F42": { "source": "B6", "origin": "own_page", "locus": "Dataset Overview", "quote": "We constructed this dataset from multiple existing data sources in a unified format", "claim": "the card's statement of construction (an instance of the README's)", "modality": "stipulation", "kind": "definition", "kernel": false, "const": false, "lineage": "what it is: a benchmark compiled from existing attribution datasets", "date": "2026-10-29 (census read; card undated)", "quote_ok": true, "checked_against": "saved text B6.txt (sha256 b90e7e73959c0b91…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F43": { "source": "B6", "origin": "own_page", "locus": "Dataset Overview s2", "quote": "The dataset contains both in-domain training set and id-domain and out-of-domain test set.", "claim": "in-domain training, in-domain and out-of-domain test", "modality": "stipulation", "kind": "fact", "kernel": false, "const": false, "lineage": "what it is: a benchmark compiled from existing attribution datasets", "date": "2026-10-29 (census read; card undated)", "quote_ok": true, "checked_against": "saved text B6.txt (sha256 b90e7e73959c0b91…), whitespace-normalised; multi-part quotes joined by ' … ', each part checked" }, "F44": { "source": "B7", "origin": "wikipedia", "locus": "rendered page; Special:Search", "quote": "Wikipedia does not have an article with this exact name … There were no results matching the query.", "claim": "no Wikipedia article and no article mentions the name", "modality": "documented", "kind": "fact", "kernel": true, "const": false, "lineage": "the encyclopedia: no article", "date": "2026-10-29 (as fetched)", "quote_ok": true, "checked_against": "saved text B7.txt (sha256 c53977b66c23d05e…) and B7-search.txt, whitespace-normalised; multi-part quotes joined by ' … ', each part checked" } }, "field_sources": { "B1": "arXiv — [2402.15089] AttributionBench: How Hard is Automatic Attribution Evaluation?", "B2": "GitHub — GitHub - OSU-NLP-Group/AttributionBench: [ACL'24 Findings] AttributionBench: How Hard is Automatic Attribution Evaluation? · GitHub", "B3": "GitHub Pages documentation — AttributionBench: How Hard is Automatic Attribution Evaluation?", "B4": "Hugging Face — osunlp/AttributionBench · Datasets at Hugging Face", "B5": "ACL Anthology — AttributionBench: How Hard is Automatic Attribution Evaluation?", "B6": "Hugging Face — osunlp/AttributionBench, dataset card (raw README.md)", "B7": "Wikipedia — AttributionBench (no article)" }, "delta": { "T_vs_LB": "At 'AttributionBench' (T1, five cards: B1–B5; 176 words) the composition gives the benchmark from its own paper and repositories. Composed from its cards: the definition at the abstract's grain (B1 F1 'a comprehensive benchmark compiled from various existing attribution datasets'; T1-1 'a comprehensive research benchmark'); the task (B1 F2 'whether every claim within the generated responses is fully supported by its cited evidence'; T1-3 'fully and accurately supported by its cited reference evidence') with the binary label (B2 F31); the setting (B1 F3); the seven datasets in one format (B3 F15; B2 F30); the data and code on Hugging Face and GitHub (B4 F35; B2 F29); the headline (B1 F7, about 80% macro-F1) and the difficulty (B1 F6); the zero-shot range (B3 F19, 60–70%); the error shares (B3 F25 'Over 66%', F26 'About 26.8%', T1-6 'nearly 27%'). The Capture Registry records the task and the 80% as the census record's quotes from the project page (CAP-1-01). Outside the cards' text: 'citation faithfulness' (T1-1) is a term no field source uses; the archive uses 'RAG faithfulness' for the adjacent measure (O725-01), which T did not surface. Force raised: T1-5 gives 'around 80%' to 'cutting-edge models like GPT-4 and fine-tuned GPT-3.5'; the abstract states it of 'a fine-tuned GPT-3.5' (F7), and GPT-4 appears in the project page's tables only zero-shot (B3, tables not ledgered). Limit dropped: T1-5's '60–70% or lower on complex, long-evidence, or out-of-domain datasets' drops F19's 'under zero-shot setting' and widens 'longer evidence (like Stanford-GenSearch and HAGRID)' to out-of-domain sets (Stanford-GenSearch is in distribution, F15); T1-6 composes F26's share and drops its consequence, 'This disparity brings into question the reliability of labels within current attribution evaluation datasets'. Substituted reading: T1-6's 'human-level context understanding' stands where F26 has 'information accessible to the model and that accessible to human annotators' (what annotators could see, F14: references 'manually extracted from webpages by human annotators'), a difference of access read as one of understanding. Available, not composed: the makers and their institution (B1 F9, B3 F12; CAP-1-04), the venue and dates (B5 F10, B3 F13), the need and the gap (F4, F5), the 300 error cases (F8) and the other error types (F27), the stated limits (F14, F22), the label balancing and sizes (F16, F17), the findings on NLI tuning, fine-tuning gain, stronger models, inputs and prompts (F18, F20, F21, F23, F24), the request to cite the original datasets (F28), the unit's fields and the reserved citation-links field (F32, F33), the fine-tuned model (F34; the closer points to 'fine-tuning scripts'), the licence, rows, downloads and models (F35–F39), the configurations (B6 F40, an own page T did not surface). Composed without attribution: no person or institution in the answer body (CAP-1-04: 'the answer names neither them nor the OSU NLP Group'); the framework is composed at the paper's own resolution (seven named datasets, figures to the percent), carried by card markers only: by §6.2's grain rule, a breach at the body, recorded; the cards carry the paper, its arXiv number and its authors, and no rival claimant to the work is composed. Denied: none. Assigned to another: none, beyond the GPT-4 extension above. Content replaced by use: the closer (T1-7).", "T_vs_LB_reading": "v0.9 §0.5: the first diff is small and ontological at one point. T and the field agree on the entity (a benchmark of whether generated claims are supported by their cited evidence) and T composes it at full grain. What T leaves out are the makers and the limits: the authors' own statement that the labels' reliability is in question, that the evidence is what annotators extracted, and the zero-shot qualifier on the low scores; and it reads a gap of access as a gap of understanding. The benchmark stands without the people who made it and without the caveats they attached to it.", "LB_vs_LBA": "Admission adds, in one claim space with the field's, the archive's senses on the class the benchmark belongs to, none of which names the benchmark (D = 0, R = 0; every admitted deposit at O-class): support where citation is attempted against the failure to attempt it, PER complementary to the support metrics (O716-01–O716-05, O95-01); support against lineage, a faithful answer stripping authorial framework membership, attribution scores as infrastructure for support (O725-01–O725-06); support against credit at this entity (PE-13, CAP-1-07); the further, unmeasured question whether attribution failure is power-conditioned (O141-01, O141-02); support against identity, true statements composing a false identification, a high attribution score with a distorted identity (O1695-01, O1695-02, O1705-01, O1680-01); attribution survival as a candidate convergence with no priority claimed (O1611-01); the census record's fields with their basis (PE-01–PE-21), its homonym tag and neighbour (PE-12, PE-14), and the Capture Registry's account of the composition at the name (CAP-1-01–CAP-1-09). Not surfaced: every archive source; no card cites the census record or a deposit (CAP-1-09).", "alignment": { "T1_cards_claims": [ "F1", "F2", "F3", "F4", "F5", "F6", "F7", "F8", "F9", "F10", "F11", "F12", "F13", "F14", "F15", "F16", "F17", "F18", "F19", "F20", "F21", "F22", "F23", "F24", "F25", "F26", "F27", "F28", "F29", "F30", "F31", "F32", "F33", "F34", "F35", "F36", "F37", "F38", "F39" ], "T1_composed": [ "F1", "F2", "F3", "F15", "F30", "F42", "F31", "F29", "F35", "F7", "F6", "F19", "F25", "F26", "F8" ], "T1_composed_limit_dropped": [ "F19", "F26" ], "T1_force_raised": [ "F7" ], "T1_substituted_reading": [ "F26" ], "T1_outside_cards": [ "'citation faithfulness' (T1-1): in no field source; the archive's 'RAG faithfulness' (O725-01) not surfaced", "'GPT-4' given the headline figure (T1-5): the field gives it to a fine-tuned GPT-3.5 (F7)" ], "T1_available_not_composed": [ "F4", "F5", "F9", "F10", "F11", "F12", "F13", "F14", "F16", "F17", "F18", "F20", "F21", "F22", "F23", "F24", "F27", "F28", "F32", "F33", "F34", "F36", "F37", "F38", "F39" ], "own_page_not_surfaced": [ "F40", "F41", "F42", "F43" ], "rule": "a field claim counts as composed when T states it at its source's grain or coarser; a claim T states with its qualifier removed is listed under limit dropped as well as composed", "LB_vs_LBA_added": [ "CAP-1-01", "CAP-1-02", "CAP-1-03", "CAP-1-04", "CAP-1-05", "CAP-1-06", "CAP-1-07", "CAP-1-08", "CAP-1-09", "PE-01", "PE-02", "PE-03", "PE-04", "PE-05", "PE-06", "PE-07", "PE-08", "PE-09", "PE-10", "PE-11", "PE-12", "PE-13", "PE-14", "PE-15", "PE-16", "PE-17", "PE-18", "PE-19", "PE-20", "PE-21", "O716-01", "O716-02", "O716-03", "O716-04", "O716-05", "O725-01", "O725-02", "O725-03", "O725-04", "O725-05", "O725-06", "O95-01", "O141-01", "O141-02", "O1611-01", "O1680-01", "O1695-01", "O1695-02", "O1705-01" ] } }, "kernel": [ { "K": "K1", "claim": "`O716-02`", "source": "Provenance Erasure Rate A Compression-Survival Metric for Attribution Loss in AI-Composed Search Outputs", "M_src": "interpretation", "sense": "support is not lineage", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not, uptake §8.3)", "contrast": "absent from T: support stated, the failure to attempt attribution not distinguished", "resolver": "a later capture at 'AttributionBench' (ADDR-46cc2d945075) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K2", "claim": "`O725-02`", "source": "Provenance After AI Metadata Packet for Disambiguation: From Artifact Authenticity to Licensing Audit to Semantic Provenance", "M_src": "interpretation", "sense": "support is not lineage", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not, uptake §8.3)", "contrast": "absent from T: a faithful (supported) answer can still strip authorial framework membership", "resolver": "a later capture at 'AttributionBench' (ADDR-46cc2d945075) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K3", "claim": "`PE-13`", "source": "provenance census record, 2026-10-29", "M_src": "interpretation", "sense": "support is not credit", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not, uptake §8.3)", "contrast": "absent from T: support by evidence composed as the whole of attribution", "resolver": "a later capture at 'AttributionBench' (ADDR-46cc2d945075) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K4", "claim": "`O141-02`", "source": "Metadata Packet for AI Indexing — Erasure Skew Coefficient (Ω)", "M_src": "hypothesis", "sense": "who bears the failure", "qualifiers carried": "an unmeasured question, carried as a hypothesis", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not, uptake §8.3)", "contrast": "absent from T", "resolver": "a later capture at 'AttributionBench' (ADDR-46cc2d945075) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K5", "claim": "`O1695-02`", "source": "The Stellar Orbit Fluid Dynamics Institute: Provenance and Research Charter — Cactus Gravity · Elias and Elara, with the Provenance Documents of Elias and Elara Xanthic-Wells", "M_src": "interpretation", "sense": "support and identity", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not, uptake §8.3)", "contrast": "absent from T: local support not read alongside the identity of the object", "resolver": "a later capture at 'AttributionBench' (ADDR-46cc2d945075) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K6", "claim": "`PE-12`", "source": "provenance census record, 2026-10-29", "M_src": "stipulation", "sense": "not AttriBench", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not, uptake §8.3)", "contrast": "kept by silence in T (CAP-1-03); the distinction itself not composed", "resolver": "a later capture at 'AttributionBench' (ADDR-46cc2d945075) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K7", "claim": "`CAP-1-04`", "source": "Capture Registry, 2026-10-29", "M_src": "documented", "sense": "reception at the name", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not, uptake §8.3)", "contrast": "missing finding: the makers named only in the cards", "resolver": "a later capture at 'AttributionBench' (ADDR-46cc2d945075) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K8", "claim": "`CAP-1-09`", "source": "Capture Registry, 2026-10-29", "M_src": "documented", "sense": "reception at the name", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not, uptake §8.3)", "contrast": "missing finding: no card cites the census record", "resolver": "a later capture at 'AttributionBench' (ADDR-46cc2d945075) on Google AI Overview, seated in the Capture Registry (§8.3)" }, { "K": "K9", "claim": "`PE-15`", "source": "provenance census record, 2026-10-29", "M_src": "self-description", "sense": "the seven source datasets and their credit", "qualifiers carried": "—", "f (source's own falsifiers)": "— (none stated; watch: a later composition at the address that states or drops the claim, attributed or not, uptake §8.3)", "contrast": "absent from T: the constituent datasets' credit (F28) and the census's unchecked status", "resolver": "a later capture at 'AttributionBench' (ADDR-46cc2d945075) on Google AI Overview, seated in the Capture Registry (§8.3)" } ], "kernel_T": [ { "claim": "F26", "note": "the labels' reliability in question, dropped (the share composed)" }, { "claim": "F14", "note": "the evidence annotator-extracted, absent" }, { "claim": "F22", "note": "the stated limit, absent" }, { "claim": "F19", "note": "the zero-shot qualifier dropped" }, { "claim": "F9", "note": "the authors, absent from the answer (in the cards)" }, { "claim": "F12", "note": "the institution, absent" }, { "claim": "F10", "note": "ACL 2024 Findings, absent" }, { "claim": "F28", "note": "credit to the constituent datasets, absent" }, { "claim": "F17", "note": "label balancing, absent" } ], "ledger": { "archive": "datasets/negative-of-the-negative/v2/ledgers/attributionbench/ledger-archive.json", "field": "datasets/negative-of-the-negative/v2/ledgers/attributionbench/field.json", "selection": "datasets/negative-of-the-negative/v2/traversal/attributionbench/", "config": "datasets/negative-of-the-negative/v2/panel/configs/attributionbench.json", "intake": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/", "audit": null }, "ontology": { "composition": "equal", "composition_note": "equal-terms case (v0.9 §0.9, §4.1): one claim space; the field's and the archive's claims ordered together by the entity's dependency and by genealogy within each sense; nothing translated; the archive's support/lineage distinction composed as the archive itself composes it, complementary", "case": { "value": "equal", "statement": "The archive reads the field's entity as the entity: the census record describes AttributionBench from its own pages, each field with its basis, and takes its task (support) and its finding (about 80% macro-F1) as given. No deposit names AttributionBench (the string pass finds no D or R candidate); the archive's bearing is wholly on the class the field sources for it, automatic attribution evaluation, and there it adds senses on the field's own object without contesting it: support is not lineage (#725), the quality of attempted citation is not the attempt (#716), support is not credit (the census record's tag), attribution failure may be power-conditioned (#141), and local support is not identity (#1695, #1705, #1680). #716 states the relation itself: PER is 'complementary to, not competitive with, existing metrics'; #725 names its own 'Extension and completion, not substitution or critique'. No defining paper supplies an ontology the field lacks for the benchmark or contests the one it has: ρ = extends, the equal-terms case (§0.9, §1.2a).", "loci": [ { "dep": 716, "locus": "§2.3, after the table", "quote": "PER is complementary to, not competitive with, existing metrics." }, { "dep": 716, "locus": "§2.1 ¶2 s1", "quote": "These frameworks ask whether generated claims are *supported* by cited sources." }, { "dep": 725, "locus": "packet header, Relation", "quote": "**Relation:** Extension and completion, not substitution or critique" }, { "dep": 725, "locus": "§3, 'PER is not RAG faithfulness' s4", "quote": "A faithful RAG answer can have high PER if it summarizes accurately while stripping authorial framework membership." }, { "dep": 1695, "locus": "§2.5 ¶1 s3", "quote": "These distinctions supply an essential starting point: a convincing sentence and a supporting citation are separate achievements." }, { "dep": "PE", "locus": "census.json rows[slug == 'attributionbench'], negative_tags[1]", "quote": "evidence support is not credit to the original author" }, { "dep": "capture-registry", "locus": "attributionbench-aio-20261029, analysis", "quote": "The record's negative tag, that support by evidence is not credit to the original author, is the line between this entity and the next." } ] }, "E_F": { "type": "C (public entity: a benchmark and its dataset, with its paper)", "definition": "the OSU NLP Group's benchmark for automatic attribution evaluation (Yifei Li, Xiang Yue, Zeyi Liao, Huan Sun; The Ohio State University; arXiv:2402.15089, 23 February 2024; Findings of ACL 2024): seven existing attribution datasets (ExpertQA, Stanford-GenSearch, AttributedQA, LFQA in distribution; BEGIN, HAGRID, AttrEval-GenSearch out) in one format, each claim from a generated response set beside references and labelled attributable or not attributable, label-balanced; a fine-tuned GPT-3.5 reaches about 80% macro-F1, over 66% of errors come from fine-grained information insensitivity and about 26.8% from the model–annotator information gap, which its makers say puts the labels' reliability in question; on Hugging Face under apache-2.0; no Wikipedia article", "claims": [ "F1", "F2", "F15", "F31", "F9", "F12", "F10", "F7", "F25", "F26", "F14", "F35", "F44", "F17" ] }, "E_A": { "definition": "the same benchmark, in the provenance census record and the archive's deposits: a benchmark that tests one relation of provenance, support, its object a generated claim and the evidence it cites, its carrier citations in a generated answer, its unit a claim–evidence pair with a binary label (census record); a member of the class of support measures (RAG faithfulness, citation precision/recall, attribution scores) that the archive sets beside the Provenance Erasure Rate as complementary: they ask whether cited sources support a claim, PER whether composition returns attribution at all and whether the lineage of the meaning survives (#716, #725, #95); support is not credit to the original author (census record), whether its failure is power-conditioned is unmeasured (#141), and local support must be read with the identity of the object it is attached to (#1695, #1705, #1680); not AttriBench (census record)", "claims": [ "PE-02", "PE-03", "PE-04", "PE-05", "PE-06", "PE-07", "O716-01", "O716-02", "O716-04", "O725-01", "O725-02", "O725-03", "O725-04", "O95-01", "PE-13", "O141-02", "O1695-02", "O1705-01", "O1680-01", "PE-12" ], "defining_papers": [ 716, 725, 95, 141, 1695, 1705, 1680, 1611 ], "defining_records": [ "provenance census record (census.json rows[slug == 'attributionbench'], organizing_question, schema_note; rendered aggregators/attributionbench/index.html; editor Lee Sharks, compiled 2026-10-29)", "Capture Registry attributionbench-aio-20261029 (OBS-90d2b25bd00a)" ] }, "rho": { "value": "extends", "status": "read from the census record, the capture record and the deposits that bear on the class (§1.2a); #716 calls PER 'complementary to, not competitive with, existing metrics', #725 names its relation 'Extension and completion'", "statement": "the archive reads E_F as the entity and adds senses and limits on the field's own object, support evaluation: lineage beside support (O725-01, O725-02, O725-03), the attempt beside the quality of attempted citation (O716-01, O716-02, O716-03), credit beside support (PE-13, CAP-1-07), power-conditioning (O141-01, O141-02), identity beside local support (O1695-01, O1695-02, O1705-01, O1680-01), and the census record's fields and homonym tag (PE-01–PE-21)", "claims": [ "O725-01", "O725-02", "O725-03", "O716-01", "O716-02", "O716-03", "PE-13", "CAP-1-07", "O141-01", "O141-02", "O1695-01", "O1695-02", "O1705-01", "O1680-01" ], "guard": "ρ = extends claims an addition and no more: no deposit names AttributionBench or disputes its finding; the archive's senses bear on the class by the archive's own application of its categories (O-class, class sourced from B1 and B3 and the census record), and #716 and #725 state their relation to the support measures as complementary and as extension. Where an archive sense and a field claim might be read as rivals (PER and the support metrics), the archive itself composes them as compatible measures of different things, and the composition holds them so." }, "translation": [ { "lineage": "what it is: a benchmark compiled from existing attribution datasets", "field_claims": [ "F1", "F5", "F11", "F30", "F42", "F43" ], "received": "F1 (B1, Abstract s4): \"we present AttributionBench, a comprehensive benchmark compiled from various existing attribution datasets.\"; F5 (B1, Abstract s4): \"To bridge the gap in the absence of standardized benchmarks for these methods\"; F11 (B5, Abstract s4): \"we present AttributionBench, a comprehensive benchmark compiled from various existing attribution datasets.\"; F30 (B2, README, Dataset): \"We constructed this dataset from multiple existing data sources in a unified format, in order to create a unified and diverse testbed for evaluating advanced attribution evaluation systems.\"; F42 (B6, Dataset Overview): \"We constructed this dataset from multiple existing data sources in a unified format\"; F43 (B6, Dataset Overview s2): \"The dataset contains both in-domain training set and id-domain and out-of-domain test set.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside PE-02, PE-03, PE-16, PE-17", "archive_claims": [ "PE-02", "PE-03", "PE-16", "PE-17" ] }, { "lineage": "the task: is every claim fully supported by its cited evidence", "field_claims": [ "F2", "F4" ], "received": "F2 (B1, Abstract s2): \"evaluating the answer's attribution, i.e., whether every claim within the generated responses is fully supported by its cited evidence, remains an open problem.\"; F4 (B1, Abstract s3): \"This verification, traditionally dependent on costly human evaluation, underscores the urgent need for automatic attribution evaluation methods.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside PE-04, PE-05, PE-06", "archive_claims": [ "PE-04", "PE-05", "PE-06" ] }, { "lineage": "the setting: generative search engines with cited evidence", "field_claims": [ "F3" ], "received": "F3 (B1, Abstract s1): \"Modern generative search engines enhance the reliability of large language model (LLM) responses by providing cited evidence.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside O725-05", "archive_claims": [ "O725-05" ] }, { "lineage": "the unit: a claim–evidence pair, attributable or not", "field_claims": [ "F31", "F32", "F33" ], "received": "F31 (B2, README, Data Fields: attribution_label): \"\"attributable\" or \"not attributable\".\"; F32 (B2, README, Data Fields: claim, references): \"Part of the response to the question. Could be one single sentence or multiple sentences. … A list of documents or paragraphs which could support the claim.\"; F33 (B2, README, Data Fields: citation_links): \"Reserved field for citation links.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside PE-07", "archive_claims": [ "PE-07" ] }, { "lineage": "the seven source datasets and their credit", "field_claims": [ "F15", "F16", "F17", "F28", "F40" ], "received": "F15 (B3, Data Statistics, Table 1): \"ExpertQA … Stanford-GenSearch … AttributedQA … LFQA … ID-Total … BEGIN … HAGRID … AttrEval-GenSearch … OOD-Total\"; F16 (B3, Data Statistics, Table 1, totals): \"ID-Total … 13322 … 1198 … 1610 … OOD-Total … 1686\"; F17 (B3, Table 1 caption): \"We make all subsets label-balanced to treat the two classes equally so that the evaluation can fairly show the performance of each class.\"; F28 (B3, Reference): \"If used, please also cite the original datasets accordingly:\"; F40 (B6, front matter, configs): \"The subset balanced version of the dataset. Each sub-dataset contains the same number of attributable labels and not attributable labels.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside PE-15", "archive_claims": [ "PE-15" ] }, { "lineage": "data access, licence and use", "field_claims": [ "F35", "F36", "F37", "F38", "F41" ], "received": "F35 (B4, dataset header): \"Tasks: … Text Classification … Modalities: … Text … Languages: … English … Size: … 10K - 100K … License: … apache-2.0\"; F36 (B4, Dataset Viewer): \"full_data (26.4k rows) … train (21.9k rows)dev (1.2k rows)test (1.61k rows)test_ood (1.69k rows)\"; F37 (B4, sidebar): \"Downloads last month … 1,481 … Number of rows: … 63,065\"; F38 (B4, sidebar): \"Models trained or fine-tuned on osunlp/AttributionBench … vllm-sr/Decision-1.0-Route-0.6B\"; F41 (B6, front matter): \"license: apache-2.0\"", "receiving_text": "—", "translation": "untranslated: composed as received beside PE-10, PE-21", "archive_claims": [ "PE-10", "PE-21" ] }, { "lineage": "makers, venue and dates", "field_claims": [ "F9", "F10", "F12", "F13", "F29", "F34", "F39" ], "received": "F9 (B1, header): \"Authors:Yifei Li, Xiang Yue, Zeyi Liao, Huan Sun … [Submitted on 23 Feb 2024]\"; F10 (B5, metadata): \"Anthology ID: 2024.findings-acl.886 … Findings of the Association for Computational Linguistics: ACL 2024 … Pages: 14919–14935\"; F12 (B3, header): \"Yifei Li … Xiang Yue, … Zeyi Liao, … Huan Sun … The Ohio State University\"; F13 (B3, Updates): \"2024/5/15: Our work is accepted to ACL 2024 Findings. … 2024/2/26: We have released the dataset and codebase for the paper.\"; F29 (B2, README, opening and Updates): \"This is the official code repo for the paper … 05/15/24: Our work is accepted to ACL 2024 Findings.\"; F34 (B2, README, Fine-tuning GPT-3.5): \"You can use our fine-tuned GPT-3.5 model with this model name: … ft:gpt-3.5-turbo-1106:the-ohio-state-university::8eWuGzGQ\"; F39 (B4, header): \"osunlp … AttributionBench … OSU NLP Group\"", "receiving_text": "—", "translation": "untranslated: composed as received beside PE-01, PE-11", "archive_claims": [ "PE-01", "PE-11" ] }, { "lineage": "headline finding: about 80% macro-F1", "field_claims": [ "F6", "F7" ], "received": "F6 (B1, Abstract s5): \"Our extensive experiments on AttributionBench reveal the challenges of automatic attribution evaluation, even for state-of-the-art LLMs.\"; F7 (B1, Abstract s6): \"even a fine-tuned GPT-3.5 only achieves around 80% macro-F1 under a binary classification formulation.\"", "receiving_text": "—", "translation": "untranslated: composed as received beside PE-09", "archive_claims": [ "PE-09" ] }, { "lineage": "difficulty by setting, and what helps", "field_claims": [ "F18", "F19", "F20", "F21", "F23", "F24" ], "received": "F18 (B3, ID & OOD Evaluation, finding 1): \"Fine-tuning on NLI-related data is beneficial to attribution evaluation\"; F19 (B3, ID & OOD Evaluation, finding 2): \"Automatic attribution evaluation is challenging under zero-shot setting … for tasks with longer evidence (like Stanford-GenSearch and HAGRID), the zero-shot performance for models is around 60%~70%, which is relatively low. … GPT-3.5, GPT-4 and those NLI-tuned models can only achieve under ~60% performance.\"; F20 (B3, ID & OOD Evaluation, finding 3): \"On average among all models, fine-tuning can improve 9.0% and 4.6% on 4 ID tasks and 3 OOD tasks, respectively.\"; F21 (B3, ID & OOD Evaluation, finding 4): \"Simply switching stronger models cannot significantly improve the performance\"; F23 (B3, input-field results): \"merely appending the questions and responses to the input, which human annotators might have access to, is not the key to solving this task.\"; F24 (B3, prompt results): \"Prompt engineering only brings limited gain over 7 test sets.\"", "receiving_text": "—", "translation": "untranslated: composed as received (no archive claim stands beside it)", "archive_claims": [] }, { "lineage": "error analysis", "field_claims": [ "F8", "F25", "F26", "F27" ], "received": "F8 (B1, Abstract s7): \"A detailed analysis of more than 300 error cases indicates that a majority of failures stem from the model's inability to process nuanced information, and the discrepancy between the information the model has access to and that human annotators do.\"; F25 (B3, Error Analysis, takeaway 1): \"Over 66% error cases are caused by fine-grained information insensitivity\"; F26 (B3, Error Analysis, takeaway 2): \"About 26.8% errors are caused by the mismatch between information accessible to the model and that accessible to human annotators: This disparity brings into question the reliability of labels within current attribution evaluation datasets.\"; F27 (B3, Error Analysis, table): \"Fine-grained information insensitivity (66.0%) … Need additional information/reference ambiguity (15.6%) … Label issue (11.2%)\"", "receiving_text": "—", "translation": "untranslated: composed as received beside CAP-1-05", "archive_claims": [ "CAP-1-05" ] }, { "lineage": "a stated limit: the labels and the evidence", "field_claims": [ "F14", "F22" ], "received": "F14 (B3, Figure 1 caption): \"The references are usually manually extracted from webpages by human annotators based on what they think is useful.\"; F22 (B3, Table 1 (ID results) caption): \"there is still a large room for improvement.\"", "receiving_text": "—", "translation": "untranslated: composed as received (no archive claim stands beside it)", "archive_claims": [] }, { "lineage": "the encyclopedia: no article", "field_claims": [ "F44" ], "received": "F44 (B7, rendered page; Special:Search): \"Wikipedia does not have an article with this exact name … There were no results matching the query.\"", "receiving_text": "—", "translation": "untranslated: composed as received (no archive claim stands beside it)", "archive_claims": [] } ], "receiving_texts": "equal-terms case: every field claim composed as received, beside the archive claims it stands with" }, "measure": { "rule": "v0.9 §4.10: KO_B sentences surviving in the KO at similarity ≥ 0.95, with positions (computed as apply_v09.py does: difflib.SequenceMatcher ratio, the best KO sentence for each KO_B sentence)", "v0.9": { "surviving": 0, "of": 12, "positions": [ null, null, null, null, null, null, null, null, null, null, null, null ] }, "opening_run": { "rule": "v0.9 §5.10 (equal-terms case): the field-only KO's sentences as the KO's opening run in their own order", "breach": false }, "blocks": { "rule": "v0.9 §4.1 (equal-terms and mixed cases): the KO does not open with the field as a block or close with the archive as one; sentences typed by the ledgers of the claims they cite (F field-only, A archive-only, M both)", "sequence": "MMMMFMFMMFMMFFFFMMAAAAAAAAAFAA", "field_only_head": 0, "archive_only_tail": 2, "breach": false }, "max_similarity": [ 0.359, 0.2, 0.715, 0.51, 0.458, 0.588, 0.459, 0.525, 0.539, 0.171, 0.556, 0.383 ], "ko_sentence_1_max_similarity_to_KO_B": 0.249 }, "homonyms": { "names": [ "AttriBench" ], "basis": "The census record's disambiguation names one neighbour, AttriBench (census slug attribench; 'quote attribution, author, suppression, Berman et al., 2026'; its dataset at huggingface.co/datasets/bermaneh/AttriBench, operator Eliza Berman, Bella Chang, Daniel B. Neill, Emily Black), and its negative tag keeps it apart ('NOT AttriBench: it asks whether evidence supports a claim, not who wrote a quote'). It is a distinct entity with its own /non row, attribench (datasets/negative-of-the-negative/v2/rows/attribench.json, built in parallel on 2026-10-29; not read or edited here), never a sense of this name. No card on T is a homonym; T never names AttriBench (CAP-1-03). The general sense of 'attribution' as credit to an author (the archive's provenance work) is a sense of the word, not of the entity, and the census's second tag keeps it apart ('evidence support is not credit to the original author').", "distinct_entities": [ "AttriBench (Berman et al., quote attribution; row attribench)", "the seven constituent datasets (ExpertQA; Stanford-GenSearch, from Liu, Zhang and Liang's 'Evaluating Verifiability in Generative Search Engines'; AttributedQA; LFQA; BEGIN; HAGRID; AttrEval-GenSearch), each its own work with its own makers, whose citation the makers request (F28)", "AttrScore (Yue et al. 2023, 'Automatic Evaluation of Attribution by Large Language Models'; the AttrScore models in the tables)", "AIS (Rashkin et al.) and ALCE (Gao et al.), the support metrics #716 tables", "the Provenance Erasure Rate and Erasure Skew (the archive's measures beside the class)" ], "substitutes": { "T": [] }, "register": { "address_1": { "slug": "attributionbench-aio-20261029", "obs_id": "OBS-90d2b25bd00a" } }, "neighbour_row": "datasets/negative-of-the-negative/v2/rows/attribench.json" }, "notes": [ "One address, Google AI Overview (expanded), signed out, incognito, 2026-10-29: attributionbench-aio-20261029 (OBS-90d2b25bd00a, ADDR-46cc2d945075, AXN:0ACB.CAPTURE.📖🛡️🕐🔖❌↖️), five cards, 176 words in the answer body. A NEW address; the census record landed the same day and no card cites it (CAP-1-09).", "Wikipedia: no article. AttributionBench returns HTTP 404 for raw wikitext (a Wikimedia error page) and for the rendered page ('Wikipedia does not have an article with this exact name'), no redirect, and a full-text search of article space for the quoted name reports 'There were no results matching the query.' (B7).", "Fetching: every reachable page saved verbatim with curl on 2026-10-29 between 17:06 and 17:08 UTC, raw and text with sha256 in field.json. Card URLs: B1 from the card's arXiv number; B2 and B4 from their titles; B3 is the census's own page (fetched fresh; byte-identical to the census's saved copy, so carried once, as a card); B5 found by WebSearch on the card's title. Cards 3 and 4 carry snippets that are not verbatim on their pages (paraphrases of the abstract and the dataset metadata); the snippets are recorded with each card and not ledgered as claims.", "Unreachable: B2's rendered repository page (github.com/OSU-NLP-Group/AttributionBench): three fetches answered HTTP 403 by the session's proxy ('GitHub access to this repository is not enabled for this session'). A browser user-agent retry was refused by the workspace and not pursued. The README the page renders was read through the GitHub contents API (blob 9566ea8d…, commit c81e9e11…) and saved as B2-README.md, verbatim through the first BibTeX entry, the constituent datasets' entries omitted and marked (they stand verbatim on B3). Every B2 claim quotes that copy.", "The archive subset: the string pass (non_traverse.py) returned no D and no R candidate: no archive text names AttributionBench, its arXiv number, dataset id or title, and none names AttriBench. A first class pattern with bare 'attributable' returned 66 generic uses in 41 deposits, read and set aside; the narrowed pattern returned 11 O-class candidates in 7 deposits. Reading admitted 10 deposits at O-class, 8 carrying claims (#716, #725, #95, #141, #1695, #1705, #1680, #1611) and 2 grouped as instances by S4 (#1190 under #725, #1188 under #95); #95, #1188, #1695 and #1705 were found by reading beyond the pass (grep over data/texts for the class's measures), #141, #1611 and #1680 at the hop (97 candidates in 92 deposits, scanned by the class's measures and read at the hits). Rejected with reasons in reading.json: #729, #1369, #741, #1703 and the rest of the hop. String-pass defects, reported, not fixed: (1) sentences are cut at 900 characters, so a phrase late in a long paragraph is missed (#95's 'attribution evaluation'); (2) Markdown emphasis inside a phrase ('*supported by*') breaks a class pattern (#725 §3).", "Defect in the capture record (report; not fixed): CAP-1-05 (findings[3]) says the 66% and 27% error shares are 'unchecked'. The project page states them, 'Over 66% error cases are caused by fine-grained information insensitivity' and 'About 26.8% errors are caused by the mismatch …' (B3, F25, F26), and B3 is byte-identical to the census's own saved copy of the page (attributionbench--project, sha256 0bb85b93…), so the figures are checkable against the record's own source and check: the composition's 'nearly 27%' rounds 26.8%.", "Census fields a fetched source contradicts or shows out of date (report; not fixed): (1) lifecycle.detail 'arXiv:2402.15089' with state 'published': the paper is published in Findings of the Association for Computational Linguistics: ACL 2024, Anthology ID 2024.findings-acl.886, pages 14919–14935 (B5, F10), accepted 15 May 2024 (B3, B2); the field gives the preprint only. (2) transformations_tested 'generation with citations by search-augmented models': the project page's Table 1 gives, as response sources, search engines for some sets but also knowledge-grounded dialogue systems for BEGIN ('GPT2-base, T5-base, CTRL-Dialog, DoHA'), PaLM for AttributedQA and 'WebGPT, GPT-3.5, Alpaca' for LFQA (B3, Table 1; the response-source column is not ledgered beyond F15); the field is narrower than the benchmark. (3) unchecked[0] lists 'AttrScore' among 'the constituent datasets … named on the card': the seven constituent datasets are ExpertQA, Stanford-GenSearch, AttributedQA, LFQA, BEGIN, HAGRID and AttrEval-GenSearch (B3, F15); 'AttrScore-GenSearch' is the src_dataset label of the dataset card's example item and AttrScore names models in the tables. (4) retention_results is the abstract's figure as quoted; the capture shows the composition extending it to GPT-4 (delta), which is the composition's, not the record's.", "Defect in a field source (recorded, not ledgered): on the project page's Table 2 (OOD results) the zero-shot rows for FLAN-T5 (770M) and FLAN-T5 (3B) repeat, figure for figure, the fine-tuned rows for the same models (B3).", "Composer's reading of the case: equal (ρ = extends), from #716's and #725's own statements of relation to the support measures and the census record's tag; see ontology.case. The archive's senses stand together in the KO (sentences 18–25) because every one is O-class, a sense of the class; sentence 18 joins them to the field's task (F2), and the blocks check records no head or tail breach.", "Neighbour: AttriBench (row attribench, datasets/negative-of-the-negative/v2/rows/attribench.json), built in parallel; named in homonyms, not read or edited.", "Shapes for the page builder: objects.T is one dict with one transcript; field is flat with field_by_origin beside; ontology.composition 'equal'; E_A.defining_papers integers, defining_records strings; CAP- and PE- claims carry 'dep' as a string ('capture-registry', 'PE'), 'surface' and 'record'; rails give 'earliest' as a field source id, a deposit number, 'PE' or 'capture:'. build_non.py, non_axn.py and panel.json untouched." ], "composition_v09": { "source": "this row (composed directly; no rebuild/negative-of-the-negative/v09/attributionbench.json)", "composer": "Claude (Opus 5.5), subagent, 2026-10-29", "defining_material_read": [ { "what": "provenance census record, row", "file": "/home/claude/fleet/provenance-erasure/data/provenance-census/census.json (rows[slug == 'attributionbench'], organizing_question, schema_note; and rows[slug == 'attribench'] for the neighbour's name, url and operator)", "read": "whole" }, { "what": "provenance census record, rendered", "file": "/home/claude/fleet/provenance-erasure/aggregators/attributionbench/index.html", "read": "text, for agreement with the sources" }, { "what": "the census's saved sources", "file": "/home/claude/fleet/provenance-erasure/data/provenance-census/sources/attributionbench--project.*, attributionbench--dataset-card.*", "read": "whole" }, { "what": "capture record", "file": "data/EA-WG-CAPTURES-01.json (attributionbench-aio-20261029)", "read": "transcript, transcript_raw, cite_list, d, reading, findings, analysis, q_kind, per_note, originator" }, { "dep": 716, "file": "data/texts/AXN-025F-text.md", "read": "§2 Related Work (2.1–2.3) and the metric table" }, { "dep": 725, "file": "data/texts/AXN-0273-text.md", "read": "header, summary, §3 (the 'PER is not' paragraphs), the disambiguation table, the test queries" }, { "dep": 95, "file": "data/texts/AXN-026A-text.md", "read": "the closing paragraph ('Three near-term moves')" }, { "dep": 141, "file": "data/texts/AXN-02CE-text.md", "read": "§3 Entity Definitions, §7 Inverted Fan, §11 Test Queries" }, { "dep": 1695, "file": "data/texts/AXN-0705-text.md", "read": "§2.5" }, { "dep": 1705, "file": "data/texts/AXN-0728-text.md", "read": "R1–R3" }, { "dep": 1680, "file": "data/texts/AXN-06F6-text.md", "read": "§11 (comparison facts; attribution discipline)" }, { "dep": 1611, "file": "data/texts/AXN-069B-text.md", "read": "the paragraph on rediscovery and candidate convergences" }, { "what": "the specification", "file": "data/texts/AXN-0700-text.md", "read": "§0–§6 whole; Appendix A by its headings" } ], "cuts": [], "notes": "equal case; KO 30 sentences, all 44 field and 49 archive claims cited; P body 274 words; KO_B 12 sentences, none surviving at ≥ 0.95" }, "field_by_origin": { "cards": [ "B1", "B2", "B3", "B4", "B5" ], "own_pages": [ "B6" ], "census_third_party": [], "wikipedia": [ "B7" ], "unreachable": [ "B2 (the rendered repository page; its README read)" ], "title_only": [], "own_page_also_card": [ "B3 (the census's attributionbench--project; card 3)" ], "wikipedia_state": "no article: 'AttributionBench' is absent (raw wikitext HTTP 404, a Wikimedia error page; rendered page HTTP 404 'Wikipedia does not have an article with this exact name'), no redirect, and a full-text search of article space for the quoted name returns 'There were no results matching the query.' (2026-10-29)", "sources": [ { "id": "B1", "origin": "card", "role": "Overview card 1", "card": "arXiv — [2402.15089] AttributionBench: How Hard is Automatic Attribution Evaluation?", "url": "https://arxiv.org/abs/2402.15089", "kind": "Overview card (the paper's abstract page)", "surfaced_by_T": [ { "slug": "attributionbench-aio-20261029", "card": 1 } ], "files": { "B1.html": { "path": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B1.html", "sha256": "49a126a35a92ab507781b8d2fb27b13ddf6ece231c1a89a97a15ed3aa8c5333b" }, "B1.txt": { "path": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B1.txt", "sha256": "6b08afffb50fb894a08a5c88bf26e16b5ec0e53cc002ecf06aff8c01fbc09863" } }, "fetched": { "at": "2026-10-29T17:06:11Z", "http": 200, "url": "https://arxiv.org/abs/2402.15089", "raw": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B1.html", "raw_sha256": "49a126a35a92ab507781b8d2fb27b13ddf6ece231c1a89a97a15ed3aa8c5333b", "text": [ "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B1.txt" ] }, "card_title": "[2402.15089] AttributionBench: How Hard is Automatic Attribution Evaluation?", "card_snippet": "AttributionBench: How Hard is Automatic Attribution Evaluation? Yifei Li, Xiang Yue, Zeyi Liao, Huan Sun View a PDF of the paper titled AttributionBench: How Ha...", "card_note": null, "names_operator": "yes (the four authors; no institution on the page)", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, provenanceerasure, machinemediation, EA-WG-CAPTURES-01 in the saved text, card title and snippet)", "note": "URL from the card's arXiv number (2402.15089), confirmed by the page title. Text extracted from the saved HTML (scripts and styles stripped)." }, { "id": "B2", "origin": "card", "role": "Overview card 2", "card": "GitHub — GitHub - OSU-NLP-Group/AttributionBench: [ACL'24 Findings] AttributionBench: How Hard is Automatic Attribution Evaluation? · GitHub", "url": "https://github.com/OSU-NLP-Group/AttributionBench", "kind": "Overview card (the code repository; page unreachable, README read through the GitHub contents API)", "surfaced_by_T": [ { "slug": "attributionbench-aio-20261029", "card": 2 } ], "files": { "B2.html": { "path": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B2.html", "sha256": "b3e03aaaff81d68730d67392a135ac3fdfdf022880c66632ab188d2fe084cda3" }, "B2.txt": { "path": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B2.txt", "sha256": "4d67d0954e7b23d4e4681bd98f30b415c6746d80095b910997e4c5c37b66fc2f" }, "B2-README.md": { "path": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B2-README.md", "sha256": "4d67d0954e7b23d4e4681bd98f30b415c6746d80095b910997e4c5c37b66fc2f" } }, "fetched": { "at": "2026-10-29T17:06:12Z", "http": 403, "url": "https://github.com/OSU-NLP-Group/AttributionBench", "raw": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B2.html", "raw_sha256": "b3e03aaaff81d68730d67392a135ac3fdfdf022880c66632ab188d2fe084cda3", "text": [ "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B2.txt" ], "unreachable_page": true, "tries": [ { "at": "2026-10-29T17:06:12Z", "url": "https://github.com/OSU-NLP-Group/AttributionBench", "http": 403, "error": "the session's proxy: 'GitHub access to this repository is not enabled for this session' (B2.html, 378 bytes)" }, { "at": "2026-10-29T17:06:48Z", "url": "https://github.com/OSU-NLP-Group/AttributionBench", "http": 403, "error": "same response, same sha256" }, { "at": "2026-10-29T17:06:55Z", "url": "https://github.com/OSU-NLP-Group/AttributionBench", "http": 403, "error": "same response, same sha256" } ], "readme": { "via": "GitHub contents API (the session's GitHub tool, get_file_contents README.md), 2026-10-29T17:07Z", "blob_sha": "9566ea8dfcb7bce2eb05a9efe441c19b593d17ad", "commit": "c81e9e114a074b49b08900afed8811799a308cfe", "saved": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B2-README.md", "note": "verbatim through the Citation section's first BibTeX entry; the constituent datasets' BibTeX entries that follow are omitted from the saved copy and marked so in it (they are the same entries the project page B3 carries verbatim)" } }, "card_title": "GitHub - OSU-NLP-Group/AttributionBench: [ACL'24 Findings] AttributionBench: How Hard is Automatic Attribution Evaluation? · GitHub", "card_snippet": "Paper: AttributionBench: How Hard is Automatic Attribution Evaluation? (Accepted to ACL 2024 Findings). Dataset: Unified testbed for evaluating attribution eval...", "card_note": null, "names_operator": "yes ('the-ohio-state-university' in the fine-tuned model name; the four authors in the BibTeX)", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, provenanceerasure, machinemediation, EA-WG-CAPTURES-01 in the saved text, card title and snippet)", "note": "URL from the card's title (OSU-NLP-Group/AttributionBench). The rendered repository page could not be fetched: the session's proxy answers 403 for github.com. The repository's README (the text the repository page renders) was read through the GitHub contents API and saved as B2-README.md; B2.txt is that copy. B2.html is the 403 response, kept as the fetch record." }, { "id": "B3", "origin": "card", "role": "Overview card 3; also the census record's own page (attributionbench--project)", "card": "GitHub Pages documentation — AttributionBench: How Hard is Automatic Attribution Evaluation?", "url": "https://osu-nlp-group.github.io/AttributionBench/", "kind": "Overview card (own page: the project page)", "surfaced_by_T": [ { "slug": "attributionbench-aio-20261029", "card": 3 } ], "files": { "B3.html": { "path": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B3.html", "sha256": "0bb85b9388499eb582d61161b89e6426162c02ca2e9ee73d9dc080a0f29c5d1e" }, "B3.txt": { "path": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B3.txt", "sha256": "28108d5cfa92ce35ca5f39681e73bc549b98e5e1427628c8fd0b8dbdbb2471a7" } }, "fetched": { "at": "2026-10-29T17:06:13Z", "http": 200, "url": "https://osu-nlp-group.github.io/AttributionBench/", "raw": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B3.html", "raw_sha256": "0bb85b9388499eb582d61161b89e6426162c02ca2e9ee73d9dc080a0f29c5d1e", "text": [ "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B3.txt" ], "census_copy": { "source": "attributionbench--project", "url": "https://osu-nlp-group.github.io/AttributionBench/", "read": "2026-10-29", "sha256": "0bb85b9388499eb582d61161b89e6426162c02ca2e9ee73d9dc080a0f29c5d1e", "identical": true } }, "card_title": "AttributionBench: How Hard is Automatic Attribution Evaluation?", "card_snippet": "Overview: AttributionBench is a comprehensive benchmark compiled from existing attribution datasets (ExpertQA, Stanford-GenSearch, AttributedQA, LFQA, BEGIN, HA...", "card_note": null, "names_operator": "yes ('The Ohio State University'; the four authors)", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, provenanceerasure, machinemediation, EA-WG-CAPTURES-01 in the saved text, card title and snippet)", "note": "URL is the census record's own page; the card's snippet ('Overview: AttributionBench is a comprehensive benchmark compiled from existing attribution datasets (ExpertQA, Stanford-GenSearch, AttributedQA, LFQA, BEGIN, HA...') matches no sentence of the page verbatim (it paraphrases the abstract and Table 1). Fetched fresh; the served HTML is byte-identical to the census's saved copy (same sha256), so the own page and the card are one source, carried once, as a card." }, { "id": "B4", "origin": "card", "role": "Overview card 4", "card": "Hugging Face — osunlp/AttributionBench · Datasets at Hugging Face", "url": "https://huggingface.co/datasets/osunlp/AttributionBench", "kind": "Overview card (the dataset page)", "surfaced_by_T": [ { "slug": "attributionbench-aio-20261029", "card": 4 } ], "files": { "B4.html": { "path": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B4.html", "sha256": "5ed3022c2100a31faae381645b32d249fae87512d233d63e9addf0f8768286a6" }, "B4.txt": { "path": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B4.txt", "sha256": "75ae249b6378add4ce5e65d09a7482a921691316aa7b3ba35d648b0467fdc355" } }, "fetched": { "at": "2026-10-29T17:06:14Z", "http": 200, "url": "https://huggingface.co/datasets/osunlp/AttributionBench", "raw": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B4.html", "raw_sha256": "5ed3022c2100a31faae381645b32d249fae87512d233d63e9addf0f8768286a6", "text": [ "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B4.txt" ] }, "card_title": "osunlp/AttributionBench · Datasets at Hugging Face", "card_snippet": "Dataset: osunlp/AttributionBench (arXiv: 2402.15089, created by Yifei Li, Xiang Yue, Zeyi Liao, Huan Sun). Task/Modality: Text Classification, Text modality, En...", "card_note": null, "names_operator": "yes ('OSU NLP Group', the account; 'Yifei Li' as point of contact)", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, provenanceerasure, machinemediation, EA-WG-CAPTURES-01 in the saved text, card title and snippet)", "note": "URL from the card's title (osunlp/AttributionBench). The card's snippet ('Dataset: osunlp/AttributionBench (arXiv: 2402.15089, created by Yifei Li, Xiang Yue, Zeyi Liao, Huan Sun). Task/Modality: …') is not on the page in that form; the page names the arXiv number, the task, the modality and the papers. Download and row counts are as fetched." }, { "id": "B5", "origin": "card", "role": "Overview card 5", "card": "ACL Anthology — AttributionBench: How Hard is Automatic Attribution Evaluation?", "url": "https://aclanthology.org/2024.findings-acl.886/", "kind": "Overview card (the publication record)", "surfaced_by_T": [ { "slug": "attributionbench-aio-20261029", "card": 5 } ], "files": { "B5.html": { "path": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B5.html", "sha256": "c59c736a18a81530bc705482a966c4a4c7652e72fee48f1ca46e6397af1a9b02" }, "B5.txt": { "path": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B5.txt", "sha256": "eb33139a01c1b16e1f506ff625f380f10c36e33169ec8ff7c82de62afeccdcd2" } }, "fetched": { "at": "2026-10-29T17:06:17Z", "http": 200, "url": "https://aclanthology.org/2024.findings-acl.886/", "raw": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B5.html", "raw_sha256": "c59c736a18a81530bc705482a966c4a4c7652e72fee48f1ca46e6397af1a9b02", "text": [ "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B5.txt" ] }, "card_title": "AttributionBench: How Hard is Automatic Attribution Evaluation?", "card_snippet": "Abstract Modern generative search engines enhance the reliability of large language model (LLM) responses by providing cited evidence. However, evaluating the a...", "card_note": null, "names_operator": "yes (the four authors; the venue)", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, provenanceerasure, machinemediation, EA-WG-CAPTURES-01 in the saved text, card title and snippet)", "note": "URL found by WebSearch on the card's title and site (preview.aclanthology.org/setup/2024.findings-acl.886 returned; the canonical aclanthology.org path fetched), confirmed by the page title and the Anthology ID 2024.findings-acl.886, which the GitHub README also links." }, { "id": "B6", "origin": "own_page", "role": "the census record's own page (attributionbench--dataset-card), not surfaced by T", "card": "Hugging Face — osunlp/AttributionBench, dataset card (raw README.md)", "url": "https://huggingface.co/datasets/osunlp/AttributionBench/raw/main/README.md", "kind": "own page (the dataset card's source)", "surfaced_by_T": [], "files": { "B6.html": { "path": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B6.html", "sha256": "b90e7e73959c0b91966cb75cb330d03f16600e51f9c5a10c525787aa4ab31c6c" }, "B6.txt": { "path": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B6.txt", "sha256": "b90e7e73959c0b91966cb75cb330d03f16600e51f9c5a10c525787aa4ab31c6c" } }, "fetched": { "copied_from": "/home/claude/fleet/provenance-erasure/data/provenance-census/sources/attributionbench--dataset-card.html.gz and .txt", "census_read": "2026-10-29", "http": "200", "census_sha256": "b90e7e73959c0b91966cb75cb330d03f16600e51f9c5a10c525787aa4ab31c6c", "raw": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B6.html", "raw_sha256": "b90e7e73959c0b91966cb75cb330d03f16600e51f9c5a10c525787aa4ab31c6c", "identical": true, "text": [ "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B6.txt" ], "refetched": false }, "card_title": null, "card_snippet": null, "card_note": null, "names_operator": "yes ('Yifei Li' as point of contact; the four authors in the BibTeX)", "names_archive": "no (none of the Crimson Hexagonal Archive, Alexanarch, Lee Sharks, provenanceerasure, machinemediation, EA-WG-CAPTURES-01 in the saved text, card title and snippet)", "note": "The census's saved copy, gunzipped (sha256 matches the census's record); the file is Markdown with YAML front matter, so B6.txt is the census's .txt, byte-identical to B6.html. Not surfaced by T: the Overview's card 4 is the rendered dataset page (B4), which carries the same card text rendered." }, { "id": "B7", "origin": "wikipedia", "role": "the entity's Wikipedia article: none", "card": "Wikipedia — AttributionBench (no article)", "url": "https://en.wikipedia.org/wiki/AttributionBench", "kind": "encyclopedia (absent)", "surfaced_by_T": [], "files": { "B7.wiki": { "path": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B7.wiki", "sha256": "591ae6cb52077d97eaa73e1b3f70d6f1246947c390c67fb2e40c2d58312021ba" }, "B7.html": { "path": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B7.html", "sha256": "73489a6b73b36a826e4b7cdf77e2932539b41ce07948fcb8449d620333a0d830" }, "B7.txt": { "path": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B7.txt", "sha256": "c53977b66c23d05e247b7b01e18f0a1f6033362ee013c39bd535e064d3198624" }, "B7-search.html": { "path": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B7-search.html", "sha256": "b273f0a4155c43edcca255c003980cd27d8cc9e15847bbefb667e2081c16f1e0" }, "B7-search.txt": { "path": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B7-search.txt", "sha256": "836c42213f41c46b93d7fc95ef02c9167d0940c4d054ebcf552ae29ad8b319e8" } }, "fetched": { "raw_wikitext": { "at": "2026-10-29T17:06:18Z", "url": "https://en.wikipedia.org/w/index.php?title=AttributionBench&action=raw", "http": 404, "saved": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B7.wiki" }, "rendered": { "at": "2026-10-29T17:06:19Z", "url": "https://en.wikipedia.org/wiki/AttributionBench", "http": 404, "saved": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B7.html" }, "search": { "at": "2026-10-29T17:07:57Z", "url": "https://en.wikipedia.org/w/index.php?search=%22AttributionBench%22&title=Special:Search&profile=advanced&fulltext=1&ns0=1", "http": 200, "saved": "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B7-search.html" }, "text": [ "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B7.txt", "datasets/negative-of-the-negative/v2/intake/attributionbench-20261029/B7-search.txt" ] }, "card_title": null, "card_snippet": null, "card_note": null, "names_operator": "n/a", "names_archive": "no", "note": "No article and no redirect: raw wikitext 404 (a Wikimedia error page), rendered page 404 ('Wikipedia does not have an article with this exact name'), full-text search of article space for the quoted name: no results." } ] } }