{
  "schema_version": "1.0.0",
  "title": "The information in diagnostic tests",
  "short_title": "Diagnostic information",
  "review_type": "mapping_review",
  "methodology_profile": "systematic-review-general-v1",
  "corresponding_owner": {
    "name": "Shuhan He",
    "email": "she8@mgb.org",
    "orcid": "https://orcid.org/0000-0002-9902-5077"
  },
  "contributors": [
    {
      "name": "Shuhan He",
      "orcid": "https://orcid.org/0000-0002-9902-5077",
      "affiliation_keys": [
        "A1",
        "A2",
        "A3",
        "A4"
      ]
    },
    {
      "name": "Joshua W. Joseph",
      "orcid": "https://orcid.org/0000-0001-9704-6635",
      "affiliation_keys": [
        "A1"
      ]
    },
    {
      "name": "David M. Liebovitz",
      "orcid": "https://orcid.org/0000-0002-2518-5940",
      "affiliation_keys": [
        "A5"
      ]
    },
    {
      "name": "Cory Rohlfsen",
      "orcid": "https://orcid.org/0000-0002-4118-5174",
      "affiliation_keys": [
        "A6"
      ]
    },
    {
      "name": "Brian W. Locke",
      "orcid": "https://orcid.org/0000-0002-3588-5238",
      "affiliation_keys": [
        "A7"
      ]
    },
    {
      "name": "Allison Goff",
      "orcid": "https://orcid.org/0000-0002-1557-618X",
      "affiliation_keys": [
        "A4"
      ]
    },
    {
      "name": "Pedram Safari",
      "orcid": "https://orcid.org/0000-0002-0786-1415",
      "affiliation_keys": [
        "A4"
      ]
    },
    {
      "name": "Christopher Kabrhel",
      "orcid": "https://orcid.org/0000-0002-8699-7176",
      "affiliation_keys": [
        "A8",
        "A3"
      ]
    },
    {
      "name": "Amal Mohamed",
      "orcid": "https://orcid.org/0009-0006-0281-8625",
      "affiliation_keys": [
        "A1"
      ]
    },
    {
      "name": "Ali S. Raja",
      "orcid": "https://orcid.org/0000-0002-8909-3586",
      "affiliation_keys": [
        "A9"
      ]
    },
    {
      "name": "Joshua N. Goldstein",
      "orcid": "https://orcid.org/0000-0002-6406-1828",
      "affiliation_keys": [
        "A9"
      ]
    }
  ],
  "affiliations": [
    {
      "id": "A1",
      "name": "Division of AI, Department of Emergency Medicine, Mass General Brigham, Boston, Massachusetts, USA"
    },
    {
      "id": "A2",
      "name": "Laboratory of Computer Science, Massachusetts General Hospital, Boston, Massachusetts, USA"
    },
    {
      "id": "A3",
      "name": "Harvard Medical School, Boston, Massachusetts, USA"
    },
    {
      "id": "A4",
      "name": "Master of Science in Data Analytics, Health Sciences, Mass General Brigham University, Boston, Massachusetts, USA"
    },
    {
      "id": "A5",
      "name": "Departments of Medicine and Preventive Medicine, Northwestern University Feinberg School of Medicine, Chicago, Illinois, USA"
    },
    {
      "id": "A6",
      "name": "Department of Internal Medicine, University of Nebraska Medical Center, Omaha, Nebraska, USA"
    },
    {
      "id": "A7",
      "name": "Department of Pulmonary and Critical Care, Intermountain Medical Center, Murray, Utah, USA"
    },
    {
      "id": "A8",
      "name": "Department of Emergency Medicine, Massachusetts General Hospital, Boston, Massachusetts, USA"
    },
    {
      "id": "A9",
      "name": "Department of Emergency Medicine, Mass General Brigham, Boston, Massachusetts, USA"
    }
  ],
  "funding": [
    {
      "source": "National Academy of Medicine",
      "award": "2026A008797",
      "role": "The manuscript states that the funder had no role in design, acquisition, analysis, interpretation, writing, or submission."
    }
  ],
  "conflicts_of_interest": "The authors declare support from the National Academy of Medicine for the submitted work (Agreement No. 2026A008797); no other competing interests are declared.",
  "license": "CC-BY-4.0",
  "background": "Diagnostic testing changes what is known about a target condition. Sensitivity and specificity describe test performance; likelihood ratios and post-test probabilities describe an observed result. Mutual information expresses the expected reduction in uncertainty before either result is known. Earlier studies established diagnostic applications of this measure. This study assembles a broad empirical reference scale from accessible diagnostic-review data so readers can interpret a diagnostic profile at a stated starting probability and compare its information with conventional accuracy measures. Information describes learning about a binary target. Clinical value also depends on consequences, available actions, and patient preferences.",
  "review_question": "Across reproducibly accessible diagnostic accuracy reviews, how much uncertainty about a binary target condition is a test or threshold expected to resolve at a stated starting probability, and how does this information relate to conventional test-performance measures?",
  "objectives": [
    "Build an empirical reference distribution of expected diagnostic information from source-defined tests and thresholds.",
    "Compare uncertainty resolved at 5%, 20%, and 50% starting disease probability and across a 1%-50% grid.",
    "Examine agreement and differences between diagnostic information and Youden's J using source-comparable examples.",
    "Distinguish expected information before testing from the probability update and uncertainty change after each result."
  ],
  "framework": {
    "name": "PIRD",
    "population_or_domain": "Review-defined populations and settings across binary diagnostic targets; retain source-defined population and reference standard when available.",
    "intervention_or_exposure": "A source-defined diagnostic test, rule, threshold, or analysis definition.",
    "comparators": "Conventional measures from the same sensitivity and specificity; descriptive within-review contrasts where definitions are comparable.",
    "outcomes": [
      "Mutual information in bits",
      "Percentage of starting binary uncertainty resolved",
      "Result-specific probability and uncertainty changes"
    ]
  },
  "primary_outcomes": [
    "Distribution of 100 x I(D;T)/H(D) at 5%, 20%, and 50% starting probabilities, with corresponding information in bits; each pooled estimate has equal weight."
  ],
  "secondary_outcomes": [
    "Information profiles from 1% to 50% in one-percentage-point steps.",
    "Spearman correlations with Youden's J and descriptive near-tie comparisons.",
    "CEA information crossings and Ottawa ankle rule result branches.",
    "Sensitivity to zero-cell correction and the regularized direct-binomial estimator."
  ],
  "eligibility": {
    "inclusion": [
      "Diagnostic-review data with explicit study identifiers, source links, and non-negative integer TP, FP, FN, TN counts.",
      "Groups with at least five unique study identifiers, one result per identifier, and at least three studies with both disease states represented.",
      "Aggregate sensitivity plus specificity at least one in the source-defined orientation, with one canonical representation after exact row-signature deduplication.",
      "Historical Cochrane groups linked to review-author-reported pooled findings; modern Cochrane main analyses; at most one canonical main table per OSF, Zenodo, or PMC review under the recorded priority rules."
    ],
    "exclusion": [
      "Too few studies, repeated identifiers within a group, too few studies representing both disease states, or below-chance performance in the original orientation.",
      "Counts reconstructed from rounded accuracy measures, pooled-only or consequence tables, and unsupported narrative transcription.",
      "Duplicate representations and alternatives not selected by source-specific primary designation; preserve their status in the audit record."
    ],
    "study_designs": [
      "Secondary methodological synthesis of diagnostic accuracy reviews with accessible study-level cross-classification data; original designs are inherited from each source review."
    ]
  },
  "information_sources": [
    "Fixed historical and modern Cochrane sources",
    "OSF public deposits",
    "Zenodo public deposits",
    "Figshare public deposits",
    "Dryad public deposits",
    "SRDR+ public deposits",
    "PubMed Central open-access review tables"
  ],
  "search_strategies": [
    {
      "source": "Fixed historical and modern Cochrane sources",
      "platform": "Cochrane reference dataset and published review packages",
      "query": "Retrieve CL145_open_set_20181101.zip from https://zenodo.org/records/1303259 (SHA-256 f921cf6f1a9236c86f89e7c646497639f45e8ff8fcf56008c1b27422829fec4f), and the already-held published packages for CD014911 and CD015089. Parse available study-level 2x2 tables with scripts/cochrane_dta_atlas.py at the pinned commit. This is fixed-corpus acquisition, not a new exhaustive Cochrane search.",
      "status": "executed",
      "planned_or_executed_at": "2026-08-16",
      "filters": [],
      "limits": "63 historical reviews and two selected modern reviews. The count and checksum identify the normalized corpus manifest. Acquisition and analysis are documented by 16 August 2026; the first download date of already-held packages is not recorded.",
      "result_count": 65,
      "export_checksum": "1a17f47b78c0e2ba7c2f326111e18d2a387d04780b2a36e7b7fe54a874f51321",
      "responsible": {
        "kind": "person",
        "name": "Shuhan He"
      }
    },
    {
      "source": "OSF public deposits",
      "platform": "Frozen OSF metadata inventory and public-file inspection",
      "query": "Run each recorded query separately:\n\"diagnostic test accuracy\"\n\"diagnostic accuracy\"\n\"sensitivity and specificity\"\nsensitivity specificity meta-analysis\n\"true positive\" \"false positive\"\n\"2x2\" diagnostic",
      "status": "executed",
      "planned_or_executed_at": "2026-08-17",
      "filters": [],
      "limits": "The count and checksum identify the saved discovery inventory. Eligible evidence consists of publicly retrievable structured diagnostic-review data.",
      "result_count": 1152,
      "export_checksum": "1f46eedb7af4d6d0724d55156ba98ebb92572f3b0c4378e521400d2cd634ec44",
      "responsible": {
        "kind": "person",
        "name": "Shuhan He"
      }
    },
    {
      "source": "Zenodo public deposits",
      "platform": "Zenodo public metadata API",
      "query": "Run each recorded query separately:\n\"diagnostic test accuracy\"\n\"diagnostic accuracy\" AND \"systematic review\"\n\"diagnostic accuracy\" AND \"meta-analysis\"\n\"test accuracy\" AND \"meta-analysis\"\n\"sensitivity and specificity\" AND \"meta-analysis\"\n\"true positive\" AND \"false positive\" AND \"meta-analysis\"\n\"2x2\" AND diagnostic AND \"meta-analysis\"",
      "status": "executed",
      "planned_or_executed_at": "2026-08-17",
      "filters": [],
      "limits": "The count and checksum identify the saved discovery inventory. Eligible evidence consists of publicly retrievable structured diagnostic-review data.",
      "result_count": 799,
      "export_checksum": "0fa8e301ef599db6657089e96ef341e816f790e444a2c5fe28da202d4e61a44c",
      "responsible": {
        "kind": "person",
        "name": "Shuhan He"
      }
    },
    {
      "source": "Figshare public deposits",
      "platform": "Figshare public metadata API",
      "query": "Run each recorded query separately:\n\"diagnostic test accuracy\"\n\"diagnostic accuracy\" AND \"systematic review\"\n\"diagnostic accuracy\" AND \"meta-analysis\"\n\"test accuracy\" AND \"meta-analysis\"\n\"sensitivity and specificity\" AND \"meta-analysis\"\n\"true positive\" AND \"false positive\" AND \"meta-analysis\"\n\"2x2\" AND diagnostic AND \"meta-analysis\"",
      "status": "executed",
      "planned_or_executed_at": "2026-08-17",
      "filters": [],
      "limits": "The count and checksum identify the saved discovery inventory. Eligible evidence consists of publicly retrievable structured diagnostic-review data.",
      "result_count": 262,
      "export_checksum": "b38dfd72097e51a97cc52f0d679097e3897e29249ad7a7cf11c50a14bb0efb4f",
      "responsible": {
        "kind": "person",
        "name": "Shuhan He"
      }
    },
    {
      "source": "Dryad public deposits",
      "platform": "Dryad public metadata API",
      "query": "Run each recorded query separately:\n\"diagnostic test accuracy\"\n\"diagnostic accuracy\" AND \"systematic review\"\n\"diagnostic accuracy\" AND \"meta-analysis\"\n\"test accuracy\" AND \"meta-analysis\"\n\"sensitivity and specificity\" AND \"meta-analysis\"\n\"true positive\" AND \"false positive\" AND \"meta-analysis\"\n\"2x2\" AND diagnostic AND \"meta-analysis\"",
      "status": "executed",
      "planned_or_executed_at": "2026-08-17",
      "filters": [],
      "limits": "The count and checksum identify the saved discovery inventory. Eligible evidence consists of publicly retrievable structured diagnostic-review data.",
      "result_count": 11,
      "export_checksum": "e2e5e7913d05a578b36fbaaca5f107bf5d2bb1d0b8905ad2953c43e4e1a5d52f",
      "responsible": {
        "kind": "person",
        "name": "Shuhan He"
      }
    },
    {
      "source": "SRDR+ public deposits",
      "platform": "DataCite SRDR metadata and fixed public SRDR inventory",
      "query": "publisher:\"Systematic Review Data Repository\"; retain the fixed inventory of 172 projects and apply the source-specific discovery and diagnostic-candidate rules in scripts/open_repository_dta.py.",
      "status": "executed",
      "planned_or_executed_at": "2026-08-17",
      "filters": [],
      "limits": "The count and checksum identify the saved discovery inventory. Eligible evidence consists of publicly retrievable structured diagnostic-review data.",
      "result_count": 172,
      "export_checksum": "0576f936db6c27b7cde5147f346289b08c399d7ef90e5f96902a066c669ff88a",
      "responsible": {
        "kind": "person",
        "name": "Shuhan He"
      }
    },
    {
      "source": "PubMed Central open-access review tables",
      "platform": "NCBI E-Utilities and PMC JATS XML",
      "query": "((\"diagnostic test accuracy\"[Title/Abstract]) OR (\"diagnostic accuracy\"[Title/Abstract])) AND (\"2x2\"[All Fields] OR \"2 x 2\"[All Fields] OR (\"true positive\"[All Fields] AND \"false positive\"[All Fields] AND \"false negative\"[All Fields] AND \"true negative\"[All Fields])) AND (systematic review[Title/Abstract] OR meta-analysis[Title/Abstract]) AND open access[filter] AND 2020:2026[Publication Date] NOT protocol[Title]",
      "status": "executed",
      "planned_or_executed_at": "2026-08-17",
      "filters": [
        "open access",
        "2020:2026 publication dates",
        "exclude protocol in title"
      ],
      "limits": "The fixed query returned 1,518 articles on 2026-08-17; all were retrieved. Only explicit identifiers and integer TP, FP, FN, TN columns were normalized. Count and checksum identify articles.csv.",
      "result_count": 1518,
      "export_checksum": "c0dbf4a9a700ef837e76b1f8551a1cb09c237d93a1ce1c8b9465ff731b81f8f3",
      "responsible": {
        "kind": "person",
        "name": "Shuhan He"
      }
    }
  ],
  "date_limits": {
    "from": null,
    "to": null,
    "justification": "No common date window: historical Cochrane is a fixed 2018 release; PMC uses 2020-2026 publication years; other sources use recorded inventories or queries. Source-specific limits govern inclusion."
  },
  "language_limits": {
    "languages": [],
    "justification": "Search terms were English. The recorded API queries contain no language filter; eligibility depended on accessible structured study-level tables. No translation workflow was used in the recorded computational extraction."
  },
  "grey_literature_plan": "Public OSF, Zenodo, Figshare, Dryad, and SRDR deposits were examined for reusable diagnostic-review data. Dryad and SRDR contributed no primary pooled estimates. This accessible evidence sample is not an exhaustive search of unpublished studies or all diagnostic medicine.",
  "deduplication_method": "Retain source identifiers and compare exact normalized count-row signatures. Keep one result per identifier within a group; repeated identifiers make that group ineligible. The same study may contribute to different diagnostic profiles, so study-result appearances are not unique studies.",
  "screening_method": {
    "independent_screeners": 1,
    "conflict_resolution": "Shuhan He directed the computational selection and author audit. Fixed source priorities resolved representations; ambiguous groups were excluded or retained outside the primary sample. Source identities and metadata remained visible. The recorded workflow used one screener; no inter-reviewer agreement statistic was reported.",
    "ai_role": "Codex assisted retrieval, parsing and code drafting. Selection used deterministic rules with author oversight."
  },
  "full_text_method": "Inspect structured review deposits and machine-readable PMC tables for explicit study-level counts. Preserve article, table, file and review identifiers and exclusion reasons. Selection used the single author-directed workflow described above.",
  "data_extraction_method": "Parse explicit study identifiers and integer TP, FP, FN, TN cells while retaining source location, orientation and checksums. Do not reconstruct counts from rounded accuracy metrics. Apply canonical workbook/table rules. Shuhan He directed extraction and rechecked 50 table decisions and 1,640 extracted cells; the recorded audit found no acceptance or transcription errors. This was an author audit within the same workflow, with source identities retained.",
  "risk_of_bias_method": "Interpret the estimates in the context of source-reported risk of bias and clinical setting. The atlas did not undertake a uniform study-level QUADAS-2 assessment; source reporting varies.",
  "synthesis_plan": "Pool each eligible source-defined test or threshold separately with bivariate random-effects REML on sensitivity/specificity logits, adding 0.5 to all four cells in every study. Back-transform the fitted mean logits. Compute entropy, mutual information and percentage uncertainty resolved at the specified starting probabilities. Give each pooled estimate equal reference-distribution weight. Report pooled-mean intervals and plug-in predictive ranges conditional on fitted heterogeneity. Bootstrap reference medians over pooled estimates with 2,000 percentile replicates and a fixed seed; this does not propagate within-estimate uncertainty. Repeat analyses with correction only in zero-containing studies and with the regularized direct-binomial estimator. Treat within-review contrasts and CEA crossings as descriptive; no paired superiority inference or clinical utility ranking is supported. Distinguish signed entropy change, posterior-to-prior KL divergence, log2 likelihood ratio, and expected mutual information.\n\nThe completed analysis contains 444 converged eligible bivariate fits. The primary reference scale retains 273 pooled estimates from 210 reviews and 4,104 study-result appearances. At a 20% starting probability, the median estimate removes 29.2% of starting uncertainty (0.211 of 0.722 bits).",
  "certainty_assessment": "Report source fidelity, numerical diagnostics, estimator sensitivity, model-based uncertainty and sample limitations. This methodological reference scale does not assign GRADE ratings to clinical effects or recommendations.",
  "reporting_guidelines": [
    "PRISMA-P",
    "The BMJ Research Methods and Reporting guidance"
  ],
  "anticipated_start_date": "",
  "anticipated_completion_date": "",
  "milestones": {
    "pilot_search_started_at": "2026-08-15",
    "formal_search_started_at": "2026-08-16",
    "screening_started_at": "2026-08-16",
    "extraction_started_at": "2026-08-16",
    "synthesis_started_at": "2026-08-16"
  },
  "timing_explanation": "The study was registered internally with ConductScience on 16 August 2026. The public protocol record and its DOI were established on 13 September 2026. The protocol has a versioned amendment history: source-priority rules developed during staged audits were fixed before the final reference analysis. This version documents the completed methods and distinguishes the original internal registration from subsequent amendments and public posting.",
  "automation": {
    "agent_name": "OpenAI Codex",
    "agent_version": null,
    "provider": "OpenAI",
    "roles": [
      "Retrieval, parsing and research-code drafting",
      "Language editing and protocol preparation from versioned study records"
    ],
    "human_reviewed_fields": [],
    "source_artifacts": [
      "https://github.com/ShuhanCS/diagnostic-information/blob/03ad6916639926a1d19d33602415323b0cada39d/docs/manuscript/releases/v1.11.0/sources/draft-v1.11.0.md",
      "https://github.com/ShuhanCS/diagnostic-information/blob/03ad6916639926a1d19d33602415323b0cada39d/docs/manuscript/releases/v1.11.0/sources/supplement-v1.11.0.md",
      "https://github.com/ShuhanCS/diagnostic-information/blob/03ad6916639926a1d19d33602415323b0cada39d/docs/manuscript/AUTHORSHIP.json",
      "https://github.com/ShuhanCS/diagnostic-information/blob/03ad6916639926a1d19d33602415323b0cada39d/docs/specs/2026-08-16-full-cochrane-dta-atlas-spec.md",
      "https://github.com/ShuhanCS/diagnostic-information/blob/03ad6916639926a1d19d33602415323b0cada39d/docs/manuscript/releases/v1.11.0/MANIFEST.json",
      "https://github.com/ShuhanCS/diagnostic-information/blob/03ad6916639926a1d19d33602415323b0cada39d/scripts/open_repository_dta.py",
      "https://github.com/ShuhanCS/diagnostic-information/blob/03ad6916639926a1d19d33602415323b0cada39d/scripts/pmc_pilot.py",
      "https://github.com/ShuhanCS/diagnostic-information/blob/03ad6916639926a1d19d33602415323b0cada39d/analysis/cochrane_atlas/corpus_manifest.csv",
      "https://github.com/ShuhanCS/diagnostic-information/blob/03ad6916639926a1d19d33602415323b0cada39d/analysis/open_repository_atlas/osf/source_manifest.csv",
      "https://github.com/ShuhanCS/diagnostic-information/blob/03ad6916639926a1d19d33602415323b0cada39d/analysis/open_repository_atlas/zenodo/source_manifest.csv",
      "https://github.com/ShuhanCS/diagnostic-information/blob/03ad6916639926a1d19d33602415323b0cada39d/analysis/open_repository_atlas/figshare/source_manifest.csv",
      "https://github.com/ShuhanCS/diagnostic-information/blob/03ad6916639926a1d19d33602415323b0cada39d/analysis/open_repository_atlas/dryad/source_manifest.csv",
      "https://github.com/ShuhanCS/diagnostic-information/blob/03ad6916639926a1d19d33602415323b0cada39d/analysis/open_repository_atlas/srdr/source_manifest.csv",
      "https://github.com/ShuhanCS/diagnostic-information/blob/03ad6916639926a1d19d33602415323b0cada39d/analysis/pmc_open_atlas/articles.csv",
      "https://github.com/ShuhanCS/diagnostic-information/blob/8d28cbdbee2229719e69f8be7de27aacb2c90a49/docs/specs/2026-08-16-full-cochrane-dta-atlas-spec.md",
      "https://github.com/ShuhanCS/diagnostic-information/blob/d737c0e010ac5cb45d814cc6e7616f44801607ce/docs/specs/2026-08-15-diagnostic-information-reproducibility-pilot-spec.md",
      "https://github.com/ShuhanCS/diagnostic-information/blob/857cd0f27a164681e01b094cd87591900fd649c8/analysis/cochrane_atlas/summary.json",
      "https://github.com/ShuhanCS/diagnostic-information/blob/03ad6916639926a1d19d33602415323b0cada39d/docs/plans/2026-09-08-data-and-preprint-plan.md",
      "https://github.com/ShuhanCS/diagnostic-information/blob/03ad6916639926a1d19d33602415323b0cada39d/docs/deposits/diagnostic-information-v1.11.0/README.md"
    ],
    "limitations": "Selection used deterministic code and author oversight. Protocol text is CC BY 4.0; third-party source data retain their own terms. The publisher maintains the internal registration of 16 August 2026 and public posting and DOI history of 13 September 2026."
  }
}