From 527d7142e1034d8c2d542f930d2dcea748c3910d Mon Sep 17 00:00:00 2001 From: Kasra Vand Date: Mon, 18 May 2026 17:01:25 -0400 Subject: [PATCH] Prepare HBP 3.0 NAR release metadata --- .zenodo.json | 39 ++++++++++++ ARTIFACT_MANIFEST.tsv | 11 ++++ BUILD_METADATA.json | 31 ++++++++++ CHECKSUMS.txt | 18 ++++++ CITATION.cff | 18 ++++++ DATA_SOURCES.md | 39 ++++++++++++ DATA_SOURCES.tsv | 34 +++++++++++ LICENSES.md | 31 ++++++++++ MANIFEST.md | 42 +++++++++++++ PROVENANCE_SCHEMA.md | 73 +++++++++++++++++++++++ README.md | 89 ++++++++++++++++++++++++++++ RELEASE_NOTES.md | 28 +++++++++ docs/schemas/drug_discovery.md | 18 ++++++ docs/schemas/gene_profile.md | 16 +++++ docs/schemas/guideline_signal.md | 20 +++++++ docs/schemas/population_frequency.md | 18 ++++++ docs/schemas/protein_context.md | 18 ++++++ docs/schemas/structural_variant.md | 18 ++++++ scripts/generate_checksums.sh | 37 ++++++++++++ 19 files changed, 598 insertions(+) create mode 100644 .zenodo.json create mode 100644 ARTIFACT_MANIFEST.tsv create mode 100644 BUILD_METADATA.json create mode 100644 CHECKSUMS.txt create mode 100644 CITATION.cff create mode 100644 DATA_SOURCES.md create mode 100644 DATA_SOURCES.tsv create mode 100644 LICENSES.md create mode 100644 MANIFEST.md create mode 100644 PROVENANCE_SCHEMA.md create mode 100644 RELEASE_NOTES.md create mode 100644 docs/schemas/drug_discovery.md create mode 100644 docs/schemas/gene_profile.md create mode 100644 docs/schemas/guideline_signal.md create mode 100644 docs/schemas/population_frequency.md create mode 100644 docs/schemas/protein_context.md create mode 100644 docs/schemas/structural_variant.md create mode 100755 scripts/generate_checksums.sh diff --git a/.zenodo.json b/.zenodo.json new file mode 100644 index 0000000..ceab7cb --- /dev/null +++ b/.zenodo.json @@ -0,0 +1,39 @@ +{ + "title": "HeartBioPortal DataHub: HBP 3.0 NAR release", + "creators": [ + { + "name": "HeartBioPortal contributors" + }, + { + "name": "TBD: confirm author list before release" + } + ], + "description": "Data integration, provenance, artifact publishing, and serving-datamart tooling for the HeartBioPortal 3.0 NAR Database Issue manuscript release.", + "access_right": "open", + "upload_type": "software", + "license": "cc-by-4.0", + "keywords": [ + "HeartBioPortal", + "cardiovascular genomics", + "data integration", + "provenance", + "NAR Database Issue" + ], + "related_identifiers": [ + { + "identifier": "https://github.com/HeartBioPortal/DataHub", + "relation": "isSupplementTo", + "scheme": "url" + }, + { + "identifier": "https://heartbioportal.org/", + "relation": "isSupplementTo", + "scheme": "url" + }, + { + "identifier": "https://github.com/HeartBioPortal", + "relation": "isPartOf", + "scheme": "url" + } + ] +} diff --git a/ARTIFACT_MANIFEST.tsv b/ARTIFACT_MANIFEST.tsv new file mode 100644 index 0000000..204634c --- /dev/null +++ b/ARTIFACT_MANIFEST.tsv @@ -0,0 +1,11 @@ +artifact_name hbp_layer artifact_path_or_pattern artifact_type key_fields record_count gene_count variant_count source_count build_version schema_version compressed redistributable license_notes description +population_frequency_datamart Population-frequency context datamart/population_frequency*; datamart/association_serving_slim.duckdb tables/duckdb rsid; allele; population_label; population_group; sample_size TBD; verify from production QA TBD TBD; expected about 18.1M rsIDs if production QA confirms 37 if production QA confirms v3.0.0-nar TBD mixed requires review requires source-license review Approximate 594.3M source-specific observations was not locally verified; confirm before release. +association_artifacts Association and phenotype evidence analyzed_data_unified/association/final/**; association/final/** json/json.gz/duckdb gene; variant_id; phenotype_path; p_value; ancestry; consequence; clinical_significance TBD; verify from production QA TBD TBD TBD v3.0.0-nar TBD mixed requires review requires source-license review Gene-level association artifacts, variant-index payloads, phenotype rollups, and serving summaries. +structural_variant_gene_payloads Structural-variant evidence analyzed_data/dbvar/dbvar_structural_variants_nstd229.json.zip json zip gene; sv_id; sv_type; coordinates; clinical_significance 3072942 75192 3040582 1 v3.0.0-nar structural_variant_legacy yes requires review requires TOPMed/dbVar source-term review Local dbVar nstd229 report verifies records, variants, and genes. +exon_enriched_sv_artifacts Structural-variant evidence analyzed_data/dbvar/dbvar_structural_variants_nstd229.exons.json.zip json zip gene; sv_id; transcript_overlap; exon_overlap TBD; verify from production QA TBD TBD 1 v3.0.0-nar TBD yes requires review requires TOPMed/dbVar source-term review Exon-enriched structural-variant payloads derived from dbVar nstd229 and annotation overlap. +protein_context_artifacts Protein context secondary_analyses/final/protein_context/*.json.gz json.gz gene; transcript_id; translation_id; protein_accession; feature_id TBD; verify from production QA 5304 local untracked payload files observed; production expected value requires QA TBD Ensembl/UniProt/EBI/InterPro sources v3.0.0-nar TBD yes requires review requires source attribution and terms review Production target mentions about 66.9k isoforms and more than 3.2M feature annotations; not locally verified. +gene_profile_artifacts Gene profiles secondary_analyses/final/gene_profile/*.json.gz; generated gene-profile paths json/json.gz gene; source_id; xref TBD; verify from production QA TBD TBD HGNC/NCBI/UniProt/GOA/Reactome/etc. v3.0.0-nar TBD mixed requires review requires source-license review Gene summary, nomenclature, ontology, pathway, and cross-reference payloads. +expression_payloads Expression layer secondary_analyses/final/expression/*.json.gz; configured expression outputs json/json.gz gene; tissue; cell_type; expression_value TBD; verify from production QA TBD TBD TBD v3.0.0-nar TBD mixed requires review requires source-license review Expression payloads imported or transformed for HBP gene dossiers. +shared_architecture_summaries Shared-architecture layer secondary_analyses/final/sga/*.json.gz; configured SGA outputs json/json.gz gene; cvd_phenotype; trait_phenotype; shared_variant_count TBD; verify from production QA TBD TBD inherits association source count v3.0.0-nar TBD mixed requires review inherits association source restrictions Derived overlap summaries between CVD and trait association variants. +drug_discovery_payloads Drug-discovery / Drugs & Compounds layer drug-discovery output paths; per-gene drug payloads json/json.gz gene; molecule_id; molecule_name; target_id; source TBD; reported 17128 records requires production QA TBD; reported 1839 gene files requires production QA TBD Open Targets; DrugBank v3.0.0-nar TBD mixed requires review DrugBank raw data license-restricted; Open Targets terms require confirmation Reported 17,128 records, 1,839 gene files, and 1,454 unique molecules were not locally verified. +guideline_derived_payloads Clinical guidelines / guideline graph links HCG/HCG-KG derived import paths; hcgkg_llm artifacts json/graph/vector gene; guideline_id; recommendation_id; evidence_class; evidence_level; snippet_id TBD; verify from HCG/HCG-KG release QA TBD TBD AHA/ACC guideline sources; future ESC v3.0.0-nar TBD mixed requires review guideline source licenses require review Gene-first guideline context consumed by HBP search_summary and guideline_detail when present. diff --git a/BUILD_METADATA.json b/BUILD_METADATA.json new file mode 100644 index 0000000..ed176a3 --- /dev/null +++ b/BUILD_METADATA.json @@ -0,0 +1,31 @@ +{ + "project": "HeartBioPortal", + "component": "DataHub", + "release": "3.0.0-nar", + "created_at": "2026-05-18T00:00:00-04:00", + "git_commit": "784decace6458cd0791e8149e09eb76552a31bee", + "datahub_schema_version": "TBD; confirm before release", + "genome_assembly": "GRCh38 if applicable; confirm per artifact", + "major_layers": [ + "association and phenotype evidence", + "population-frequency context", + "variant annotation", + "structural-variant evidence", + "protein context", + "gene profiles", + "clinical guideline graph links", + "drug-discovery / drugs and compounds", + "expression", + "shared genetic architecture" + ], + "source_manifest": "DATA_SOURCES.tsv", + "artifact_manifest": "ARTIFACT_MANIFEST.tsv", + "restricted_data_policy": "Do not redistribute controlled individual-level human data, credentials, restricted raw source data, raw DrugBank full database files, or license-uncertain third-party source files.", + "controlled_individual_level_data": "not redistributed", + "notes": [ + "Update git_commit after final release commit is selected.", + "Counts marked TBD in ARTIFACT_MANIFEST.tsv require production QA confirmation.", + "Local dbVar nstd229 report verifies 3072942 records, 3040582 variants, and 75192 gene-level payloads.", + "DrugBank v5.1.12 raw data are license-restricted and should not be included unless redistribution permission is confirmed." + ] +} diff --git a/CHECKSUMS.txt b/CHECKSUMS.txt new file mode 100644 index 0000000..190ab6f --- /dev/null +++ b/CHECKSUMS.txt @@ -0,0 +1,18 @@ +ecff620da4e234325c5616ee5e8efd732e074748b23705c99a5cf6c06ac24451 .zenodo.json +f2df420efc5b1962e21fc9f431714605873150300bdbf03b5f59ab300cff57b7 ARTIFACT_MANIFEST.tsv +76544cb7d297551bfadcb8816d433bb8b8855ac917a2f144bd1e5a3c480a9da7 BUILD_METADATA.json +a4435542f0fb7ca1b8dbd1c14861a2c4628b637483a31c72bb52de6bd9647bc8 CITATION.cff +7509ecb44ad4137b71ec083a454f50cb4987bed4f04f3d20c3af2abb8139f83b DATA_SOURCES.md +04982f407a637edfc0736a1b74fc59bf991430872f858d9a6ef9e3ad62c76a9d DATA_SOURCES.tsv +fbf48fe05e2e55283171b4c7a90b6388cb3e82d44eb7eb5c70c75e0a2973714a LICENSE +4f23d36d7930cf3c6702de7135ecd7ef781235809b50c608f56c654c0ec8aba5 LICENSES.md +8f2e1fa3f4d582335b1486d465da986836f26d45ea334333692e11bf1f8279ea MANIFEST.md +5b14a73d64211e75c7ea7cf988741a96727babc44e094041d261d91893dcfcd5 PROVENANCE_SCHEMA.md +1dcdf166eb05e0a3c1214fd06904628ea4cb9b673304b929db1bf6103d5518c0 README.md +ba24cd85ffe6811b29928c46268ee28106b340c828e1453530b87255224c8e86 RELEASE_NOTES.md +96962af6c01ef76ffcc75697b328d0513e1ef75cc97a97b0ab5c4585e72248e6 docs/schemas/drug_discovery.md +2984ea49a550a3d3b91e51a0346a5316ba1e3520eb884a55ed877925a09317c1 docs/schemas/gene_profile.md +96b9ce0a18ab071c26c1270727b13a4cf45d8d4833842ea322c60e40cbf0b694 docs/schemas/guideline_signal.md +1e90d2145fbc4049d2f8354f9c30eae04ccd2e41ab923348541ec1506aa032cf docs/schemas/population_frequency.md +febfe8f9b31139c063b8ac4a5953a1e646da363359697e707fc3c7122d9feba7 docs/schemas/protein_context.md +7fb87293e8b1446941a9f5dcf3bc9d42c7d6fed242751fd7a86dd771efea7c73 docs/schemas/structural_variant.md diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..8d68f70 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,18 @@ +cff-version: 1.2.0 +message: "If you use this software or data-processing workflow, please cite the HeartBioPortal 3.0 manuscript and this archived release." +title: "HeartBioPortal DataHub" +version: "3.0.0-nar" +date-released: "2026-05-18" +license: "CC-BY-4.0" +repository-code: "https://github.com/HeartBioPortal/DataHub" +url: "https://heartbioportal.org/" +authors: + - name: "HeartBioPortal contributors" + - name: "TBD: confirm author list before release" +abstract: "Data integration, provenance, artifact publishing, and serving-datamart tooling for the HeartBioPortal 3.0 NAR Database Issue manuscript release." +keywords: + - HeartBioPortal + - cardiovascular genomics + - data integration + - provenance + - NAR Database Issue diff --git a/DATA_SOURCES.md b/DATA_SOURCES.md new file mode 100644 index 0000000..b0257f5 --- /dev/null +++ b/DATA_SOURCES.md @@ -0,0 +1,39 @@ +# Data Sources Summary + +`DATA_SOURCES.tsv` is the machine-readable source inventory for the HBP 3.0 NAR release. This companion file summarizes the source families by HBP layer. + +## Association and phenotype evidence + +DataHub normalizes association rows from HBP legacy CVD/trait layers, Million Veteran Program summary-statistics inputs when available, GWAS Catalog when included, and other association profiles into canonical gene, variant, phenotype, p-value, ancestry, consequence, clinical-significance, and provenance fields. The final artifacts are association JSON/JSON.GZ payloads, variant-index payloads, phenotype rollups, and serving DuckDB tables. Source provenance should preserve input file, source dataset, source version, phenotype path, rsID/variant ID, p-value, genome build, and transformation notes. Controlled or non-public inputs must not be redistributed. + +## Population-frequency context + +Population-frequency context is expected to combine source-specific allele-frequency observations from resources such as ALFA, gnomAD v4, 1000 Genomes, 1000 Genomes 30X, TOPMed-derived public frequency resources, PAGE, HGDP-CEPH, HapMap, ExAC, SGDP, and 38KJPN where present in the production build. DataHub harmonizes rsID, allele, population label, population group, sample size, study/resource, genome build, and source provenance into population-frequency datamarts. Production totals must be verified from QA before release; the approximate 594.3 million frequency observations across 18.1 million rsIDs and 37 resources are not hard-coded in `ARTIFACT_MANIFEST.tsv` because they were not locally verifiable. + +## Variant annotation + +Variant annotation uses dbSNP, ClinVar, Ensembl Variation, ClinGen where included, and source-specific legacy fields. DataHub preserves rsID/variant IDs as variant-level keys for chart aggregation and records clinical significance, variation type, most-severe-consequence-like fields, source record IDs, source dataset, genome build, and transformation steps. Source licenses follow the original providers. + +## Structural-variant evidence + +Structural-variant evidence is currently represented by dbVar nstd102/ClinVar structural-variant seed payloads and dbVar nstd229/TOPMed structural-variant call-set artifacts. DataHub normalizes source DB, study/submission, SV ID, SV type, coordinates, event length, clinical significance when present, gene overlap, transcript overlap, and exon overlap. The local nstd229 report verifies 3,072,942 records, 3,040,582 variants, and 75,192 gene-level payloads. TOPMed-related licensing and redistribution constraints require final review before public archival of source or derived bulk artifacts. + +## Protein context + +Protein context connects variant associations to protein architecture through Ensembl gene/transcript/translation IDs, canonical and protein-coding isoforms, exon-to-protein coordinate mapping, RefSeq/UniProt cross-references, Ensembl features, EBI Proteins features, and InterPro domains/families/motifs/regions. DataHub harmonizes source-specific API records into gene-level protein-context payloads. Production totals around 66.9k isoforms and more than 3.2 million protein feature annotations require production QA confirmation before release. + +## Gene profiles + +Gene profiles integrate nomenclature, gene summaries, protein cross-references, ontology/pathway membership, and curated source metadata from sources such as HGNC, NCBI Gene, UniProtKB, GOA/Gene Ontology, Reactome, Human Protein Atlas, and ClinGen where included. DataHub should preserve source IDs, source versions, access dates, cross-reference IDs, and source-specific licensing notes. + +## Clinical guidelines / guideline graph links + +Clinical guideline artifacts are generated primarily by HCG and HCG-KG. DataHub consumes release JSON, graph exports, or vector/serving artifacts when present and links genes to guideline snippets, recommendations, evidence classes, evidence levels, conditions, biomarkers, drugs/interventions, and source documents. Guideline snippets are context for interpretation and are not automated medical advice. + +## Drug-discovery / Drugs & Compounds layer + +The drugs and compounds layer uses Open Targets Platform GraphQL API v4 and licensed DrugBank v5.1.12 inputs where available. DataHub should preserve the GraphQL query, variables, access date, source field names, molecule source, molecule ID, target ID, source action type, indication, trial phase/status, source version, and source license. The raw DrugBank full database is license-restricted and must not be committed or archived unless redistribution permission is confirmed. The reported drug-layer total of 17,128 gene-drug records across 1,839 gene files and 1,454 unique molecule names requires production QA confirmation. + +## Expression and shared-architecture layers + +Expression payloads are imported from source-specific expression resources and existing HBP payloads where present. Shared genetic architecture is derived from association artifacts by comparing gene-level CVD and trait variant overlap. Both layers inherit redistribution constraints from their source data and must preserve source dataset, source version, input file, transformation, and HBP build version. diff --git a/DATA_SOURCES.tsv b/DATA_SOURCES.tsv new file mode 100644 index 0000000..d88b969 --- /dev/null +++ b/DATA_SOURCES.tsv @@ -0,0 +1,34 @@ +source_name source_category source_url_or_endpoint access_method source_version_or_release accessed_or_downloaded_date license_or_terms redistribution_status hbp_layer input_files_or_api normalized_output final_artifact provenance_fields_preserved notes +HGNC gene nomenclature https://www.genenames.org/ download/API TBD TBD HGNC data reuse terms redistributable with attribution/terms Gene profiles config/sources/hgnc.json gene symbol/cross-reference profile gene-profile artifacts source_name; source_version; source_url_or_endpoint; accessed_at; input_record_id Confirm release snapshot before archive. +NCBI Gene gene annotation https://www.ncbi.nlm.nih.gov/gene download/API TBD TBD NCBI public domain and usage guidelines public-domain/public-source terms Gene profiles config/sources/ncbi_gene.json gene summaries and cross-references gene-profile artifacts source_name; source_url_or_endpoint; accessed_at; input_record_id Confirm exact FTP/API endpoint in final build metadata. +UniProtKB protein annotation https://www.uniprot.org/ REST/download TBD TBD Creative Commons Attribution 4.0 International redistributable with attribution Protein context config/sources/uniprotkb.json protein names, accessions, features, cross-references protein-context artifacts source_name; source_version; source_url_or_endpoint; accessed_at; input_record_id Preserve UniProt accession and reviewed status. +GOA / Gene Ontology gene ontology annotation https://geneontology.org/ download TBD TBD Gene Ontology Consortium data license redistributable with attribution Gene profiles config/sources/goa.json GO annotations and evidence codes gene-profile artifacts source_name; source_version; input_record_id; evidence code Include GO term and evidence provenance where used. +Ensembl protein/transcript/variant annotation https://rest.ensembl.org/ REST API TBD TBD Ensembl Terms of Use redistributable subject to Ensembl terms Protein context; Variant annotation config/sources/ensembl_protein_context.json; config/sources/ensembl_variation.json transcript, translation, exon, coordinate, consequence context protein-context artifacts; variant annotations source_name; source_url_or_endpoint; accessed_at; genome_build; input_record_id Genome assembly should be recorded as GRCh38 when applicable. +EBI Proteins protein feature annotation https://www.ebi.ac.uk/proteins/api/doc/ REST API TBD TBD UniProt/EMBL-EBI terms of use redistributable subject to source terms Protein context config/sources/ebi_proteins.json protein sequences and feature annotations protein-context artifacts source_name; source_url_or_endpoint; accessed_at; input_record_id Preserve feature source and protein accession. +InterPro protein domains/families https://www.ebi.ac.uk/interpro/ REST/download TBD TBD EMBL-EBI terms of use redistributable subject to source terms Protein context config/sources/interpro.json domain/family/motif/region annotations protein-context artifacts source_name; source_url_or_endpoint; accessed_at; input_record_id Preserve InterPro accession and member database when available. +Reactome pathway annotation https://reactome.org/ download/API TBD TBD Reactome open license redistributable with attribution Gene profiles config/sources/reactome.json pathway memberships gene-profile artifacts source_name; source_version; input_record_id Present in source configs; verify whether included in current HBP 3.0 production build. +Human Protein Atlas expression/protein context https://www.proteinatlas.org/ download TBD TBD Human Protein Atlas terms of use redistribution requires source-term review Gene profiles; Expression config/sources/human_protein_atlas.json tissue/cell expression summaries expression payloads; gene-profile artifacts source_name; source_version; accessed_at; input_file Present in configs; verify current build use and license constraints. +ClinGen clinical curation https://clinicalgenome.org/ download/API TBD TBD ClinGen terms and attribution policy redistribution requires source-term review Variant annotation; Gene profiles config/sources/clingen.json gene/variant validity and actionability context gene-profile or annotation artifacts source_name; source_version; accessed_at; input_record_id Verify current production inclusion. +NCBI dbSNP variant identifier/annotation https://www.ncbi.nlm.nih.gov/snp/ download/API TBD TBD NCBI Disclaimer and Copyright Notice public-source terms Variant annotation; Association context config/sources/dbsnp.json rsID and short variant annotations association artifacts; variant-index artifacts source_name; source_version; rsid; input_record_id Legacy association layers preserve rsID/variant IDs as deduplication keys. +NCBI ALFA population frequency https://www.ncbi.nlm.nih.gov/snp/docs/gsr/alfa/ download/API TBD TBD NCBI Disclaimer and Copyright Notice public-source terms Population-frequency context config/sources/alfa.json allele-frequency observations population-frequency datamart source_name; source_version; rsid; allele; population_label; sample_size Production total must be verified from QA before release. +gnomAD v4 population frequency https://gnomad.broadinstitute.org/ download/API v4 if used TBD gnomAD terms of use redistribution subject to gnomAD terms Population-frequency context config/sources/gnomad.json allele-frequency observations population-frequency datamart source_name; source_version; rsid; allele; population_label; sample_size Confirm exact v4 release and redistribution terms. +1000 Genomes population frequency https://www.internationalgenome.org/ download TBD TBD 1000 Genomes data use policy redistributable subject to source terms Population-frequency context config/sources/thousand_genomes.json allele-frequency observations population-frequency datamart source_name; source_version; rsid; allele; population_label; sample_size Includes source family; split 30X row when exact pipeline inputs are confirmed. +1000 Genomes 30X population frequency https://www.internationalgenome.org/data-portal/data-collection/30x-grch38 download TBD TBD 1000 Genomes data use policy redistributable subject to source terms Population-frequency context TBD allele-frequency observations population-frequency datamart source_name; source_version; rsid; allele; population_label; sample_size Requested HBP 3.0 source family; verify exact input path. +TOPMed-derived public frequency resources population frequency https://bravo.sph.umich.edu/ download/API TBD TBD TOPMed / NHLBI data access policy redistribution requires source-term review Population-frequency context config/sources/topmed_bravo.json allele-frequency observations population-frequency datamart source_name; source_version; rsid; allele; population_label; sample_size Public frequency summaries only; no controlled participant-level data. +PAGE population frequency TBD download/API TBD TBD TBD requires confirmation Population-frequency context TBD allele-frequency observations population-frequency datamart source_name; source_version; rsid; allele; population_label; sample_size Verify whether present in current production source bundle. +HGDP-CEPH population frequency TBD download TBD TBD TBD requires confirmation Population-frequency context TBD allele-frequency observations population-frequency datamart source_name; source_version; rsid; allele; population_label; sample_size Verify exact source and license. +HapMap population frequency https://www.genome.gov/10001688/international-hapmap-project download TBD TBD TBD requires confirmation Population-frequency context TBD allele-frequency observations population-frequency datamart source_name; source_version; rsid; allele; population_label; sample_size Verify current production use. +ExAC population frequency https://gnomad.broadinstitute.org/downloads#exac-variants download TBD TBD ExAC/gnomAD terms of use redistribution subject to source terms Population-frequency context TBD allele-frequency observations population-frequency datamart source_name; source_version; rsid; allele; population_label; sample_size Verify current production use and exact release. +SGDP population frequency https://www.simonsfoundation.org/simons-genome-diversity-project/ download TBD TBD TBD requires confirmation Population-frequency context TBD allele-frequency observations population-frequency datamart source_name; source_version; rsid; allele; population_label; sample_size Verify current production use and redistribution rights. +38KJPN population frequency https://jmorp.megabank.tohoku.ac.jp/ download/API TBD TBD ToMMo/JMorp terms redistribution requires source-term review Population-frequency context TBD allele-frequency observations population-frequency datamart source_name; source_version; rsid; allele; population_label; sample_size Verify exact source and license. +dbVar nstd102 / ClinVar structural variants structural variant evidence https://www.ncbi.nlm.nih.gov/dbvar/ download nstd102 TBD NCBI Disclaimer and Copyright Notice public-source terms Structural-variant evidence analyzed_data/dbvar/dbvar_structural_variants_nstd102_seed.json.zip gene-level SV seed records structural-variant gene payloads source_db; study_or_submission; sv_id; coordinates; clinical_significance; gene_overlap Seed artifact is checked in as compressed JSON zip. +dbVar nstd229 / TOPMed structural-variant call set structural variant evidence https://www.ncbi.nlm.nih.gov/dbvar/studies/nstd229/ download nstd229 TBD NCBI Disclaimer and Copyright Notice; TOPMed/NHLBI policy may apply redistribution requires source-term review Structural-variant evidence raw_data/dbvar/dbvar_structural_variants_nstd229.csv.zip; analyzed_data/dbvar/dbvar_structural_variants_nstd229.report.json gene-level SV payloads; exon-enriched payloads analyzed_data/dbvar/dbvar_structural_variants_nstd229.json.zip; analyzed_data/dbvar/dbvar_structural_variants_nstd229.exons.json.zip source_db; study_or_submission; sv_id; sv_type; coordinates; event_length; gene_overlap; exon_overlap Local report verifies 3,072,942 records and 75,192 genes for nstd229 ingestion. +ClinVar variant clinical assertion https://www.ncbi.nlm.nih.gov/clinvar/ download/API TBD TBD NCBI Disclaimer and Copyright Notice public-source terms Variant annotation config/sources/clinvar.json clinical significance annotations association artifacts; variant annotations source_name; source_version; input_record_id; clinical_significance Legacy association fields preserve ClinVar clinical significance where present. +GWAS Catalog association and phenotype evidence https://www.ebi.ac.uk/gwas/ download/API TBD TBD EMBL-EBI Terms of Use redistributable subject to source terms Association and phenotype evidence config/sources/gwas_catalog.json variant-trait associations association artifacts; serving DuckDB source_name; source_version; rsid; phenotype; p_value; input_record_id Verify whether included in the current production build. +Million Veteran Program summary statistics association and phenotype evidence TBD restricted/controlled source TBD TBD MVP data-use terms controlled/non-redistributable raw source data Association and phenotype evidence config/export_manifests/association/overrides/million_veteran_program.json prepared summary-statistics rows association artifacts; serving DuckDB source_name; source_version; phenotype; p_value; rsid; cohort_or_study Do not redistribute controlled or non-public raw MVP files. +Open Targets Platform drug-discovery/target evidence https://api.platform.opentargets.org/api/v4/graphql GraphQL API v4 TBD TBD Open Targets Platform data terms / CC0 if confirmed redistribution requires terms confirmation Drug-discovery / Drugs & Compounds layer config/sources/open_targets.json; GraphQL query and variables per run gene-drug/target-disease evidence drug-discovery payloads source_name; source_version; source_url_or_endpoint; accessed_at; source field names Store GraphQL query, variables, access date, and source field names where available. +DrugBank drug-discovery/compound knowledgebase https://go.drugbank.com/releases/5-1-12/downloads/all-full-database local licensed dataset 5.1.12 TBD CC BY-NC 4.0 / academic non-commercial terms, requires confirmation from source page license-restricted; do not redistribute raw data unless permitted Drug-discovery / Drugs & Compounds layer licensed DrugBank full database; derived per-gene records gene-drug records drug-discovery payloads molecule_source; molecule_id; target_id; source_action_type; source_indication; source_license HBP-derived fields should preserve source/version/license metadata; do not commit raw full database. +HBP legacy dbSNP / legacy association layers legacy association evidence TBD local legacy files TBD TBD TBD requires confirmation Association and phenotype evidence config/prep_profiles/legacy_cvd_raw.json; config/prep_profiles/legacy_trait_raw.json canonical association rows association artifacts; variant-index artifacts source_name; source_version; rsid; phenotype; p_value; input_file Legacy layers remain for compatibility; source licenses must be traced before archive. +Expression sources expression evidence TBD import existing payloads TBD TBD source-specific terms requires source-by-source confirmation Expression layer config/profiles/expression.json; config/secondary_analyses/expression.json normalized expression payloads expression payloads source_name; source_version; input_file; transformation Verify active source set and redistribution rights before release. +Shared genetic architecture layer derived analysis DataHub association artifacts derived from HBP association data v3.0.0-nar TBD inherits source restrictions from association inputs derived outputs require source-license review Shared-architecture layer config/secondary_analyses/sga.json gene-level shared-variant summaries shared-architecture summaries source datasets; phenotype paths; variant IDs; transformation Derived layer should preserve association-source provenance. +Clinical guideline graph links guideline context HCG/HCG-KG repositories file import / graph export v3.0.0-nar TBD guideline source licenses and HCG/HCG-KG licenses guideline snippets require source-license review Clinical guidelines / guideline graph links HCG release JSON; HCG-KG graph/vector artifacts gene-guideline links and snippets guideline-derived payloads guideline_id; source_document; excerpt/snippet; evidence_class; evidence_level DataHub consumes guideline-derived artifacts when present; HCG/HCG-KG own extraction and KG construction. diff --git a/LICENSES.md b/LICENSES.md new file mode 100644 index 0000000..481d3aa --- /dev/null +++ b/LICENSES.md @@ -0,0 +1,31 @@ +# License and Redistribution Summary + +This file summarizes licensing surfaces for the HBP 3.0 NAR release. It is not legal advice; final release approval should confirm source-specific terms. + +## 1. HBP software license + +DataHub's repository `LICENSE` states that repository metadata and documentation are licensed under Creative Commons Attribution 4.0 International (CC BY 4.0), while raw data may retain the licenses specified by original sources. Confirm whether code is intended to share the same license statement or needs a separate software license before public release. + +## 2. HBP-derived redistributable outputs + +HBP-derived metadata, schemas, manifests, documentation, and small examples may be redistributed under the repository license when they do not include restricted third-party data. Derived data artifacts inherit source-license constraints and should not be archived publicly until each source family has been reviewed. + +## 3. Third-party open/public-domain or permissive sources + +Examples likely include NCBI resources such as dbSNP, ClinVar, dbVar, NCBI Gene, and ALFA, subject to NCBI disclaimer and usage guidelines. These still require attribution/provenance and should preserve source version, access date, and source URL. + +## 4. Third-party attribution-required sources + +Sources such as UniProtKB, Gene Ontology, Reactome, EMBL-EBI resources, Ensembl, InterPro, GWAS Catalog, and related open scientific resources generally require source attribution and may carry specific terms of use. Preserve source names, URLs, versions, access dates, and license labels in DataHub artifacts. + +## 5. Third-party non-commercial or license-restricted sources + +DrugBank v5.1.12 is documented for this release as a licensed academic source. Treat it as CC BY-NC 4.0 / non-commercial academic-use terms only after final confirmation from the source page. Raw DrugBank full database files must not be committed or archived unless redistribution permission is explicit. Derived HBP drug records should preserve source/version/license metadata. + +Open Targets Platform GraphQL-derived records should document the API endpoint, GraphQL query, variables, access date, source field names, and source terms. The expected endpoint is `https://api.platform.opentargets.org/api/v4/graphql`; confirm the exact license/terms for the release. + +TOPMed-derived public frequency resources and dbVar nstd229/TOPMed structural-variant call-set derivatives require final source-term review before bulk redistribution. + +## 6. Controlled-access or non-redistributable source data + +Controlled individual-level human data are not redistributed through HBP. Restricted cohort data, controlled-access genotype/phenotype data, non-public MVP raw/source files, credentials, API keys, and protected data must not be committed or included in GitHub or Zenodo releases. diff --git a/MANIFEST.md b/MANIFEST.md new file mode 100644 index 0000000..2b687ea --- /dev/null +++ b/MANIFEST.md @@ -0,0 +1,42 @@ +# Repository Manifest + +This manifest describes release-relevant repository areas for the HBP 3.0 NAR manuscript archive. + +## Code + +- `src/datahub/`: reusable Python package for source manifests, adapters, validation, publication, secondary analyses, and serving helpers. +- `scripts/`: command-line entrypoints for ingestion, association publication, structural-variant ingestion, secondary analyses, QA reporting, and checksum generation. +- `tests/`: focused tests for config validation, manifests, association publication, serving builders, and secondary-analysis helpers. + +## Configuration + +- `config/sources/`: source-family metadata used by DataHub ingestion and provenance docs. +- `config/profiles/`: source profile definitions for ingestion layers. +- `config/prep_profiles/`: raw-to-prepared profiles for legacy association inputs. +- `config/export_manifests/`: rules for preserving, promoting, deriving, and publishing association fields. +- `config/output_contracts/`: output contracts for serving and legacy artifacts. +- `config/schemas/`: JSON Schemas for DataHub configuration surfaces. + +## Documentation and release metadata + +- `README.md`: project overview, setup, HBP 3.0 role, and release checklist. +- `docs/`: full DataHub documentation site. +- `DATA_SOURCES.tsv` and `DATA_SOURCES.md`: source inventory and human-readable source summary. +- `ARTIFACT_MANIFEST.tsv`: HBP 3.0 artifact-family inventory. +- `BUILD_METADATA.json`: machine-readable release metadata template. +- `LICENSES.md`: software, output, third-party, and restricted-data license summary. +- `PROVENANCE_SCHEMA.md`: provenance fields expected in HBP 3.0 DataHub outputs. +- `docs/schemas/`: documentation-only schemas for release-facing HBP layers. +- `CITATION.cff` and `.zenodo.json`: citation and archive metadata. +- `RELEASE_NOTES.md`: HBP 3.0 manuscript release notes. + +## Data and generated artifacts + +- `raw_data/`: small source files or checked-in source seeds only. Do not add controlled, restricted, or license-uncertain bulk source data. +- `analyzed_data/`: curated or generated artifacts that are small enough and permitted for repository distribution. +- `secondary_analyses/`: generated secondary-analysis outputs may be very large and are not assumed to be release-ready unless explicitly documented and redistributable. +- `datamart/`: serving DuckDB outputs are production artifacts and should be archived only when size and redistribution rights are confirmed. + +## Generated versus committed + +Committed release-support files are metadata, docs, schemas, configs, small seed data, and scripts. Large generated artifacts, production DuckDB files, controlled individual-level data, raw DrugBank full data, and license-uncertain third-party data should be generated or staged outside the repository unless release approval is explicit. diff --git a/PROVENANCE_SCHEMA.md b/PROVENANCE_SCHEMA.md new file mode 100644 index 0000000..a161738 --- /dev/null +++ b/PROVENANCE_SCHEMA.md @@ -0,0 +1,73 @@ +# DataHub Provenance Schema + +DataHub artifacts should preserve enough provenance to trace an HBP field back to the source family, source version, access method, input record, and transformation. + +## Standard fields + +| Field | Description | +| --- | --- | +| `source_name` | Human-readable source name. | +| `source_version` | Source release, database version, or API version. | +| `source_url_or_endpoint` | Source URL, FTP path, API endpoint, or DOI. | +| `access_method` | Download, REST API, GraphQL API, licensed local file, or derived DataHub artifact. | +| `accessed_at` | API access or web access date/time. | +| `downloaded_at` | File download date/time. | +| `license` | Source license or terms label. | +| `input_file` | Raw or prepared input path when file-based. | +| `input_record_id` | Stable input record identifier or row key. | +| `source_record_id` | Source database identifier. | +| `source_dataset` | Dataset, study, cohort, or source subcollection. | +| `cohort_or_study` | Cohort, study, submission, or trial label. | +| `population_label` | Original population label. | +| `population_group` | Harmonized population group when available. | +| `sample_size` | Sample count, allele count, or source-provided denominator. | +| `genome_build` | Genome assembly, such as GRCh38. | +| `transformation` | DataHub transformation label. | +| `normalization_steps` | Summary of normalization applied. | +| `hbp_artifact` | Generated HBP artifact path or family. | +| `hbp_build_version` | HBP release/build version, for example `3.0.0-nar`. | + +## Drug records + +Drug-discovery payloads should additionally preserve: + +- `molecule_source` +- `molecule_id` +- `target_id` +- `source_action_type` +- `source_indication` +- `source_trial_phase_or_status` +- `source_license` + +For Open Targets records, also preserve the GraphQL endpoint, query, variables, access date, and source field names where available. For DrugBank-derived records, preserve the DrugBank release version and non-commercial/license-restricted status. + +## Structural variants + +Structural-variant payloads should additionally preserve: + +- `source_db` +- `study_or_submission` +- `sv_id` +- `sv_type` +- `coordinates` +- `event_length` +- `clinical_significance` +- `gene_overlap` +- `transcript_overlap` +- `exon_overlap` +- `zygosity_if_available` + +## Population frequencies + +Population-frequency rows should preserve: + +- `rsid` +- `allele` +- `frequency` +- `population_label` +- `population_group` +- `sample_size` +- `study` or `resource` +- `BioProject` or `BioSample` if available +- `build` +- `provenance` diff --git a/README.md b/README.md index c756963..8d59ae7 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,95 @@ Config JSON files are validated by JSON Schemas in `config/schemas/`. DataHub still supports legacy HeartBioPortal-compatible analyzed payloads, but the codebase now also maintains a newer serving-artifact path based on DuckDB. The legacy path exists for compatibility; the unified DuckDB-first path is the strategic direction. +## How this repository supports HBP 3.0 + +DataHub is the canonical HBP 3.0 data-owner repository. It prepares source manifests, normalizes source-specific fields, publishes association and secondary-analysis artifacts, and builds serving datamarts consumed by the HeartBioPortal backend and frontend. + +Related HBP 3.0 repositories: + +- HeartBioPortal organization: https://github.com/HeartBioPortal +- Live site: https://heartbioportal.org/ +- HCG guideline extraction resource: https://github.com/HeartBioPortal/HCG +- HCG-KG guideline knowledge graph resource: https://github.com/HeartBioPortal/HCG-KG + +## Manuscript release + +This repository supports the HeartBioPortal 3.0 NAR Database Issue manuscript release (`v3.0.0-nar`). The release-support files in this repository describe source provenance, licensing constraints, generated artifacts, reproducibility expectations, and files that should or should not be included in a public archive. + +Release metadata and manifests: + +- `CITATION.cff` +- `.zenodo.json` +- `RELEASE_NOTES.md` +- `MANIFEST.md` +- `DATA_SOURCES.tsv` +- `DATA_SOURCES.md` +- `ARTIFACT_MANIFEST.tsv` +- `BUILD_METADATA.json` +- `LICENSES.md` +- `PROVENANCE_SCHEMA.md` +- `docs/schemas/*.md` +- `scripts/generate_checksums.sh` + +## Preparing the HBP 3.0 NAR release + +Use this checklist before creating a GitHub release or Zenodo archive: + +1. Confirm the release branch and commit: + +```bash +git status --short --branch +git rev-parse HEAD +``` + +2. Validate the code and docs in the target environment: + +```bash +python -m pytest +mkdocs build --strict +``` + +3. Regenerate or review manifest files: + +- Review `DATA_SOURCES.tsv` and `DATA_SOURCES.md` against the current source configs and pipeline inputs. +- Review `ARTIFACT_MANIFEST.tsv` against production QA reports, generated artifact directories, and serving DB tables. +- Update `BUILD_METADATA.json` with the final release commit, build date, schema version, and verified production metrics. + +4. Verify counts from production artifacts where available: + +```bash +datahub-report-artifact-qa --help +``` + +Counts that cannot be verified from committed local artifacts should remain `TBD; verify from production QA`. + +5. Generate release checksums for release-relevant static files: + +```bash +scripts/generate_checksums.sh +``` + +6. Include in Zenodo: + +- repository source code +- config schemas and manifests +- documentation +- small examples or seed metadata that are redistributable +- generated release manifests and checksum files + +7. Do not include in Zenodo unless redistribution has been confirmed: + +- controlled individual-level human data +- API keys, credentials, tokens, or secrets +- raw DrugBank full database files +- large source datasets with unclear redistribution rights +- controlled-access or license-restricted third-party source files +- massive generated artifacts unless they are intended, permitted, and documented for the release package + +## Security and privacy + +No controlled individual-level human data should be committed to this repository. Do not commit API keys, credentials, protected data, tokens, or restricted source data. Source-specific licensing controls redistribution of third-party data; if redistribution rights are uncertain, document the source in `DATA_SOURCES.tsv` or `LICENSES.md` rather than committing the data. + ## License See `LICENSE`. diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md new file mode 100644 index 0000000..4fcb126 --- /dev/null +++ b/RELEASE_NOTES.md @@ -0,0 +1,28 @@ +# Release Notes + +## v3.0.0-nar + +This release supports the HeartBioPortal 3.0 NAR Database Issue manuscript-associated archive. + +Included release-support material: + +- source manifest and human-readable source summary +- generated-artifact manifest with redistributability notes +- build metadata template for final release stamping +- provenance field schema for DataHub outputs +- documentation-only schemas for HBP 3.0 serving layers +- citation metadata and Zenodo metadata +- checksum-generation script for release-relevant static files + +Major HBP 3.0-relevant areas documented in this release: + +- association and phenotype evidence +- population-frequency context +- variant and structural-variant annotation +- protein context +- gene profiles +- expression and shared-architecture layers +- drug-discovery payloads +- clinical-guideline graph links consumed from HCG/HCG-KG outputs + +Third-party data remain subject to the original source licenses and terms. Restricted, controlled-access, or license-uncertain raw source files should not be redistributed through this release unless permissions are confirmed. diff --git a/docs/schemas/drug_discovery.md b/docs/schemas/drug_discovery.md new file mode 100644 index 0000000..91856d4 --- /dev/null +++ b/docs/schemas/drug_discovery.md @@ -0,0 +1,18 @@ +# Drug Discovery Schema + +Documentation-only schema for HBP 3.0 drug-discovery payloads. + +| Field | Description | +| --- | --- | +| `gene` | Target gene symbol. | +| `molecule_name` | Drug or molecule name. | +| `molecule_id` | Source molecule identifier. | +| `molecule_source` | Open Targets, DrugBank, or other source. | +| `target_id` | Source target identifier. | +| `source_action_type` | Mechanism/action type when available. | +| `source_indication` | Disease or indication context. | +| `source_trial_phase_or_status` | Trial phase or approval/status field. | +| `source_license` | Source license or terms. | +| `provenance` | Source and transformation provenance object. | + +DrugBank-derived records must preserve DrugBank version and license status and must not expose raw full-database content unless redistribution is permitted. diff --git a/docs/schemas/gene_profile.md b/docs/schemas/gene_profile.md new file mode 100644 index 0000000..91d8b24 --- /dev/null +++ b/docs/schemas/gene_profile.md @@ -0,0 +1,16 @@ +# Gene Profile Schema + +Documentation-only schema for HBP 3.0 gene-profile artifacts. + +| Field | Description | +| --- | --- | +| `gene` | HGNC-approved or HBP-normalized gene symbol. | +| `gene_id` | Source or HBP gene identifier. | +| `aliases` | Known aliases or previous symbols. | +| `summary` | Source-grounded gene summary. | +| `cross_references` | NCBI, Ensembl, UniProt, HGNC, and other IDs. | +| `functions` | GO, pathway, or curated functional annotations. | +| `sources` | Source provenance objects. | +| `hbp_build_version` | HBP build version. | + +Required provenance: `source_name`, `source_version`, `source_url_or_endpoint`, `accessed_at`, `input_record_id`, `license`. diff --git a/docs/schemas/guideline_signal.md b/docs/schemas/guideline_signal.md new file mode 100644 index 0000000..be9abc5 --- /dev/null +++ b/docs/schemas/guideline_signal.md @@ -0,0 +1,20 @@ +# Guideline Signal Schema + +Documentation-only schema for HBP 3.0 guideline-derived signals consumed from HCG or HCG-KG. + +| Field | Description | +| --- | --- | +| `gene` | Gene symbol linked to guideline context. | +| `guideline_id` | Guideline source identifier. | +| `source_document` | Source document title/path/URL. | +| `recommendation_id` | Recommendation identifier when available. | +| `excerpt_or_snippet` | Source-grounded snippet when redistribution is allowed. | +| `evidence_class` | Class of recommendation. | +| `evidence_level` | Level of evidence. | +| `condition` | Disease or condition context. | +| `biomarker` | Biomarker context. | +| `drug_or_intervention` | Drug or intervention context. | +| `relationship_type` | Graph or extraction relationship label. | +| `provenance` | Source, extraction, and licensing provenance. | + +Guideline-derived signals provide context only and are not automated medical advice or direct clinical actionability. diff --git a/docs/schemas/population_frequency.md b/docs/schemas/population_frequency.md new file mode 100644 index 0000000..742825d --- /dev/null +++ b/docs/schemas/population_frequency.md @@ -0,0 +1,18 @@ +# Population Frequency Schema + +Documentation-only schema for HBP 3.0 population-frequency artifacts. + +| Field | Description | +| --- | --- | +| `rsid` | dbSNP rsID when available. | +| `variant_id` | HBP or source variant identifier. | +| `allele` | Allele being counted. | +| `frequency` | Allele frequency. | +| `population_label` | Source population label. | +| `population_group` | Harmonized population group. | +| `sample_size` | Source-provided denominator or sample size. | +| `source_name` | Frequency source or resource. | +| `genome_build` | Genome assembly. | +| `provenance` | Source and transformation provenance object. | + +Rows should preserve source-specific labels and avoid collapsing distinct cohorts without a provenance trail. diff --git a/docs/schemas/protein_context.md b/docs/schemas/protein_context.md new file mode 100644 index 0000000..4d1833e --- /dev/null +++ b/docs/schemas/protein_context.md @@ -0,0 +1,18 @@ +# Protein Context Schema + +Documentation-only schema for HBP 3.0 protein-context artifacts. + +| Field | Description | +| --- | --- | +| `gene` | Gene symbol. | +| `ensembl_gene_id` | Ensembl gene identifier. | +| `transcripts` | Transcript and translation records. | +| `canonical` | Whether the transcript/isoform is canonical when known. | +| `protein_length` | Amino-acid length. | +| `translation_exons` | Exon-to-protein coordinate mapping. | +| `refseq_xrefs` | RefSeq cross-references. | +| `uniprot_xrefs` | UniProt cross-references. | +| `features` | Ensembl, EBI Proteins, UniProt, and InterPro protein features. | +| `provenance` | Source and transformation provenance object. | + +Protein features should include feature type, source, start/end amino-acid coordinates, IDs, labels, and source license where available. diff --git a/docs/schemas/structural_variant.md b/docs/schemas/structural_variant.md new file mode 100644 index 0000000..947f18c --- /dev/null +++ b/docs/schemas/structural_variant.md @@ -0,0 +1,18 @@ +# Structural Variant Schema + +Documentation-only schema for HBP 3.0 structural-variant artifacts. + +| Field | Description | +| --- | --- | +| `gene` | Gene overlapped by the structural variant. | +| `sv_id` | Source structural-variant identifier. | +| `sv_type` | Deletion, duplication, insertion, inversion, CNV, or source-provided type. | +| `coordinates` | Genomic coordinates and assembly. | +| `event_length` | Event length when available. | +| `clinical_significance` | Source clinical-significance assertion. | +| `gene_overlap` | Gene overlap annotation. | +| `transcript_overlap` | Transcript overlap annotation when available. | +| `exon_overlap` | Exon overlap annotation when available. | +| `source_db` | Source database, such as dbVar. | +| `study_or_submission` | Source study or submission ID. | +| `provenance` | Source and transformation provenance object. | diff --git a/scripts/generate_checksums.sh b/scripts/generate_checksums.sh new file mode 100755 index 0000000..e5f4e29 --- /dev/null +++ b/scripts/generate_checksums.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +output="CHECKSUMS.txt" +tmp="${output}.tmp" + +files=( + "README.md" + "LICENSE" + "CITATION.cff" + ".zenodo.json" + "RELEASE_NOTES.md" + "MANIFEST.md" + "DATA_SOURCES.tsv" + "DATA_SOURCES.md" + "ARTIFACT_MANIFEST.tsv" + "BUILD_METADATA.json" + "LICENSES.md" + "PROVENANCE_SCHEMA.md" +) + +find docs/schemas -type f -name '*.md' | sort >> "${tmp}.files" +for file in "${files[@]}"; do + if [[ -f "$file" ]]; then + printf '%s\n' "$file" >> "${tmp}.files" + fi +done + +sort -u "${tmp}.files" | while IFS= read -r file; do + shasum -a 256 "$file" +done > "$tmp" + +rm -f "${tmp}.files" +mv "$tmp" "$output" +echo "Wrote $output"