implementing forge trace

CHANGELOG.md

@@ -1,3 +1,5 @@
+- V 2.2.2.0:
+    - Forged packages now contain a new column `SourcePackage`, which records which original package (and version) a sample originally came from.
 - V 2.2.1.0:
     - Made the display of description and source url in the explorer page of the of the server's html API indendent of each other.
     - Made the source column hide when the necessary URL is missing on the archive page.

poseidon-hs.cabal

@@ -13,7 +13,7 @@ extra-source-files:  README.md,
                      CHANGELOG.md
 
 library
-    exposed-modules:    
+    exposed-modules:
                         -- core
                         Poseidon.Core.Package, Poseidon.Core.GenotypeData, Poseidon.Core.BibFile,
                         Poseidon.Core.Janno, Poseidon.Core.ColumnTypesJanno, Poseidon.Core.ColumnTypesSSF,
@@ -54,7 +54,7 @@ executable trident
                         filepath, co-log
     other-modules:      Paths_poseidon_hs
     default-language:   Haskell2010
-    
+
 executable poseidon-server
     main-is:            Main-server.hs
     hs-source-dirs:     src-executables
@@ -76,7 +76,7 @@ Test-Suite poseidon-tools-tests
                         filepath, pipes, pipes-safe, pipes-ordered-zip,
                         unordered-containers, cassava, containers, process, country,
                         optparse-applicative, parsec
-    other-modules:      
+    other-modules:
                         -- golden
                         PoseidonGoldenTests.GoldenTestsValidateChecksumsSpec,
                         PoseidonGoldenTests.GoldenTestsRunCommands,

src/Poseidon/CLI/Trident/Forge.hs

@@ -4,17 +4,21 @@ module Poseidon.CLI.Trident.Forge where
 
 import           Poseidon.Core.BibFile          (BibEntry (..), BibTeX,
                                                  writeBibTeXFile)
-import           Poseidon.Core.ColumnTypesJanno (PoseidonID (..))
-import           Poseidon.Core.ColumnTypesUtils (ListColumn (..),
+import           Poseidon.Core.ColumnTypesJanno (JannoNrSNPs (..),
+                                                 PoseidonID (..))
+import           Poseidon.Core.ColumnTypesUtils (CsvNamedRecord (..),
+                                                 ListColumn (..), getCsvNR,
                                                  getMaybeListColumn)
 import           Poseidon.Core.EntityTypes      (EntityInput,
+                                                 IndividualInfo (..),
                                                  PacNameAndVersion (..),
                                                  PoseidonEntity (..),
                                                  SignedEntity (..),
                                                  checkIfAllEntitiesExist,
                                                  isLatestInCollection,
                                                  makePacNameAndVersion,
                                                  readEntityInputs,
+                                                 renderNameWithVersion,
                                                  resolveUniqueEntityIndices)
 import           Poseidon.Core.GenotypeData     (GenoDataSource (..),
                                                  GenotypeDataSpec (..),
@@ -53,6 +57,8 @@ import           Control.Exception              (catch, throwIO)
 import           Control.Monad                  (filterM, forM, forM_, unless,
                                                  when)
 import qualified Data.ByteString                as B
+import qualified Data.ByteString.Char8          as BC
+import qualified Data.HashMap.Strict            as HM
 import           Data.List                      (intercalate, nub)
 import           Data.Maybe                     (catMaybes, mapMaybe)
 import           Data.Time                      (getCurrentTime)
@@ -62,7 +68,6 @@ import qualified Data.Vector.Unboxed.Mutable    as VUM
 import           Pipes                          (MonadIO (liftIO), cat, (>->))
 import qualified Pipes.Prelude                  as P
 import           Pipes.Safe                     (SafeT, runSafeT)
-import           Poseidon.Core.ColumnTypesJanno (JannoNrSNPs (..))
 import           SequenceFormats.Eigenstrat     (EigenstratSnpEntry (..),
                                                  GenoEntry (..), GenoLine,
                                                  writeEigenstrat)
@@ -170,7 +175,16 @@ runForge (
     -- collect data --
     -- janno
     let (JannoRows jannoRows) = getJointJanno relevantPackages
-        newJanno@(JannoRows relevantJannoRows) = JannoRows $ map (jannoRows !!) relevantIndices
+        newJanno@(JannoRows relevantJannoRows) = JannoRows $ do --list monad
+            i <- relevantIndices
+            let sourcePac = indInfoPac $ fst indInfoCollection !! i
+            let jannoRow = jannoRows !! i
+            let addColsHM = getCsvNR . jAdditionalColumns $ jannoRow
+            let newTraceEntry = case HM.lookup "SourcePackage" addColsHM of
+                    Just ft -> ft <> ";" <> BC.pack (renderNameWithVersion sourcePac)
+                    Nothing -> BC.pack $ renderNameWithVersion sourcePac
+            let addColsHMwithTrace = HM.insert "SourcePackage" newTraceEntry addColsHM
+            return $ jannoRow {jAdditionalColumns = CsvNamedRecord addColsHMwithTrace}
 
     -- seqSource
     let seqSourceRows = mconcat $ map posPacSeqSource relevantPackages

src/Poseidon/Core/ColumnTypesUtils.hs

@@ -186,4 +186,3 @@ renderCsvParseError (CsvParseError expected actual leftover) =
     "expected data type: " ++ expected ++ ", " ++
     "broken value: " ++ actual ++ ", " ++
     "problematic characters: " ++ show leftover ++ ")"
-

test/PoseidonGoldenTests/GoldenTestCheckSumFile.txt

@@ -69,12 +69,12 @@ ebf0b456d5e09686ee7131db06848104 rectify init/Schiffels/POSEIDON.yml
 083fe7ef4206c979356a3a2454d780b1 rectify init/Schiffels/Schiffels.janno
 5b5c20071f3e53346fdfa04f9dbb0223 forge forge/ForgePac1/POSEIDON.yml
 1286a2580e4bfbed7d804d5f3fe125f7 forge forge/ForgePac1/ForgePac1.geno
-848cf0fab32e4078a88cc17ad8ec3381 forge forge/ForgePac1/ForgePac1.janno
+29301e7d9cc37b2a5f889c244b691d17 forge forge/ForgePac1/ForgePac1.janno
 a1daaa1ebc9c93405acb4a0f24ffa355 forge forge/ForgePac1/ForgePac1.ssf
 b4f71aff4fbc11594008c3811781cc43 forge forge/ForgePac1/ForgePac1.bib
 d860abe4b07926c1114a52cee70e8b0d forge forge/ForgePac1_vcf/POSEIDON.yml
 9e0c33410f399caf99446f01a7ed7809 forge forge/ForgePac1_vcf/ForgePac1_vcf.geno
-848cf0fab32e4078a88cc17ad8ec3381 forge forge/ForgePac1_vcf/ForgePac1_vcf.janno
+29301e7d9cc37b2a5f889c244b691d17 forge forge/ForgePac1_vcf/ForgePac1_vcf.janno
 a1daaa1ebc9c93405acb4a0f24ffa355 forge forge/ForgePac1_vcf/ForgePac1_vcf.ssf
 b4f71aff4fbc11594008c3811781cc43 forge forge/ForgePac1_vcf/ForgePac1_vcf.bib
 cbd8d1e4e4d6f6b5e65cc1f8fa33cccb forge forge/ForgePac2/POSEIDON.yml
@@ -83,44 +83,44 @@ cbd8d1e4e4d6f6b5e65cc1f8fa33cccb forge forge/ForgePac2/POSEIDON.yml
 7bd06bf634e6c29815a74532f696d753 forge forge/ForgePac3/ForgePac3.geno
 d817ecc64d504e1351d85996903f09b2 forge forge/ForgePac3/ForgePac3.snp
 ad8add34e464ee620bb392c00636e7f2 forge forge/ForgePac3/ForgePac3.ind
-3163a6c626e7c4a55162a33601cd344a forge forge/ForgePac3/ForgePac3.janno
+ce3d441d5a4928e8c9a38caa76389fde forge forge/ForgePac3/ForgePac3.janno
 a534f60c502a792f5c8404a9fc3054b9 forge forge/ForgePac3/ForgePac3.ssf
 1f8edbb1cf8ad7306fb3c16ce692b560 forge forge/ForgePac4/POSEIDON.yml
 f348944d864cbaa595f138b11d154248 forge forge/ForgePac4/ForgePac4.bim
 217cbb31258a3a7c9522d8dc9bda2386 forge forge/ForgePac4/ForgePac4.bed
 3001829ad782e27375a6132583c16f05 forge forge/ForgePac4/ForgePac4.fam
-6bdd8baf96d8f1437f1b35bbec9654a1 forge forge/ForgePac4/ForgePac4.janno
+dedbb0e02eb0182346c29ee4bb51f8e6 forge forge/ForgePac4/ForgePac4.janno
 103c6305eaa79811cff5b51aa401c465 forge forge/ForgePac4/ForgePac4.ssf
 970ee8383d391056973c0dffd460fd27 forge forge/ForgePac5/POSEIDON.yml
 5a3783db28cf9da6356b3c6105564be5 forge forge/ForgePac5/ForgePac5.geno
-fe8ab20f55cf750f1249b9598f40df0a forge forge/ForgePac5/ForgePac5.janno
+668da542fc043b2ee7d2647d0972562c forge forge/ForgePac5/ForgePac5.janno
 1327f2ba8c436fd1050813a082f6af44 forge forge/ForgePac5/ForgePac5.ssf
 1286a2580e4bfbed7d804d5f3fe125f7 forge forge/ForgePac6/ForgePac6.geno
 6e3ecc6695234ba1e71045278634d06b forge forge/ForgePac6/ForgePac6.snp
 2bd17be137df5cb11830b09b833b937d forge forge/ForgePac6/ForgePac6.ind
 1286a2580e4bfbed7d804d5f3fe125f7 forge forge/ForgePac7/ForgePac7.geno
 6e3ecc6695234ba1e71045278634d06b forge forge/ForgePac7/ForgePac7.snp
 2bd17be137df5cb11830b09b833b937d forge forge/ForgePac7/ForgePac7.ind
-d480376e41da91f056779969ba2ca6c8 forge forge/ForgePac7/ForgePac7.janno
-5d3ed7862824c9b3860412ec895af182 forge forge/ForgePac8/ForgePac8.janno
+645bfe63c413f95f1c4604a8368ca3c9 forge forge/ForgePac7/ForgePac7.janno
+f297ff0b018c1ceb44661601fdde9a9d forge forge/ForgePac8/ForgePac8.janno
 24882e62b96340abc4e74b354d3b67e0 forge forge/ForgePac8/ForgePac8.ssf
 84bac408ab864c357bb2a6e1db5cda20 forge forge/ForgePac9/ForgePac9.geno
-c900c31ba87ded89b5d57eff2c543993 forge forge/ForgePac9/ForgePac9.janno
+073169cdbb2093980fd88fbbcc3cf8dd forge forge/ForgePac9/ForgePac9.janno
 487220b6fff6349c1d274945b26ecd8e forge forge/ForgePac9/ForgePac9.ssf
 bcccc5136aade390f711fee3695ef62b forge forge/ForgePac10/ForgePac10.geno
-318fb50802161edca4c63d882a1e0731 forge forge/ForgePac10/ForgePac10.janno
+08aead8dddc3e4a91ec035dadd908a28 forge forge/ForgePac10/ForgePac10.janno
 a62c05bb53e72309fc5062a3ea701c5a forge forge/ForgePac10/ForgePac10.ssf
 bdef2c307caa92ee7a3e946d0264171a forge forge/ForgePac10/ForgePac10.bib
 0332344057c0c4dce2ff7176f8e1103d forge forge/ForgePac11/ForgePac11.geno
-d134f76c49bb7385e0493244d2900a4e forge forge/ForgePac11/ForgePac11.janno
+cd580b0b9be93017415d31f88f693e10 forge forge/ForgePac11/ForgePac11.janno
 a62c05bb53e72309fc5062a3ea701c5a forge forge/ForgePac11/ForgePac11.ssf
 0bb988d9e1c981a47f332bea8a16e984 forge forge/ForgePac12/ForgePac12.ind
-35a717d522094e1feba7468590608b2f forge forge/ForgePac13/ForgePac13.janno
-7ec893f4dde2f136d66b5e3f2ee6aca9 forge forge/ForgePac14/ForgePac14.janno
-b7b2f1ad925f7e1e66e2a8040d5985fb forge forge/ForgePac15/ForgePac15.janno
-b7b2f1ad925f7e1e66e2a8040d5985fb forge forge/ForgePac16/ForgePac16.janno
-b17d37da626be3a61212447692899a19 forge forge/ForgePac17/ForgePac17.janno
-d6f09712aed72a3662cddedc4b9d9c11 forge forge/ForgePac18/ForgePac18.janno
+b312f55762d710d6357b8d4b6326f4bb forge forge/ForgePac13/ForgePac13.janno
+37815bf48c845e27f924a45df5cebed9 forge forge/ForgePac14/ForgePac14.janno
+c06f2fd3672ec2f997c0bb0591d678a1 forge forge/ForgePac15/ForgePac15.janno
+c06f2fd3672ec2f997c0bb0591d678a1 forge forge/ForgePac16/ForgePac16.janno
+3c037bb39bbb01a5e46861bccea772a7 forge forge/ForgePac17/ForgePac17.janno
+e7dd731e1e73debe7fd6f4ca99185793 forge forge/ForgePac18/ForgePac18.janno
 3fc77f5a5b83ac4fc7b082c9b3ee4ba1 forge forge/ForgePac18/ForgePac18.fam
 842885ffe256819b264991384020c2ac forge forge/ForgePac18/ForgePac18.bed
 6644be1a1c8967753b4903ce22228dbf forge forge/ForgePac19/POSEIDON.yml
@@ -133,10 +133,10 @@ ad7e56177aad0a720f0bde13d47f2ac1 forge forge/ForgePac19/CHANGELOG.md
 9b22cab26cdae87bd79d24c289e48433 forge forge/ForgePac19/ForgePac19.fam
 504775c8994e25615b7662fc0495ff12 forge forge/ForgePac19/ForgePac19.janno
 3288b6ff2d7b02eec0f34a8434e4b7e3 forge forge/ForgePac20/POSEIDON.yml
-589e62622e1ee6f0ad6268935dd540ba forge forge/ForgePac20/ForgePac20.janno
+2957d88d1fc24e4de8100c5ee4cb5806 forge forge/ForgePac20/ForgePac20.janno
 ef1646043331ed9d67e4072ee46eb9f4 forge forge/ForgePac21/POSEIDON.yml
 8b8e1852b1e3cdd0ab0176dddce13bba forge forge/ForgePac21/ForgePac21.vcf
-848cf0fab32e4078a88cc17ad8ec3381 forge forge/ForgePac21/ForgePac21.janno
+29301e7d9cc37b2a5f889c244b691d17 forge forge/ForgePac21/ForgePac21.janno
 a1daaa1ebc9c93405acb4a0f24ffa355 forge forge/ForgePac21/ForgePac21.ssf
 b4f71aff4fbc11594008c3811781cc43 forge forge/ForgePac21/ForgePac21.bib
 d4a05cfef045648238a94a9d621cf667 chronicle chronicle/chronicle1.yml

test/PoseidonGoldenTests/GoldenTestData/forge/ForgePac1/ForgePac1.janno

@@ -1,7 +1,7 @@
-Poseidon_ID	Genetic_Sex	Group_Name	Individual_ID	Species	Alternative_IDs	Alternative_IDs_Context	Relation_To	Relation_Degree	Relation_Type	Collection_ID	Custodian_Institution	Cultural_Era	Cultural_Era_URL	Archaeological_Culture	Archaeological_Culture_URL	Country	Country_ISO	Location	Site	Latitude	Longitude	Date_Type	Date_C14_Labnr	Date_C14_Uncal_BP	Date_C14_Uncal_BP_Err	Date_BC_AD_Start	Date_BC_AD_Median	Date_BC_AD_Stop	Chromosomal_Anomalies	MT_Haplogroup	Y_Haplogroup	Source_Material	Nr_Libraries	Library_Names	Capture_Type	UDG	Library_Built	Genotype_Ploidy	Data_Preparation_Pipeline_URL	Endogenous	Nr_SNPs	Coverage_on_Target_SNPs	Damage	Contamination	Contamination_Err	Contamination_Meas	Genetic_Source_Accession_IDs	Primary_Contact	Publication	Note	Keywords	AddCol1	AddCol2
-XXX002	F	POP2	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	9	n/a	n/a	n/a	n/a	n/a	n/a	n/a	Schiffels2016	n/a	n/a	v1	v2
-XXX004	F	POP2	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	9	n/a	n/a	n/a	n/a	n/a	n/a	n/a	Schiffels2016	n/a	n/a	v1	v2
-XXX005	M	POP2	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	7	n/a	n/a	n/a	n/a	n/a	n/a	n/a	Schiffels2016;TestPaper1	n/a	n/a	v1	v2
-XXX006	F	POP2	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	9	n/a	n/a	n/a	n/a	n/a	n/a	n/a	Schiffels2016;TestPaper1	n/a	n/a	v1	v2
-SAMPLE2	F	3	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	7	n/a	n/a	n/a	n/a	n/a	n/a	n/a	Wang2020;TestPaper1	n/a	n/a	n/a	n/a
-SAMPLE4	F	5	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	7	n/a	n/a	n/a	n/a	n/a	n/a	n/a	Wang2020;TestPaper1;TestBook2	n/a	n/a	n/a	n/a
+Poseidon_ID	Genetic_Sex	Group_Name	Individual_ID	Species	Alternative_IDs	Alternative_IDs_Context	Relation_To	Relation_Degree	Relation_Type	Collection_ID	Custodian_Institution	Cultural_Era	Cultural_Era_URL	Archaeological_Culture	Archaeological_Culture_URL	Country	Country_ISO	Location	Site	Latitude	Longitude	Date_Type	Date_C14_Labnr	Date_C14_Uncal_BP	Date_C14_Uncal_BP_Err	Date_BC_AD_Start	Date_BC_AD_Median	Date_BC_AD_Stop	Chromosomal_Anomalies	MT_Haplogroup	Y_Haplogroup	Source_Material	Nr_Libraries	Library_Names	Capture_Type	UDG	Library_Built	Genotype_Ploidy	Data_Preparation_Pipeline_URL	Endogenous	Nr_SNPs	Coverage_on_Target_SNPs	Damage	Contamination	Contamination_Err	Contamination_Meas	Genetic_Source_Accession_IDs	Primary_Contact	Publication	Note	Keywords	AddCol1	AddCol2	SourcePackage
+XXX002	F	POP2	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	9	n/a	n/a	n/a	n/a	n/a	n/a	n/a	Schiffels2016	n/a	n/a	v1	v2	Schiffels_2016-1.0.1
+XXX004	F	POP2	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	9	n/a	n/a	n/a	n/a	n/a	n/a	n/a	Schiffels2016	n/a	n/a	v1	v2	Schiffels_2016-1.0.1
+XXX005	M	POP2	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	7	n/a	n/a	n/a	n/a	n/a	n/a	n/a	Schiffels2016;TestPaper1	n/a	n/a	v1	v2	Schiffels_2016-1.0.1
+XXX006	F	POP2	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	9	n/a	n/a	n/a	n/a	n/a	n/a	n/a	Schiffels2016;TestPaper1	n/a	n/a	v1	v2	Schiffels_2016-1.0.1
+SAMPLE2	F	3	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	7	n/a	n/a	n/a	n/a	n/a	n/a	n/a	Wang2020;TestPaper1	n/a	n/a	n/a	n/a	Wang_2020-0.1.0
+SAMPLE4	F	5	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	n/a	7	n/a	n/a	n/a	n/a	n/a	n/a	n/a	Wang2020;TestPaper1;TestBook2	n/a	n/a	n/a	n/a	Wang_2020-0.1.0