added VCF test files and updated metric output names

Author: ecrum19

.gitignore

@@ -4,8 +4,10 @@ scripts/node_modules
 !time-*
 
 run_output/
-tsv_vcf_files/
-test_vcf_files/
+test_vcf_files/*
+!test_vcf_files/test-100.vcf
+!test_vcf_files/test-1k.vcf
+!test_vcf_files/test-10k.vcf
 RMLStreamer-v2.5.0-standalone.jar
 out/
 tsv/

README.md

@@ -89,8 +89,25 @@ Outputs:
     - `./out/decompressed/*.nq`
 - `./run_metrics` for logs and metrics
   - `run_metrics/metrics.csv` includes both conversion and compression metrics per run
+  - conversion step artifacts:
+    - `run_metrics/conversion-time-<output_name>-<run_id>.txt`
+    - `run_metrics/conversion-metrics-<output_name>-<run_id>.json`
+  - compression step artifacts:
+    - `run_metrics/compression-time-<method>-<output_name>-<run_id>.txt`
+    - `run_metrics/compression-metrics-<output_name>-<run_id>.json`
   - `run_metrics/.wrapper_logs/wrapper-<timestamp>.log` stores detailed Docker/stdout/stderr command output
 
+Small VCF fixtures for RDF size/inflation test runs:
+- `test_vcf_files/test-100.vcf` (100 total lines)
+- `test_vcf_files/test-1k.vcf` (1000 total lines)
+- `test_vcf_files/test-10k.vcf` (10000 total lines)
+
+Example inflation check:
+```bash
+python3 vcf_rdfizer.py --mode full --input test_vcf_files/infl1k.vcf --compression none --keep-tsv
+wc -l out/infl1k/infl1k.nq
+```
+
 ## How Dependencies Are Handled
 
 The Docker image bundles:

src/compression.sh

@@ -204,12 +204,15 @@ for OUT in "${OUTPUT_DIRS[@]}"; do
   fi
 
   BASENAME=$(basename "$OUT")
-  RUN_KEY="${RUN_ID}-${BASENAME}"
+  SAFE_BASENAME=$(printf "%s" "$BASENAME" | tr -cs 'A-Za-z0-9._-' '_')
+  if [[ -z "$SAFE_BASENAME" ]]; then
+    SAFE_BASENAME="rdf"
+  fi
 
-  TIME_LOG_GZIP="$LOGDIR/time-gzip-$RUN_KEY.txt"
-  TIME_LOG_BROTLI="$LOGDIR/time-brotli-$RUN_KEY.txt"
-  TIME_LOG_HDT="$LOGDIR/time-hdt-$RUN_KEY.txt"
-  METRICS_JSON="$LOGDIR/compression-$RUN_KEY.json"
+  TIME_LOG_GZIP="$LOGDIR/compression-time-gzip-${SAFE_BASENAME}-${RUN_ID}.txt"
+  TIME_LOG_BROTLI="$LOGDIR/compression-time-brotli-${SAFE_BASENAME}-${RUN_ID}.txt"
+  TIME_LOG_HDT="$LOGDIR/compression-time-hdt-${SAFE_BASENAME}-${RUN_ID}.txt"
+  METRICS_JSON="$LOGDIR/compression-metrics-${SAFE_BASENAME}-${RUN_ID}.json"
 
   OUT_SIZE=$(stat_size "$OUT")
   TRIPLES_JSON=$(count_triples_json "$OUT")

src/run_conversion.sh

@@ -15,8 +15,12 @@ mkdir -p "$LOGDIR" "$OUT_DIR"
 
 RUN_ID=${RUN_ID:-$(date +%Y%m%dT%H%M%S)}
 TIMESTAMP=${TIMESTAMP:-$(date +"%Y-%m-%dT%H:%M:%S")}
-TIME_LOG="$LOGDIR/time-$RUN_ID.txt"
-METRICS_JSON="$LOGDIR/metrics-$RUN_ID.json"
+SAFE_OUT_NAME=$(printf "%s" "$OUT_NAME" | tr -cs 'A-Za-z0-9._-' '_')
+if [[ -z "$SAFE_OUT_NAME" ]]; then
+  SAFE_OUT_NAME="rdf"
+fi
+TIME_LOG="$LOGDIR/conversion-time-${SAFE_OUT_NAME}-${RUN_ID}.txt"
+METRICS_JSON="$LOGDIR/conversion-metrics-${SAFE_OUT_NAME}-${RUN_ID}.json"
 METRICS_CSV="$LOGDIR/metrics.csv"
 METRICS_HEADER="run_id,timestamp,output_name,output_dir,exit_code_java,wall_seconds_java,user_seconds_java,sys_seconds_java,max_rss_kb_java,input_mapping_size_bytes,input_vcf_size_bytes,output_dir_size_bytes,output_triples,jar,mapping_file,output_path,combined_nq_size_bytes,gzip_size_bytes,brotli_size_bytes,hdt_size_bytes,exit_code_gzip,exit_code_brotli,exit_code_hdt,wall_seconds_gzip,user_seconds_gzip,sys_seconds_gzip,max_rss_kb_gzip,wall_seconds_brotli,user_seconds_brotli,sys_seconds_brotli,max_rss_kb_brotli,wall_seconds_hdt,user_seconds_hdt,sys_seconds_hdt,max_rss_kb_hdt,compression_methods"
 

test/test_compression_unit.py

@@ -154,6 +154,10 @@ def test_compression_updates_existing_metrics_row_with_mocked_tools(self):
             self.assertTrue((out_root / "gzip" / "rdf.nq.gz").exists())
             self.assertTrue((out_root / "brotli" / "rdf.nq.br").exists())
             self.assertTrue((out_root / "hdt" / "rdf.hdt").exists())
+            self.assertTrue((logdir / "compression-time-gzip-rdf-run-compress-1.txt").exists())
+            self.assertTrue((logdir / "compression-time-brotli-rdf-run-compress-1.txt").exists())
+            self.assertTrue((logdir / "compression-time-hdt-rdf-run-compress-1.txt").exists())
+            self.assertTrue((logdir / "compression-metrics-rdf-run-compress-1.json").exists())
 
             row = read_metrics_row(metrics_csv, run_id, "rdf")
             self.assertEqual(row["run_id"], run_id)

test/test_run_conversion_unit.py

@@ -67,6 +67,8 @@ def test_run_conversion_writes_nq_and_metrics_without_real_java(self):
             merged_nq = out_dir / "rdf" / "rdf.nq"
             self.assertTrue(merged_nq.exists())
             self.assertIn("<s> <p> <o> .", merged_nq.read_text())
+            self.assertTrue((metrics_dir / "conversion-time-rdf-run123.txt").exists())
+            self.assertTrue((metrics_dir / "conversion-metrics-rdf-run123.json").exists())
 
             metrics_csv = metrics_dir / "metrics.csv"
             self.assertTrue(metrics_csv.exists())

test_vcf_files/test-100.vcf

@@ -0,0 +1,100 @@
+##fileformat=VCFv4.2
+##fileDate=20200615
+##fileEncoding=US-ASCII
+##source=CLC Genomics Grid Worker 20.9.9 Beta 1 build 209900
+##reference=/CLC_References/homo_sapiens/sequence/hg38_no_alt_analysis_set/Homo_sapiens_sequence_hg38_no_alt_analysis_set
+##contig=<ID=1,length=248956422>
+##contig=<ID=2,length=242193529>
+##FILTER=<ID=PASS,Description=All filters passed>
+##INFO=<ID=AC,Number=A,Type=Integer,Description=Allele count in genotypes>
+##INFO=<ID=AF,Number=A,Type=Float,Description=Allele Frequency>
+##INFO=<ID=DP,Number=1,Type=Integer,Description=Total Depth>
+##FORMAT=<ID=GT,Number=1,Type=String,Description=Genotype>
+##FORMAT=<ID=DP,Number=1,Type=Integer,Description=Read Depth>
+#CHROM	POS	ID	REF	ALT	QUAL	FILTER	INFO	FORMAT	HG002
+1	1013	rs0000001	A	G	101	PASS	AC=1;AF=0.1;DP=11	GT:DP	0/1:11
+2	1026	rs0000002	C	T	102	PASS	AC=1;AF=0.2;DP=12	GT:DP	0/1:12
+1	1039	rs0000003	G	A	103	PASS	AC=1;AF=0.3;DP=13	GT:DP	0/1:13
+2	1052	rs0000004	T	C	104	PASS	AC=1;AF=0.4;DP=14	GT:DP	0/1:14
+1	1065	rs0000005	A	G	105	PASS	AC=1;AF=0.5;DP=15	GT:DP	0/1:15
+2	1078	rs0000006	C	T	106	PASS	AC=1;AF=0.6;DP=16	GT:DP	0/1:16
+1	1091	rs0000007	G	A	107	PASS	AC=1;AF=0.7;DP=17	GT:DP	0/1:17
+2	1104	rs0000008	T	C	108	PASS	AC=1;AF=0.8;DP=18	GT:DP	0/1:18
+1	1117	rs0000009	A	G	109	PASS	AC=1;AF=0.9;DP=19	GT:DP	0/1:19
+2	1130	rs0000010	C	T	110	PASS	AC=1;AF=0.0;DP=20	GT:DP	0/1:20
+1	1143	rs0000011	G	A	111	PASS	AC=1;AF=0.1;DP=21	GT:DP	0/1:21
+2	1156	rs0000012	T	C	112	PASS	AC=1;AF=0.2;DP=22	GT:DP	0/1:22
+1	1169	rs0000013	A	G	113	PASS	AC=1;AF=0.3;DP=23	GT:DP	0/1:23
+2	1182	rs0000014	C	T	114	PASS	AC=1;AF=0.4;DP=24	GT:DP	0/1:24
+1	1195	rs0000015	G	A	115	PASS	AC=1;AF=0.5;DP=25	GT:DP	0/1:25
+2	1208	rs0000016	T	C	116	PASS	AC=1;AF=0.6;DP=26	GT:DP	0/1:26
+1	1221	rs0000017	A	G	117	PASS	AC=1;AF=0.7;DP=27	GT:DP	0/1:27
+2	1234	rs0000018	C	T	118	PASS	AC=1;AF=0.8;DP=28	GT:DP	0/1:28
+1	1247	rs0000019	G	A	119	PASS	AC=1;AF=0.9;DP=29	GT:DP	0/1:29
+2	1260	rs0000020	T	C	120	PASS	AC=1;AF=0.0;DP=30	GT:DP	0/1:30
+1	1273	rs0000021	A	G	121	PASS	AC=1;AF=0.1;DP=31	GT:DP	0/1:31
+2	1286	rs0000022	C	T	122	PASS	AC=1;AF=0.2;DP=32	GT:DP	0/1:32
+1	1299	rs0000023	G	A	123	PASS	AC=1;AF=0.3;DP=33	GT:DP	0/1:33
+2	1312	rs0000024	T	C	124	PASS	AC=1;AF=0.4;DP=34	GT:DP	0/1:34
+1	1325	rs0000025	A	G	125	PASS	AC=1;AF=0.5;DP=35	GT:DP	0/1:35
+2	1338	rs0000026	C	T	126	PASS	AC=1;AF=0.6;DP=36	GT:DP	0/1:36
+1	1351	rs0000027	G	A	127	PASS	AC=1;AF=0.7;DP=37	GT:DP	0/1:37
+2	1364	rs0000028	T	C	128	PASS	AC=1;AF=0.8;DP=38	GT:DP	0/1:38
+1	1377	rs0000029	A	G	129	PASS	AC=1;AF=0.9;DP=39	GT:DP	0/1:39
+2	1390	rs0000030	C	T	130	PASS	AC=1;AF=0.0;DP=40	GT:DP	0/1:40
+1	1403	rs0000031	G	A	131	PASS	AC=1;AF=0.1;DP=41	GT:DP	0/1:41
+2	1416	rs0000032	T	C	132	PASS	AC=1;AF=0.2;DP=42	GT:DP	0/1:42
+1	1429	rs0000033	A	G	133	PASS	AC=1;AF=0.3;DP=43	GT:DP	0/1:43
+2	1442	rs0000034	C	T	134	PASS	AC=1;AF=0.4;DP=44	GT:DP	0/1:44
+1	1455	rs0000035	G	A	135	PASS	AC=1;AF=0.5;DP=45	GT:DP	0/1:45
+2	1468	rs0000036	T	C	136	PASS	AC=1;AF=0.6;DP=46	GT:DP	0/1:46
+1	1481	rs0000037	A	G	137	PASS	AC=1;AF=0.7;DP=47	GT:DP	0/1:47
+2	1494	rs0000038	C	T	138	PASS	AC=1;AF=0.8;DP=48	GT:DP	0/1:48
+1	1507	rs0000039	G	A	139	PASS	AC=1;AF=0.9;DP=49	GT:DP	0/1:49
+2	1520	rs0000040	T	C	140	PASS	AC=1;AF=0.0;DP=50	GT:DP	0/1:50
+1	1533	rs0000041	A	G	141	PASS	AC=1;AF=0.1;DP=51	GT:DP	0/1:51
+2	1546	rs0000042	C	T	142	PASS	AC=1;AF=0.2;DP=52	GT:DP	0/1:52
+1	1559	rs0000043	G	A	143	PASS	AC=1;AF=0.3;DP=53	GT:DP	0/1:53
+2	1572	rs0000044	T	C	144	PASS	AC=1;AF=0.4;DP=54	GT:DP	0/1:54
+1	1585	rs0000045	A	G	145	PASS	AC=1;AF=0.5;DP=55	GT:DP	0/1:55
+2	1598	rs0000046	C	T	146	PASS	AC=1;AF=0.6;DP=56	GT:DP	0/1:56
+1	1611	rs0000047	G	A	147	PASS	AC=1;AF=0.7;DP=57	GT:DP	0/1:57
+2	1624	rs0000048	T	C	148	PASS	AC=1;AF=0.8;DP=58	GT:DP	0/1:58
+1	1637	rs0000049	A	G	149	PASS	AC=1;AF=0.9;DP=59	GT:DP	0/1:59
+2	1650	rs0000050	C	T	150	PASS	AC=1;AF=0.0;DP=60	GT:DP	0/1:60
+1	1663	rs0000051	G	A	151	PASS	AC=1;AF=0.1;DP=61	GT:DP	0/1:61
+2	1676	rs0000052	T	C	152	PASS	AC=1;AF=0.2;DP=62	GT:DP	0/1:62
+1	1689	rs0000053	A	G	153	PASS	AC=1;AF=0.3;DP=63	GT:DP	0/1:63
+2	1702	rs0000054	C	T	154	PASS	AC=1;AF=0.4;DP=64	GT:DP	0/1:64
+1	1715	rs0000055	G	A	155	PASS	AC=1;AF=0.5;DP=65	GT:DP	0/1:65
+2	1728	rs0000056	T	C	156	PASS	AC=1;AF=0.6;DP=66	GT:DP	0/1:66
+1	1741	rs0000057	A	G	157	PASS	AC=1;AF=0.7;DP=67	GT:DP	0/1:67
+2	1754	rs0000058	C	T	158	PASS	AC=1;AF=0.8;DP=68	GT:DP	0/1:68
+1	1767	rs0000059	G	A	159	PASS	AC=1;AF=0.9;DP=69	GT:DP	0/1:69
+2	1780	rs0000060	T	C	160	PASS	AC=1;AF=0.0;DP=70	GT:DP	0/1:70
+1	1793	rs0000061	A	G	161	PASS	AC=1;AF=0.1;DP=71	GT:DP	0/1:71
+2	1806	rs0000062	C	T	162	PASS	AC=1;AF=0.2;DP=72	GT:DP	0/1:72
+1	1819	rs0000063	G	A	163	PASS	AC=1;AF=0.3;DP=73	GT:DP	0/1:73
+2	1832	rs0000064	T	C	164	PASS	AC=1;AF=0.4;DP=74	GT:DP	0/1:74
+1	1845	rs0000065	A	G	165	PASS	AC=1;AF=0.5;DP=75	GT:DP	0/1:75
+2	1858	rs0000066	C	T	166	PASS	AC=1;AF=0.6;DP=76	GT:DP	0/1:76
+1	1871	rs0000067	G	A	167	PASS	AC=1;AF=0.7;DP=77	GT:DP	0/1:77
+2	1884	rs0000068	T	C	168	PASS	AC=1;AF=0.8;DP=78	GT:DP	0/1:78
+1	1897	rs0000069	A	G	169	PASS	AC=1;AF=0.9;DP=79	GT:DP	0/1:79
+2	1910	rs0000070	C	T	170	PASS	AC=1;AF=0.0;DP=80	GT:DP	0/1:80
+1	1923	rs0000071	G	A	171	PASS	AC=1;AF=0.1;DP=81	GT:DP	0/1:81
+2	1936	rs0000072	T	C	172	PASS	AC=1;AF=0.2;DP=82	GT:DP	0/1:82
+1	1949	rs0000073	A	G	173	PASS	AC=1;AF=0.3;DP=83	GT:DP	0/1:83
+2	1962	rs0000074	C	T	174	PASS	AC=1;AF=0.4;DP=84	GT:DP	0/1:84
+1	1975	rs0000075	G	A	175	PASS	AC=1;AF=0.5;DP=85	GT:DP	0/1:85
+2	1988	rs0000076	T	C	176	PASS	AC=1;AF=0.6;DP=86	GT:DP	0/1:86
+1	2001	rs0000077	A	G	177	PASS	AC=1;AF=0.7;DP=87	GT:DP	0/1:87
+2	2014	rs0000078	C	T	178	PASS	AC=1;AF=0.8;DP=88	GT:DP	0/1:88
+1	2027	rs0000079	G	A	179	PASS	AC=1;AF=0.9;DP=89	GT:DP	0/1:89
+2	2040	rs0000080	T	C	180	PASS	AC=1;AF=0.0;DP=90	GT:DP	0/1:90
+1	2053	rs0000081	A	G	181	PASS	AC=1;AF=0.1;DP=91	GT:DP	0/1:91
+2	2066	rs0000082	C	T	182	PASS	AC=1;AF=0.2;DP=92	GT:DP	0/1:92
+1	2079	rs0000083	G	A	183	PASS	AC=1;AF=0.3;DP=93	GT:DP	0/1:93
+2	2092	rs0000084	T	C	184	PASS	AC=1;AF=0.4;DP=94	GT:DP	0/1:94
+1	2105	rs0000085	A	G	185	PASS	AC=1;AF=0.5;DP=95	GT:DP	0/1:95
+2	2118	rs0000086	C	T	186	PASS	AC=1;AF=0.6;DP=96	GT:DP	0/1:96