Skip to content

Commit 427fc80

Browse files
ecrum19ecrum19
committed
fixed bug where compressed rdf file removed with raw rdf 2
1 parent 20051e7 commit 427fc80

5 files changed

Lines changed: 51 additions & 54 deletions

File tree

‎README.md‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -73,12 +73,12 @@ python3 vcf_rdfizer.py --mode compress --nq ./out/sample/sample.nt --compression
7373

7474
5. Decompression-only mode (auto output path under `./out/`):
7575
```bash
76-
python3 vcf_rdfizer.py --mode decompress --compressed-input ./out/gzip/sample.nt.gz
76+
python3 vcf_rdfizer.py --mode decompress --compressed-input ./out/sample/sample.nt.gz
7777
```
7878

7979
6. Decompression-only mode with explicit output file:
8080
```bash
81-
python3 vcf_rdfizer.py --mode decompress --compressed-input ./out/sample.hdt --decompress-out ./out/sample_from_hdt.nt
81+
python3 vcf_rdfizer.py --mode decompress --compressed-input ./out/sample/sample.hdt --decompress-out ./out/sample_from_hdt.nt
8282
```
8383

8484
Outputs:
@@ -87,12 +87,12 @@ Outputs:
8787
- conversion outputs per TSV basename in `./out/<sample>/`
8888
- merged N-Triples file:
8989
- `./out/<sample>/<sample>.nt` (retained only when `--keep-rdf` is set)
90-
- compressed outputs are written directly under `./out/`:
91-
- `./out/*.gz`
92-
- `./out/*.br`
93-
- `./out/*.hdt`
90+
- compressed outputs are written in each sample directory:
91+
- `./out/<sample>/<sample>.nt.gz` (or `.nq.gz` for legacy inputs)
92+
- `./out/<sample>/<sample>.nt.br` (or `.nq.br` for legacy inputs)
93+
- `./out/<sample>/<sample>.hdt`
9494
- decompressed outputs (decompression mode default):
95-
- `./out/*.nt`
95+
- `./out/<sample>/<sample>.nt`
9696
- `./run_metrics` for logs and metrics
9797
- `run_metrics/metrics.csv` includes both conversion and compression metrics per run
9898
- conversion step artifacts:

‎src/compression.sh‎

Lines changed: 4 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -87,20 +87,6 @@ if (( DO_HDT == 1 )); then
8787
fi
8888
fi
8989

90-
GZIP_ROOT="$OUT_ROOT_DIR"
91-
BROTLI_ROOT="$OUT_ROOT_DIR"
92-
HDT_ROOT="$OUT_ROOT_DIR"
93-
94-
if (( DO_GZIP == 1 )); then
95-
mkdir -p "$GZIP_ROOT"
96-
fi
97-
if (( DO_BROTLI == 1 )); then
98-
mkdir -p "$BROTLI_ROOT"
99-
fi
100-
if (( DO_HDT == 1 )); then
101-
mkdir -p "$HDT_ROOT"
102-
fi
103-
10490
# Resolve output directories to compress
10591
OUTPUT_DIRS=()
10692
if [[ -n "$OUT_NAME" ]]; then
@@ -209,6 +195,7 @@ for OUT in "${OUTPUT_DIRS[@]}"; do
209195
OVERALL_EXIT=1
210196
continue
211197
fi
198+
mkdir -p "$OUT"
212199

213200
BASENAME=$(basename "$OUT")
214201
SAFE_BASENAME=$(printf "%s" "$BASENAME" | tr -cs 'A-Za-z0-9._-' '_')
@@ -310,7 +297,7 @@ for OUT in "${OUTPUT_DIRS[@]}"; do
310297

311298
# ----- gzip combined RDF with timing -----
312299
if (( DO_GZIP == 1 )) && [[ -n "${SOURCE_RDF:-}" ]]; then
313-
GZ_PATH="$GZIP_ROOT/${BASENAME}.${SOURCE_EXT}.gz"
300+
GZ_PATH="$OUT/${BASENAME}.${SOURCE_EXT}.gz"
314301
EXIT_CODE_GZIP=0
315302

316303
if have_gnu_time; then
@@ -356,7 +343,7 @@ for OUT in "${OUTPUT_DIRS[@]}"; do
356343

357344
# ----- brotli combined RDF with timing -----
358345
if (( DO_BROTLI == 1 )) && [[ -n "${SOURCE_RDF:-}" ]]; then
359-
BROTLI_PATH="$BROTLI_ROOT/${BASENAME}.${SOURCE_EXT}.br"
346+
BROTLI_PATH="$OUT/${BASENAME}.${SOURCE_EXT}.br"
360347
EXIT_CODE_BROTLI=0
361348

362349
if have_gnu_time; then
@@ -401,7 +388,7 @@ for OUT in "${OUTPUT_DIRS[@]}"; do
401388

402389
# ----- Convert combined RDF to HDT with timing -----
403390
if (( DO_HDT == 1 )) && [[ -n "${SOURCE_RDF:-}" ]]; then
404-
HDT_PATH="$HDT_ROOT/$BASENAME.hdt"
391+
HDT_PATH="$OUT/$BASENAME.hdt"
405392
EXIT_CODE_HDT=0
406393

407394
HDT_CMD="bash \"$HDT\" \"$SOURCE_RDF\" \"$HDT_PATH\""

‎test/test_compression_unit.py‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -151,9 +151,9 @@ def test_compression_updates_existing_metrics_row_with_mocked_tools(self):
151151
)
152152

153153
self.assertEqual(result.returncode, 0, msg=result.stderr)
154-
self.assertTrue((out_root / "rdf.nq.gz").exists())
155-
self.assertTrue((out_root / "rdf.nq.br").exists())
156-
self.assertTrue((out_root / "rdf.hdt").exists())
154+
self.assertTrue((output / "rdf.nq.gz").exists())
155+
self.assertTrue((output / "rdf.nq.br").exists())
156+
self.assertTrue((output / "rdf.hdt").exists())
157157
self.assertTrue((logdir / "compression-time-gzip-rdf-run-compress-1.txt").exists())
158158
self.assertTrue((logdir / "compression-time-brotli-rdf-run-compress-1.txt").exists())
159159
self.assertTrue((logdir / "compression-time-hdt-rdf-run-compress-1.txt").exists())
@@ -208,9 +208,9 @@ def test_compression_prefers_nt_when_present(self):
208208
text=True,
209209
)
210210
self.assertEqual(result.returncode, 0, msg=result.stderr)
211-
self.assertTrue((out_root / "rdf.nt.gz").exists())
212-
self.assertTrue((out_root / "rdf.nt.br").exists())
213-
self.assertTrue((out_root / "rdf.hdt").exists())
211+
self.assertTrue((output / "rdf.nt.gz").exists())
212+
self.assertTrue((output / "rdf.nt.br").exists())
213+
self.assertTrue((output / "rdf.hdt").exists())
214214

215215
def test_compression_none_updates_metrics_without_generating_outputs(self):
216216
"""Compression mode none leaves no compressed artifacts and records zero sizes."""
@@ -251,9 +251,9 @@ def test_compression_none_updates_metrics_without_generating_outputs(self):
251251
)
252252

253253
self.assertEqual(result.returncode, 0, msg=result.stderr)
254-
self.assertFalse((out_root / "rdf.nq.gz").exists())
255-
self.assertFalse((out_root / "rdf.nq.br").exists())
256-
self.assertFalse((out_root / "rdf.hdt").exists())
254+
self.assertFalse((output / "rdf.nq.gz").exists())
255+
self.assertFalse((output / "rdf.nq.br").exists())
256+
self.assertFalse((output / "rdf.hdt").exists())
257257

258258
row = read_metrics_row(metrics_csv, run_id, "rdf")
259259
self.assertEqual(row["compression_methods"], "none")

‎test/test_vcf_rdfizer_unit.py‎

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -195,9 +195,9 @@ def fake_run(cmd, cwd=None, env=None):
195195
self.assertEqual(rc, 0)
196196
self.assertEqual(len(commands), 2)
197197
self.assertIn("gzip -c", commands[0][-1])
198-
self.assertIn("/data/out/sample.nq.gz", commands[0][-1])
198+
self.assertIn("/data/out/sample/sample.nq.gz", commands[0][-1])
199199
self.assertIn("brotli -q 7 -c", commands[1][-1])
200-
self.assertIn("/data/out/sample.nq.br", commands[1][-1])
200+
self.assertIn("/data/out/sample/sample.nq.br", commands[1][-1])
201201

202202
def test_main_compress_mode_accepts_nt_and_preserves_extension(self):
203203
"""Compression mode accepts .nt input and emits extension-aware output names."""
@@ -237,7 +237,7 @@ def fake_run(cmd, cwd=None, env=None):
237237

238238
self.assertEqual(rc, 0)
239239
self.assertEqual(len(commands), 1)
240-
self.assertIn("/data/out/sample.nt.gz", commands[0][-1])
240+
self.assertIn("/data/out/sample/sample.nt.gz", commands[0][-1])
241241

242242
def test_main_compress_mode_requires_nq_argument(self):
243243
"""Compression mode fails validation when --nq is missing."""
@@ -331,6 +331,7 @@ def fake_run(cmd, cwd=None, env=None):
331331
self.assertEqual(rc, 0)
332332
self.assertEqual(len(commands), 1)
333333
self.assertIn("gzip -dc", commands[0][-1])
334+
self.assertTrue(any(arg.endswith("/out/sample:/data/out") for arg in commands[0]))
334335
self.assertIn("/data/out/sample.nq", commands[0][-1])
335336

336337
def test_main_decompress_mode_hdt_uses_hdt2rdf(self):
@@ -367,6 +368,7 @@ def fake_run(cmd, cwd=None, env=None):
367368
self.assertEqual(rc, 0)
368369
self.assertEqual(len(commands), 1)
369370
self.assertIn("/opt/hdt-java/hdt-java-cli/bin/hdt2rdf.sh", commands[0][-1])
371+
self.assertTrue(any(arg.endswith("/out/sample:/data/out") for arg in commands[0]))
370372
self.assertIn("/data/out/sample.nt", commands[0][-1])
371373

372374
def test_main_decompress_mode_rejects_unknown_extension(self):
@@ -519,8 +521,9 @@ def fake_run(cmd, cwd=None, env=None):
519521
(out_sample_dir / f"{output_name}.nt").write_text("<s> <p> <o> .\n")
520522
if "/opt/vcf-rdfizer/compression.sh" in cmd:
521523
output_name = next(part.split("=", 1)[1] for part in cmd if part.startswith("OUT_NAME="))
522-
out_dir.mkdir(parents=True, exist_ok=True)
523-
(out_dir / f"{output_name}.hdt").write_text("fake-hdt\n")
524+
out_sample_dir = out_dir / output_name
525+
out_sample_dir.mkdir(parents=True, exist_ok=True)
526+
(out_sample_dir / f"{output_name}.hdt").write_text("fake-hdt\n")
524527
return 0
525528

526529
old_cwd = os.getcwd()
@@ -549,7 +552,7 @@ def fake_run(cmd, cwd=None, env=None):
549552

550553
self.assertEqual(rc, 0)
551554
self.assertFalse((out_dir / "sample" / "sample.nt").exists())
552-
self.assertTrue((out_dir / "sample.hdt").exists())
555+
self.assertTrue((out_dir / "sample" / "sample.hdt").exists())
553556

554557
def test_main_full_mode_keep_rdf_preserves_nt_after_compression(self):
555558
"""Full mode keeps merged .nt outputs when --keep-rdf is provided."""
@@ -566,8 +569,9 @@ def fake_run(cmd, cwd=None, env=None):
566569
(out_sample_dir / f"{output_name}.nt").write_text("<s> <p> <o> .\n")
567570
if "/opt/vcf-rdfizer/compression.sh" in cmd:
568571
output_name = next(part.split("=", 1)[1] for part in cmd if part.startswith("OUT_NAME="))
569-
out_dir.mkdir(parents=True, exist_ok=True)
570-
(out_dir / f"{output_name}.hdt").write_text("fake-hdt\n")
572+
out_sample_dir = out_dir / output_name
573+
out_sample_dir.mkdir(parents=True, exist_ok=True)
574+
(out_sample_dir / f"{output_name}.hdt").write_text("fake-hdt\n")
571575
return 0
572576

573577
old_cwd = os.getcwd()
@@ -597,7 +601,7 @@ def fake_run(cmd, cwd=None, env=None):
597601

598602
self.assertEqual(rc, 0)
599603
self.assertTrue((out_dir / "sample" / "sample.nt").exists())
600-
self.assertTrue((out_dir / "sample.hdt").exists())
604+
self.assertTrue((out_dir / "sample" / "sample.hdt").exists())
601605

602606
def test_main_full_mode_deletes_nt_with_docker_fallback_on_permission_error(self):
603607
"""Full mode falls back to Docker-based removal when .nt unlink raises PermissionError."""
@@ -616,8 +620,9 @@ def fake_run(cmd, cwd=None, env=None):
616620
target_nt.parent.mkdir(parents=True, exist_ok=True)
617621
target_nt.write_text("<s> <p> <o> .\n")
618622
if "/opt/vcf-rdfizer/compression.sh" in cmd:
619-
out_dir.mkdir(parents=True, exist_ok=True)
620-
(out_dir / "sample.hdt").write_text("fake-hdt\n")
623+
out_sample_dir = out_dir / "sample"
624+
out_sample_dir.mkdir(parents=True, exist_ok=True)
625+
(out_sample_dir / "sample.hdt").write_text("fake-hdt\n")
621626
if isinstance(cmd, list) and cmd[-1].startswith("rm -f ") and "/data/out/sample/sample.nt" in cmd[-1]:
622627
if target_nt_resolved.exists():
623628
original_unlink(target_nt_resolved)
@@ -676,8 +681,9 @@ def fake_run(cmd, cwd=None, env=None):
676681
(out_sample_dir / f"{output_name}.nq").write_text("<s> <p> <o> <g> .\n")
677682
if "/opt/vcf-rdfizer/compression.sh" in cmd:
678683
output_name = next(part.split("=", 1)[1] for part in cmd if part.startswith("OUT_NAME="))
679-
out_dir.mkdir(parents=True, exist_ok=True)
680-
(out_dir / f"{output_name}.hdt").write_text("fake-hdt\n")
684+
out_sample_dir = out_dir / output_name
685+
out_sample_dir.mkdir(parents=True, exist_ok=True)
686+
(out_sample_dir / f"{output_name}.hdt").write_text("fake-hdt\n")
681687
return 0
682688

683689
old_cwd = os.getcwd()
@@ -706,7 +712,7 @@ def fake_run(cmd, cwd=None, env=None):
706712

707713
self.assertEqual(rc, 0)
708714
self.assertFalse((out_dir / "sample" / "sample.nq").exists())
709-
self.assertTrue((out_dir / "sample.hdt").exists())
715+
self.assertTrue((out_dir / "sample" / "sample.hdt").exists())
710716

711717
def test_main_ignores_unrelated_existing_tsv_triplets(self):
712718
"""Wrapper converts only triplets that match the CLI-selected VCF snapshot."""

‎vcf_rdfizer.py‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -588,7 +588,7 @@ def run_full_mode(
588588

589589
nt_path = out_dir / output_name / f"{output_name}.nt"
590590
nq_path = out_dir / output_name / f"{output_name}.nq"
591-
hdt_path = out_dir / f"{output_name}.hdt"
591+
hdt_path = out_dir / output_name / f"{output_name}.hdt"
592592
nt_size_before_cleanup = file_size_bytes(nt_path)
593593
nq_size_before_cleanup = file_size_bytes(nq_path)
594594

@@ -670,19 +670,22 @@ def run_compress_mode(
670670
input_container = f"/data/in/{nq_path.name}"
671671
input_stem = nq_path.stem
672672
input_ext = nq_path.suffix.lstrip(".") or "nt"
673+
target_out_dir = out_dir / input_stem
674+
ensure_dir(target_out_dir)
675+
target_out_container = f"/data/out/{input_stem}"
673676

674677
for method in methods:
675678
if method == "gzip":
676679
output_name = f"{input_stem}.{input_ext}.gz"
677-
out_container = f"/data/out/{output_name}"
680+
out_container = f"{target_out_container}/{output_name}"
678681
command = f"gzip -c {shlex.quote(input_container)} > {shlex.quote(out_container)}"
679682
elif method == "brotli":
680683
output_name = f"{input_stem}.{input_ext}.br"
681-
out_container = f"/data/out/{output_name}"
684+
out_container = f"{target_out_container}/{output_name}"
682685
command = f"brotli -q 7 -c {shlex.quote(input_container)} > {shlex.quote(out_container)}"
683686
else:
684687
output_name = f"{input_stem}.hdt"
685-
out_container = f"/data/out/{output_name}"
688+
out_container = f"{target_out_container}/{output_name}"
686689
command = (
687690
"set -euo pipefail; "
688691
"HDT_BIN=/opt/hdt-java/hdt-java-cli/bin/rdf2hdt.sh; "
@@ -714,8 +717,8 @@ def run_compress_mode(
714717
print(f" - {method}: {output_name} ✅")
715718

716719
nt_path = nq_path if nq_path.suffix == ".nt" else nq_path.with_suffix(".nt")
717-
hdt_path = out_dir / f"{input_stem}.hdt"
718-
print_nt_hdt_summary(output_root=out_dir, nt_path=nt_path, hdt_path=hdt_path, indent=" ")
720+
hdt_path = target_out_dir / f"{input_stem}.hdt"
721+
print_nt_hdt_summary(output_root=target_out_dir, nt_path=nt_path, hdt_path=hdt_path, indent=" ")
719722
print("Conversion process finished.")
720723
return 0
721724

@@ -946,7 +949,8 @@ def main():
946949
fmt = detect_compressed_format(compressed_path)
947950
validate_mode_dirs([out_dir, metrics_dir])
948951
if args.decompress_out is None:
949-
decompressed_out = out_dir / default_decompressed_name(compressed_path, fmt)
952+
default_name = default_decompressed_name(compressed_path, fmt)
953+
decompressed_out = out_dir / Path(default_name).stem / default_name
950954
else:
951955
decompressed_out = Path(args.decompress_out).expanduser().resolve()
952956
if decompressed_out.exists() and decompressed_out.is_dir():

0 commit comments

Comments
 (0)