Merge pull request #1 from ecrum19/dev

v2.0.0 - Compression and Optimization Refactoring

Author: ecrum19

.github/workflows/publish-docker.yml

@@ -9,10 +9,13 @@ on:
   workflow_dispatch:
     inputs:
       docker_tag:
-        description: "Optional explicit Docker tag (example: v1.1.1)"
+        description: "Optional explicit semantic Docker tag (example: 1.2.4 or v1.2.4)"
         required: false
         type: string
 
+env:
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+
 jobs:
   docker:
     name: build-and-push-docker
@@ -71,8 +74,8 @@ jobs:
           ref_name = os.environ.get("GITHUB_REF_NAME", "")
           head_message = os.environ.get("HEAD_COMMIT_MESSAGE", "")
           manual_tag = os.environ.get("INPUT_DOCKER_TAG", "").strip()
-          semver = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$")
-          explicit_from_commit = re.search(r"\[(?:publish-docker|docker)=([A-Za-z0-9_.-]+)\]", head_message)
+          semver = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)$")
+          explicit_from_commit = re.search(r"\[(?:publish-docker|docker)=(v?\d+\.\d+\.\d+)\]", head_message)
 
           def parse_max(tags):
               best = (-1, -1, -1)
@@ -105,16 +108,22 @@ jobs:
                   best = max(best, parse_max(tags))
               major, minor, patch = best
               if major < 0:
-                  resolved_tag = "v1.0.0"
+                  resolved_tag = "1.0.0"
               else:
-                  resolved_tag = f"v{major}.{minor}.{patch + 1}"
+                  resolved_tag = f"{major}.{minor}.{patch + 1}"
 
-          if not re.match(r"^[A-Za-z0-9_.-]+$", resolved_tag):
-              raise SystemExit(f"Invalid Docker tag: {resolved_tag}")
-          print(f"Resolved Docker tag: {resolved_tag}")
+          match = semver.fullmatch(resolved_tag)
+          if not match:
+              raise SystemExit(
+                  f"Docker release tag must match MAJOR.MINOR.PATCH or vMAJOR.MINOR.PATCH: {resolved_tag}"
+              )
+          version_tag = ".".join(match.groups())
+          version_tag_with_v = f"v{version_tag}"
+          print(f"Resolved Docker tags: {version_tag}, {version_tag_with_v}, latest")
 
           with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f:
-              f.write(f"next_version={resolved_tag}\n")
+              f.write(f"version_tag={version_tag}\n")
+              f.write(f"version_tag_with_v={version_tag_with_v}\n")
           PY
 
       - name: Docker metadata
@@ -123,7 +132,8 @@ jobs:
         with:
           images: ecrum19/vcf-rdfizer
           tags: |
-            type=raw,value=${{ steps.version.outputs.next_version }}
+            type=raw,value=${{ steps.version.outputs.version_tag }}
+            type=raw,value=${{ steps.version.outputs.version_tag_with_v }}
             type=raw,value=latest
 
       - name: Build and push

.github/workflows/publish-python.yml

@@ -3,32 +3,36 @@ name: publish-python
 on:
   push:
     tags:
-      - "v*"
+      - "v*.*.*"
   workflow_dispatch:
     inputs:
       release_tag:
-        description: "Optional Git tag to sync from, for example v1.2.3"
-        required: false
+        description: "Existing Git tag to validate, for example v1.2.4. Manual runs never publish to PyPI."
+        required: true
         type: string
 
+env:
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+
 jobs:
   build:
     name: build-sdist-wheel
     runs-on: ubuntu-latest
+    env:
+      RELEASE_TAG: ${{ github.event.inputs.release_tag || github.ref_name }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.release_tag || github.ref }}
 
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 
-      - name: Sync release metadata from tag
-        env:
-          RELEASE_TAG: ${{ github.event.inputs.release_tag || github.ref_name }}
-        run: |
-          python scripts/release.py --tag "${RELEASE_TAG}" --workflow
+      - name: Validate committed release metadata
+        run: python scripts/release.py --check-tag "${RELEASE_TAG}"
 
       - name: Build package
         run: |
@@ -38,8 +42,6 @@ jobs:
           python -m twine check dist/*
 
       - name: Verify built artifact versions
-        env:
-          RELEASE_TAG: ${{ github.event.inputs.release_tag || github.ref_name }}
         run: |
           python - <<'PY'
           from pathlib import Path
@@ -85,6 +87,27 @@ jobs:
               print(f"  {name}")
           PY
 
+      - name: Smoke-test installed wheel
+        run: |
+          python -m venv /tmp/vcf-rdfizer-wheel-test
+          /tmp/vcf-rdfizer-wheel-test/bin/python -m pip install --no-deps dist/*.whl
+          (
+            cd /tmp
+            /tmp/vcf-rdfizer-wheel-test/bin/vcf-rdfizer --help
+            /tmp/vcf-rdfizer-wheel-test/bin/python - <<'PY'
+          import importlib.resources
+          from pathlib import Path
+
+          import vcf_rdfizer
+
+          rules = importlib.resources.files("vcf_rdfizer_data").joinpath("rules/default_rules.ttl")
+          if not rules.is_file():
+              raise SystemExit("Packaged default rules are missing")
+          if not vcf_rdfizer.resolve_default_rules_path(Path(vcf_rdfizer.__file__).resolve().parent).is_file():
+              raise SystemExit("Installed wrapper cannot resolve packaged default rules")
+          PY
+          )
+
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:

.github/workflows/tests.yml

@@ -6,6 +6,9 @@ on:
       - "**"
   pull_request:
 
+env:
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+
 jobs:
   wrapper-cross-platform:
     name: wrapper-unit (${{ matrix.os }})

.gitignore

@@ -8,6 +8,8 @@ test/test_vcf_files/*
 !test/test_vcf_files/test-100.vcf
 !test/test_vcf_files/test-1k.vcf
 !test/test_vcf_files/test-10k.vcf
+!test/test_vcf_files/test-larger.vcf.gz
+
 RMLStreamer-v2.5.0-standalone.jar
 out/
 tsv/

Dockerfile

@@ -1,4 +1,5 @@
 ARG RMLSTREAMER_VERSION=2.5.0
+ARG HDT_JAVA_PACKAGE_VERSION=3.0.10
 
 FROM eclipse-temurin:11-jre AS build-hdt-cpp
 
@@ -37,6 +38,7 @@ RUN mkdir -p /opt/third_party_licenses \
 FROM eclipse-temurin:11-jre
 
 ARG RMLSTREAMER_VERSION
+ARG HDT_JAVA_PACKAGE_VERSION
 
 RUN apt-get update \
   && apt-get install -y --no-install-recommends \
@@ -50,30 +52,50 @@ RUN apt-get update \
     gzip \
     libserd-0-0 \
     nodejs \
+    python3 \
+    python3-venv \
     time \
   && rm -rf /var/lib/apt/lists/*
 
+RUN python3 -m venv /opt/pycottas-venv \
+  && /opt/pycottas-venv/bin/pip install --no-cache-dir pycottas
+
 RUN mkdir -p /opt/rmlstreamer \
   && curl -fsSL \
     -o /opt/rmlstreamer/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar \
     https://github.com/RMLio/RMLStreamer/releases/download/v${RMLSTREAMER_VERSION}/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar
 
+RUN mkdir -p /opt/hdt-java \
+  && curl -fsSL \
+    -o /tmp/hdt-java-package.tar.gz \
+    https://repo1.maven.org/maven2/org/rdfhdt/hdt-java-package/${HDT_JAVA_PACKAGE_VERSION}/hdt-java-package-${HDT_JAVA_PACKAGE_VERSION}-distribution.tar.gz \
+  && tar -xzf /tmp/hdt-java-package.tar.gz -C /opt/hdt-java --strip-components=1 \
+  && rm -f /tmp/hdt-java-package.tar.gz
+
 COPY --from=build-hdt-cpp /usr/local/bin/rdf2hdt /usr/local/bin/rdf2hdt
 COPY --from=build-hdt-cpp /usr/local/bin/hdt2rdf /usr/local/bin/hdt2rdf
 COPY --from=build-hdt-cpp /usr/local/lib/libcds* /usr/local/lib/
 COPY --from=build-hdt-cpp /usr/local/lib/libhdt* /usr/local/lib/
 COPY --from=build-hdt-cpp /opt/third_party_licenses/ /usr/share/licenses/vcf-rdfizer/
 COPY THIRD_PARTY_NOTICES.md /usr/share/licenses/vcf-rdfizer/THIRD_PARTY_NOTICES.md
 COPY src/*.sh /opt/vcf-rdfizer/
+COPY src/*.py /opt/vcf-rdfizer/
 
 RUN chmod +x /opt/vcf-rdfizer/*.sh \
+  && find /opt/hdt-java/bin -type f -exec chmod +x {} \; \
   && chmod +x /usr/local/bin/rdf2hdt \
   && chmod +x /usr/local/bin/hdt2rdf
 
 ENV RMLSTREAMER_JAR=/opt/rmlstreamer/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar
 ENV JAR=/opt/rmlstreamer/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar
+ENV HDT_JAVA_HOME=/opt/hdt-java
 ENV RDF2HDT_BIN=/usr/local/bin/rdf2hdt
 ENV HDT2RDF_BIN=/usr/local/bin/hdt2rdf
+ENV COTTAS_PYTHON_BIN=/opt/pycottas-venv/bin/python
 ENV LD_LIBRARY_PATH=/usr/local/lib
 
+# COTTAS creates a temporary DuckDB database in the container working
+# directory. The wrapper runs containers as the host UID/GID, so this path
+# must be writable without requiring root or creating root-owned host files.
+RUN mkdir -p /work && chmod 1777 /work
 WORKDIR /work