From a17559faf0575938f2b39f41389d535339c19dc0 Mon Sep 17 00:00:00 2001 From: meskill <8974488+meskill@users.noreply.github.com> Date: Wed, 22 Apr 2026 20:17:25 +0000 Subject: [PATCH 1/3] ci: add resharding test to ci --- .github/workflows/ci.yml | 2 ++ .rwx/integration.yml | 11 +---------- integration/resharding/docker-compose.yaml | 8 ++++---- integration/resharding/pgdog.toml | 3 --- integration/resharding/run.sh | 16 ++++++++++++++++ 5 files changed, 23 insertions(+), 17 deletions(-) create mode 100755 integration/resharding/run.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7782a12b8..6e9be8d23 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,6 +170,8 @@ jobs: run: bash integration/complex/run.sh - name: Dry run run: bash integration/dry_run/run.sh + - name: Resharding + run: bash integration/resharding/run.sh - name: Ensure PgDog stopped run: | if pgrep -x pgdog > /dev/null; then diff --git a/.rwx/integration.yml b/.rwx/integration.yml index 8051c7704..def8a5961 100644 --- a/.rwx/integration.yml +++ b/.rwx/integration.yml @@ -449,16 +449,7 @@ tasks: run: | export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/resharding-%p-%m.profraw" - cleanup() { - (cd integration/resharding && docker-compose down >/dev/null 2>&1 || true) - killall -TERM pgdog 2>/dev/null || true - sleep 1 - killall -KILL pgdog 2>/dev/null || true - } - trap cleanup EXIT - - timeout --signal=TERM --kill-after=90s 16m bash integration/resharding/dev.sh - + bash integration/resharding/run.sh cargo llvm-cov report --release --package pgdog --lcov --output-path resharding.lcov outputs: filesystem: false diff --git a/integration/resharding/docker-compose.yaml b/integration/resharding/docker-compose.yaml index b89eef8c0..26a26475b 100644 --- a/integration/resharding/docker-compose.yaml +++ b/integration/resharding/docker-compose.yaml @@ -1,6 +1,6 @@ services: source_0: - image: postgres:18 + image: postgres:16 command: postgres -c wal_level=logical environment: POSTGRES_USER: pgdog @@ -14,7 +14,7 @@ services: - postgres source_1: - image: postgres:18 + image: postgres:16 command: postgres -c wal_level=logical environment: POSTGRES_USER: pgdog @@ -28,7 +28,7 @@ services: - postgres destination_0: - image: postgres:18 + image: postgres:16 command: postgres -c wal_level=logical environment: POSTGRES_USER: pgdog @@ -40,7 +40,7 @@ services: - postgres destination_1: - image: postgres:18 + image: postgres:16 command: postgres -c wal_level=logical environment: POSTGRES_USER: pgdog diff --git a/integration/resharding/pgdog.toml b/integration/resharding/pgdog.toml index 2d627feb6..5aa5e7bf4 100644 --- a/integration/resharding/pgdog.toml +++ b/integration/resharding/pgdog.toml @@ -51,6 +51,3 @@ data_type = "bigint" [admin] password = "pgdog" user = "pgdog" - -[replication] -pg_dump_path = "/usr/lib/postgresql/18/bin/pg_dump" diff --git a/integration/resharding/run.sh b/integration/resharding/run.sh new file mode 100755 index 000000000..71638fcf5 --- /dev/null +++ b/integration/resharding/run.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +# Safety net: docker-compose down and any stray pgdog processes are cleaned up +# on exit even if dev.sh is interrupted mid-flight by timeout or signal. +cleanup() { + (cd "${SCRIPT_DIR}" && docker-compose down >/dev/null 2>&1 || true) + killall -TERM pgdog 2>/dev/null || true + sleep 1 + killall -KILL pgdog 2>/dev/null || true +} +trap cleanup EXIT INT TERM + +timeout --signal=TERM --kill-after=90s 16m bash "${SCRIPT_DIR}/dev.sh" From 99dc4e3944d8ceedaf733fe0b0032bd8019e7e46 Mon Sep 17 00:00:00 2001 From: meskill <8974488+meskill@users.noreply.github.com> Date: Thu, 23 Apr 2026 09:25:09 +0000 Subject: [PATCH 2/3] remove rwx --- .rwx/integration.yml | 604 ------------------------------------------- .rwx/rust.yml | 349 ------------------------- 2 files changed, 953 deletions(-) delete mode 100644 .rwx/integration.yml delete mode 100644 .rwx/rust.yml diff --git a/.rwx/integration.yml b/.rwx/integration.yml deleted file mode 100644 index def8a5961..000000000 --- a/.rwx/integration.yml +++ /dev/null @@ -1,604 +0,0 @@ -on: - github: - pull_request: - init: - commit-sha: ${{ event.git.sha }} - branch: ${{ event.git.branch }} - push: - if: ${{ event.git.branch == "main" }} - init: - commit-sha: ${{ event.git.sha }} - branch: ${{ event.git.branch }} - cli: - init: - commit-sha: ${{ event.git.sha }} - branch: local - -base: - image: ubuntu:24.04 - config: rwx/base 1.0.0 - -concurrency-pools: - - id: pgdogdev/pgdog:integration:${{ init.branch }} - if: ${{ init.branch != "main" && init.branch != "local" }} - capacity: 1 - on-overflow: cancel-running - -aliases: - build-filter: &build-filter - - Cargo.toml - - Cargo.lock - - docker-compose.yml - - examples/demo/** - - integration/** - - pgdog/** - - pgdog-config/** - - pgdog-macros/** - - pgdog-plugin/** - - pgdog-postgres-types/** - - pgdog-stats/** - - pgdog-vector/** - - plugins/** - - scripts/** - postgres-bg-processes: &postgres-bg–processes - - key: postgres - run: | - sudo service postgresql start - sudo -u postgres createuser --superuser --login "$USER" || true - sudo -u postgres createdb "$USER" || true - sudo -u postgres psql -c 'ALTER SYSTEM SET wal_level TO logical;' - sudo -u postgres psql -c 'ALTER SYSTEM SET max_prepared_transactions TO 1000;' - sudo -u postgres psql -c 'ALTER SYSTEM SET max_connections TO 1000;' - sudo -u postgres psql -c 'ALTER SYSTEM SET max_worker_processes TO 64'; - sudo -u postgres psql -c 'ALTER SYSTEM SET max_wal_senders TO 32'; - sudo -u postgres psql -c 'ALTER SYSTEM SET max_replication_slots TO 32'; - sudo service postgresql restart - touch .pg_isready - ready-check: | - [ -f .pg_isready ] && pg_isready -U "$USER" -h 127.0.0.1 - -tasks: - - key: code - call: git/clone 2.0.7 - with: - repository: https://github.com/pgdogdev/pgdog.git - ref: ${{ init.commit-sha }} - submodules: false - preserve-git-dir: true - - - key: rust - call: rust-lang/install 1.0.6 - with: - rust-version: 1.95.0 - - - key: integration-node - call: nodejs/install 1.1.13 - with: - node-version: "20.11.1" - - - key: integration-system-deps - run: | - sudo apt-get update - if [ -e /usr/bin/pg_config.libpq-dev ] && ! sudo dpkg-divert --list /usr/bin/pg_config >/dev/null 2>&1; then - sudo rm -f /usr/bin/pg_config.libpq-dev - fi - sudo apt-get install -y \ - build-essential \ - clang \ - curl \ - default-jdk \ - golang-go \ - lcov \ - libpq-dev \ - libpq5 \ - libssl-dev \ - libyaml-dev \ - lsof \ - mold \ - php-cli \ - php-pgsql \ - pkg-config \ - postgresql \ - postgresql-client \ - psmisc \ - python3 \ - python3-pip \ - python3-venv \ - python3-virtualenv \ - ruby-full - sudo install -d /usr/share/keyrings - curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo gpg --dearmor -o /usr/share/keyrings/postgresql.gpg - echo "deb [signed-by=/usr/share/keyrings/postgresql.gpg] https://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "${VERSION_CODENAME}")-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null - sudo apt-get update - sudo apt-get install -y postgresql-client-18 - sudo apt-get remove -y cmake || true - sudo pip3 install --break-system-packages cmake==3.31.6 - cmake --version - sudo gem install bundler - sudo curl -SL https://github.com/docker/compose/releases/download/v2.36.1/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose - sudo apt-get clean - cache: - ttl: 7 days - - - key: cargo-nextest - use: [integration-system-deps, rust] - run: cargo install cargo-nextest --version "0.9.78" --locked - - - key: cargo-llvm-cov - use: [cargo-nextest] - run: cargo install cargo-llvm-cov --locked --version "0.6.10" - - - key: codecov-cli - use: integration-system-deps - run: | - curl -fsSL -o codecov https://cli.codecov.io/v11.2.8/linux/codecov - chmod +x codecov - - - key: integration-build-pgdog-cov - use: [code, rust, integration-system-deps, cargo-nextest, cargo-llvm-cov] - agent: - cpus: 16 - memory: 32gb - tmpfs: true - env: - RUSTFLAGS: "-C link-dead-code" - filter: *build-filter - run: | - cargo llvm-cov clean --workspace - mkdir -p target/llvm-cov-target/profiles - cargo llvm-cov run --no-report --release --package pgdog --bin pgdog -- --help - rm -f target/llvm-cov-target/profiles/*.profraw - rm -f target/llvm-cov-target/profiles/.last_snapshot - rm -rf target/llvm-cov-target/reports - - BIN_PATH=$(find target/llvm-cov-target -type f -path '*/release/pgdog' | head -n 1) - if [ -z "$BIN_PATH" ]; then - echo "Instrumented PgDog binary not found" >&2 - exit 1 - fi - - printf '%s\n' "$(realpath "$BIN_PATH")" > "$RWX_ENV/PGDOG_BIN" - - - key: integration-pgbench - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - timeout: 15m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/pgbench-%p-%m.profraw" - - bash integration/setup.sh - - source integration/common.sh - run_pgdog "$PWD/integration" - wait_for_pgdog - psql "postgres://admin:pgdog@127.0.0.1:6432/admin" -c "SET query_timeout TO 10000" - timeout --signal=TERM --kill-after=30s 10m bash integration/pgbench/dev.sh - timeout --signal=TERM --kill-after=30s 10m bash integration/pgbench/stress.sh - stop_pgdog - - cargo llvm-cov report --release --package pgdog --lcov --output-path pgbench.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-pgbench - path: pgbench.lcov - - - key: integration-schema-sync - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/schema-sync-%p-%m.profraw" - - bash integration/setup.sh - timeout --signal=TERM --kill-after=30s 8m bash integration/schema_sync/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path schema-sync.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-schema-sync - path: schema-sync.lcov - - - key: integration-go - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - timeout: 15m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/go-%p-%m.profraw" - - max_attempts=2 - attempt=1 - while true; do - echo "integration-go attempt ${attempt}/${max_attempts}" - bash integration/setup.sh - if timeout --signal=TERM --kill-after=30s 13m bash integration/go/run.sh; then - break - fi - if [ "${attempt}" -ge "${max_attempts}" ]; then - exit 1 - fi - attempt=$((attempt + 1)) - sleep 5 - done - - cargo llvm-cov report --release --package pgdog --lcov --output-path go.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-go - path: go.lcov - - - key: integration-js - use: [integration-build-pgdog-cov, integration-node] - background-processes: *postgres-bg–processes - timeout: 10m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/js-%p-%m.profraw" - - bash integration/setup.sh - - pushd integration/js/pg_tests - npm pkg set type=module - popd - timeout --signal=TERM --kill-after=30s 6m bash integration/js/pg_tests/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path js.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-js - path: js.lcov - - - key: integration-ruby - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - timeout: 10m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/ruby-%p-%m.profraw" - - cleanup() { - killall -TERM toxiproxy-server 2>/dev/null || true - killall -TERM pgdog 2>/dev/null || true - sleep 1 - killall -KILL toxiproxy-server 2>/dev/null || true - killall -KILL pgdog 2>/dev/null || true - } - trap cleanup EXIT - - bash integration/setup.sh - bash integration/toxi/setup.sh - - export PGDOG_KEEP_RUNNING=1 - timeout --signal=TERM --kill-after=30s 6m bash integration/ruby/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path ruby.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-ruby - path: ruby.lcov - - - key: integration-java - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - timeout: 10m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/java-%p-%m.profraw" - - bash integration/setup.sh - timeout --signal=TERM --kill-after=30s 6m bash integration/java/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path java.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-java - path: java.lcov - - - key: integration-mirror - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - timeout: 10m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/mirror-%p-%m.profraw" - - bash integration/setup.sh - timeout --signal=TERM --kill-after=30s 6m bash integration/mirror/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path mirror.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-mirror - path: mirror.lcov - - - key: integration-sql - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - timeout: 10m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/sql-%p-%m.profraw" - - bash integration/setup.sh - timeout --signal=TERM --kill-after=30s 8m bash integration/sql/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path sql.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-sql - path: sql.lcov - - - key: integration-toxi - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - timeout: 12m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/toxi-%p-%m.profraw" - - cleanup() { - killall -TERM toxiproxy-server 2>/dev/null || true - killall -TERM pgdog 2>/dev/null || true - sleep 1 - killall -KILL toxiproxy-server 2>/dev/null || true - killall -KILL pgdog 2>/dev/null || true - } - trap cleanup EXIT - - bash integration/setup.sh - bash integration/toxi/setup.sh - - TOXI_LOG=/tmp/integration-toxi.log - : > "$TOXI_LOG" - - set +e - timeout --signal=TERM --kill-after=30s 8m bash integration/toxi/run.sh >"$TOXI_LOG" 2>&1 & - suite_pid=$! - - while kill -0 "$suite_pid" 2>/dev/null; do - if grep -Eq '[0-9]+ examples, 0 failures' "$TOXI_LOG"; then - killall -TERM pgdog 2>/dev/null || true - killall -TERM toxiproxy-server 2>/dev/null || true - sleep 2 - killall -KILL pgdog 2>/dev/null || true - killall -KILL toxiproxy-server 2>/dev/null || true - break - fi - sleep 1 - done - - wait "$suite_pid" - suite_status=$? - cat "$TOXI_LOG" - - if [ "$suite_status" -eq 124 ] && grep -Eq '[0-9]+ examples, 0 failures' "$TOXI_LOG"; then - suite_status=0 - fi - set -e - - if [ "$suite_status" -ne 0 ]; then - exit "$suite_status" - fi - - cargo llvm-cov report --release --package pgdog --lcov --output-path toxi.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-toxi - path: toxi.lcov - - - key: integration-rust - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - timeout: 20m - agent: - cpus: 16 - memory: 32gb - tmpfs: true - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/rust-%p-%m.profraw" - - cleanup() { - killall -TERM toxiproxy-server 2>/dev/null || true - killall -TERM pgdog 2>/dev/null || true - sleep 1 - killall -KILL toxiproxy-server 2>/dev/null || true - killall -KILL pgdog 2>/dev/null || true - } - trap cleanup EXIT - - bash integration/setup.sh - bash integration/toxi/setup.sh - - export PGDOG_KEEP_RUNNING=1 - timeout --signal=TERM --kill-after=30s 10m bash integration/rust/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path rust.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-rust - path: rust.lcov - - - key: integration-copy-data - use: integration-build-pgdog-cov - docker: true - background-processes: *postgres-bg–processes - timeout: 20m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/copy-data-%p-%m.profraw" - - bash integration/setup.sh - timeout --signal=TERM --kill-after=30s 18m bash integration/copy_data/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path copy-data.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-copy-data - path: copy-data.lcov - - - key: integration-resharding - use: integration-build-pgdog-cov - docker: true - timeout: 20m - agent: - cpus: 4 - memory: 16gb - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/resharding-%p-%m.profraw" - - bash integration/resharding/run.sh - cargo llvm-cov report --release --package pgdog --lcov --output-path resharding.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-resharding - path: resharding.lcov - - - key: integration-python - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - timeout: 12m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/python-%p-%m.profraw" - - bash integration/setup.sh - timeout --signal=TERM --kill-after=30s 10m bash integration/python/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path python.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-python - path: python.lcov - - - key: integration-load-balancer - use: integration-build-pgdog-cov - docker: true - timeout: 20m - agent: - cpus: 4 - memory: 16gb - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/load-balancer-%p-%m.profraw" - - cleanup() { - (cd integration/load_balancer && docker-compose down >/dev/null 2>&1 || true) - killall -TERM pgdog 2>/dev/null || true - sleep 1 - killall -KILL pgdog 2>/dev/null || true - } - trap cleanup EXIT - - timeout --signal=TERM --kill-after=30s 16m bash integration/load_balancer/run.sh - cargo llvm-cov report --release --package pgdog --lcov --output-path load-balancer.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-load-balancer - path: load-balancer.lcov - - - key: integration-complex - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - timeout: 15m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/complex-%p-%m.profraw" - - bash integration/setup.sh - - pushd integration/python - virtualenv venv - source venv/bin/activate - pip install -r requirements.txt - deactivate - popd - - timeout --signal=TERM --kill-after=30s 10m bash integration/complex/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path complex.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-complex - path: complex.lcov - - - key: integration-dry-run - use: [integration-build-pgdog-cov, integration-node] - background-processes: *postgres-bg–processes - timeout: 10m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/dry-run-%p-%m.profraw" - - bash integration/setup.sh - - pushd integration/dry_run - npm pkg set type=module - popd - timeout --signal=TERM --kill-after=30s 8m bash integration/dry_run/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path dry-run.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-dry-run - path: dry-run.lcov - - - key: integration-lcov - use: integration-system-deps - run: | - lcov \ - -a "$LCOV_PGBENCH" \ - -a "$LCOV_SCHEMA_SYNC" \ - -a "$LCOV_GO" \ - -a "$LCOV_JS" \ - -a "$LCOV_RUBY" \ - -a "$LCOV_JAVA" \ - -a "$LCOV_MIRROR" \ - -a "$LCOV_SQL" \ - -a "$LCOV_TOXI" \ - -a "$LCOV_RUST" \ - -a "$LCOV_COPY_DATA" \ - -a "$LCOV_RESHARDING" \ - -a "$LCOV_PYTHON" \ - -a "$LCOV_LOAD_BALANCER" \ - -a "$LCOV_COMPLEX" \ - -a "$LCOV_DRY_RUN" \ - -o integration.lcov - env: - LCOV_PGBENCH: ${{ tasks.integration-pgbench.artifacts.lcov-pgbench }} - LCOV_SCHEMA_SYNC: ${{ tasks.integration-schema-sync.artifacts.lcov-schema-sync }} - LCOV_GO: ${{ tasks.integration-go.artifacts.lcov-go }} - LCOV_JS: ${{ tasks.integration-js.artifacts.lcov-js }} - LCOV_RUBY: ${{ tasks.integration-ruby.artifacts.lcov-ruby }} - LCOV_JAVA: ${{ tasks.integration-java.artifacts.lcov-java }} - LCOV_MIRROR: ${{ tasks.integration-mirror.artifacts.lcov-mirror }} - LCOV_SQL: ${{ tasks.integration-sql.artifacts.lcov-sql }} - LCOV_TOXI: ${{ tasks.integration-toxi.artifacts.lcov-toxi }} - LCOV_RUST: ${{ tasks.integration-rust.artifacts.lcov-rust }} - LCOV_COPY_DATA: ${{ tasks.integration-copy-data.artifacts.lcov-copy-data }} - LCOV_RESHARDING: ${{ tasks.integration-resharding.artifacts.lcov-resharding }} - LCOV_PYTHON: ${{ tasks.integration-python.artifacts.lcov-python }} - LCOV_LOAD_BALANCER: ${{ tasks.integration-load-balancer.artifacts.lcov-load-balancer }} - LCOV_COMPLEX: ${{ tasks.integration-complex.artifacts.lcov-complex }} - LCOV_DRY_RUN: ${{ tasks.integration-dry-run.artifacts.lcov-dry-run }} - outputs: - artifacts: - - key: integration-lcov - path: integration.lcov - - - key: integration-codecov-upload - use: [code, codecov-cli] - env: - CODECOV_TOKEN: ${{ vaults.pgdog-development.secrets.CODECOV_TOKEN }} - LCOV_FILE: ${{ tasks.integration-lcov.artifacts.integration-lcov }} - run: | - ./codecov --verbose upload-process \ - --disable-search \ - --fail-on-error \ - -F integration \ - -f "$LCOV_FILE" - outputs: - filesystem: false diff --git a/.rwx/rust.yml b/.rwx/rust.yml deleted file mode 100644 index fb298e523..000000000 --- a/.rwx/rust.yml +++ /dev/null @@ -1,349 +0,0 @@ -on: - github: - pull_request: - init: - commit-sha: ${{ event.git.sha }} - branch: ${{ event.git.branch }} - push: - if: ${{ event.git.branch == "main" }} - init: - commit-sha: ${{ event.git.sha }} - branch: ${{ event.git.branch }} - cli: - init: - commit-sha: ${{ event.git.sha }} - branch: local - -base: - image: ubuntu:24.04 - config: rwx/base 1.0.0 - -concurrency-pools: - - id: pgdogdev/pgdog:rust:${{ init.branch }} - if: ${{ init.branch != "main" && init.branch != "local" }} - capacity: 1 - on-overflow: cancel-running - -aliases: - shared-filter: &shared-filter - - "!.claude" - - "!.git" - - "!.gitattributes" - - "!.github" - - "!.gitignore" - - "!.rwx" - - "!AGENTS.md" - - "!CLAUDE.md" - - "!CONTRIBUTING.md" - - "!LICENSE" - - "!README.md" - - "!SECURITY.md" - - "!docs" - -tasks: - - key: code - call: git/clone 2.0.7 - with: - repository: https://github.com/pgdogdev/pgdog.git - ref: ${{ init.commit-sha }} - preserve-git-dir: true - submodules: false - - - key: rust - call: rust-lang/install 1.0.6 - with: - rust-version: 1.95.0 - - - key: system-packages - run: | - sudo apt-get update - if [ -e /usr/bin/pg_config.libpq-dev ] && ! sudo dpkg-divert --list /usr/bin/pg_config >/dev/null 2>&1; then - sudo rm -f /usr/bin/pg_config.libpq-dev - fi - sudo apt-get install -y \ - build-essential \ - clang \ - curl \ - lcov \ - libpq-dev \ - libssl-dev \ - mold \ - pkg-config \ - postgresql \ - postgresql-client \ - psmisc \ - python3-pip \ - python3-venv \ - python3-virtualenv \ - ruby-full - sudo apt-get remove -y cmake || true - sudo pip3 install --break-system-packages cmake==3.31.6 - cmake --version - sudo gem install bundler - sudo apt-get clean - cache: - ttl: 7 days - - - key: cargo-nextest - use: [system-packages, rust] - run: cargo install cargo-nextest --version "0.9.78" --locked - - - key: cargo-llvm-cov - use: [cargo-nextest] - run: cargo install cargo-llvm-cov --locked --version "0.6.10" - - - key: codecov-cli - use: system-packages - run: | - curl -fsSL -o codecov https://cli.codecov.io/v11.2.8/linux/codecov - chmod +x codecov - - - key: cargo-build-workspace - use: [system-packages, code, rust, cargo-nextest] - agent: - cpus: 16 - memory: 32gb - tmpfs: true - run: cargo test --workspace --no-run - outputs: - filesystem: - deduplicate: true - filter: *shared-filter - - - key: cargo-build-workspace-cov - use: [system-packages, code, rust, cargo-nextest, cargo-llvm-cov] - agent: - cpus: 16 - memory: 32gb - tmpfs: true - run: cargo llvm-cov nextest --workspace --no-clean -E 'none()' --no-tests=pass - outputs: - filesystem: - deduplicate: true - filter: ["!**/*.profraw"] - filter: *shared-filter - - - key: cargo-test-pgdog - use: cargo-build-workspace-cov - timeout: 20m - run: | - sudo service postgresql start - sudo -u postgres createuser --superuser --login "$USER" || true - sudo -u postgres createdb "$USER" || true - - bash integration/setup.sh || (sudo service postgresql restart && bash integration/setup.sh) - - cargo llvm-cov nextest \ - --workspace \ - --no-clean \ - --lcov \ - --output-path lcov-pgdog.info \ - --no-fail-fast \ - --test-threads=1 \ - -E 'package(pgdog)' - outputs: - filesystem: false - artifacts: - - key: lcov-pgdog - path: lcov-pgdog.info - filter: *shared-filter - - - key: cargo-test-pgdog-config - use: cargo-build-workspace-cov - run: | - cargo llvm-cov nextest \ - --workspace \ - --no-clean \ - --lcov \ - --output-path lcov-pgdog-config.info \ - --no-fail-fast \ - --test-threads=1 \ - -E 'package(pgdog-config)' - outputs: - filesystem: false - artifacts: - - key: lcov-pgdog-config - path: lcov-pgdog-config.info - filter: *shared-filter - - - key: cargo-test-pgdog-vector - use: cargo-build-workspace-cov - run: | - cargo llvm-cov nextest \ - --workspace \ - --no-clean \ - --lcov \ - --output-path lcov-pgdog-vector.info \ - --no-fail-fast \ - --test-threads=1 \ - -E 'package(pgdog-vector)' - outputs: - filesystem: false - artifacts: - - key: lcov-pgdog-vector - path: lcov-pgdog-vector.info - filter: *shared-filter - - - key: cargo-test-pgdog-stats - use: cargo-build-workspace-cov - run: | - cargo llvm-cov nextest \ - --workspace \ - --no-clean \ - --lcov \ - --output-path lcov-pgdog-stats.info \ - --no-fail-fast \ - --test-threads=1 \ - -E 'package(pgdog-stats)' - outputs: - filesystem: false - artifacts: - - key: lcov-pgdog-stats - path: lcov-pgdog-stats.info - filter: *shared-filter - - - key: cargo-test-pgdog-postgres-types - use: cargo-build-workspace-cov - run: | - cargo llvm-cov nextest \ - --workspace \ - --no-clean \ - --lcov \ - --output-path lcov-pgdog-postgres-types.info \ - --no-fail-fast \ - --test-threads=1 \ - -E 'package(pgdog-postgres-types)' - outputs: - filesystem: false - artifacts: - - key: lcov-pgdog-postgres-types - path: lcov-pgdog-postgres-types.info - filter: *shared-filter - - - key: rust-core-lcov - use: system-packages - run: | - lcov \ - -a "$LCOV_PGDOG" \ - -a "$LCOV_PGDOG_CONFIG" \ - -a "$LCOV_PGDOG_VECTOR" \ - -a "$LCOV_PGDOG_STATS" \ - -a "$LCOV_PGDOG_POSTGRES_TYPES" \ - -o lcov.info - env: - LCOV_PGDOG: ${{ tasks.cargo-test-pgdog.artifacts.lcov-pgdog }} - LCOV_PGDOG_CONFIG: ${{ tasks.cargo-test-pgdog-config.artifacts.lcov-pgdog-config }} - LCOV_PGDOG_VECTOR: ${{ tasks.cargo-test-pgdog-vector.artifacts.lcov-pgdog-vector }} - LCOV_PGDOG_STATS: ${{ tasks.cargo-test-pgdog-stats.artifacts.lcov-pgdog-stats }} - LCOV_PGDOG_POSTGRES_TYPES: ${{ tasks.cargo-test-pgdog-postgres-types.artifacts.lcov-pgdog-postgres-types }} - outputs: - artifacts: - - key: core-lcov - path: lcov.info - - - key: rust-codecov-upload - use: [code, codecov-cli] - env: - CODECOV_TOKEN: ${{ vaults.pgdog-development.secrets.CODECOV_TOKEN }} - LCOV_FILE: ${{ tasks.rust-core-lcov.artifacts.core-lcov }} - run: | - ./codecov --verbose upload-process \ - --disable-search \ - --fail-on-error \ - -F unit \ - -f "$LCOV_FILE" - outputs: - filesystem: false - - - key: rust-fmt-check - use: [code, rust] - run: cargo fmt --all -- --check - outputs: - filesystem: false - filter: *shared-filter - - - key: rust-pgdog-clippy - use: [system-packages, code, rust] - agent: - cpus: 4 - run: cd pgdog && cargo clippy - outputs: - filesystem: false - filter: *shared-filter - - - key: rust-workspace-build - use: [system-packages, code, rust] - agent: - cpus: 8 - run: cargo build - outputs: - filesystem: false - filter: *shared-filter - - - key: rust-workspace-check-release - use: [system-packages, code, rust] - agent: - cpus: 4 - run: cargo check --release - outputs: - filesystem: false - filter: *shared-filter - - - key: rust-doc-tests - use: cargo-build-workspace - run: cargo test --doc - outputs: - filesystem: false - filter: *shared-filter - - - key: rust-plugin-unit - use: cargo-build-workspace - timeout: 30m - run: | - set +e - cargo nextest run -E 'package(pgdog-example-plugin)' --no-fail-fast - status=$? - set -e - - if [ "$status" -ne 0 ]; then - printf '%s\n' \ - "pgdog-example-plugin unit tests failed with exit code $status" \ - > "$RWX_WARNINGS/plugin-unit" - fi - - exit 0 - outputs: - filesystem: false - filter: *shared-filter - - - key: rust-plugin-integration - use: [system-packages, code, rust] - agent: - cpus: 8 - memory: 16gb - timeout: 30m - run: | - sudo service postgresql start - sudo -u postgres createuser --superuser --login "$USER" || true - sudo -u postgres createdb "$USER" || true - - bash integration/setup.sh || (sudo service postgresql restart && bash integration/setup.sh) - - cargo build --release -p pgdog --bin pgdog - export PGDOG_BIN="$PWD/target/release/pgdog" - - set +e - timeout --signal=TERM --kill-after=30s 11m bash integration/plugins/run.sh - status=$? - set -e - - if [ "$status" -ne 0 ]; then - printf '%s\n' \ - "pgdog-example-plugin integration tests failed with exit code $status" \ - > "$RWX_WARNINGS/plugin-integration" - fi - - exit 0 - outputs: - filesystem: false - filter: *shared-filter From 53c4d4a712caf211250b7f9c8b2878c4d4b8dc6b Mon Sep 17 00:00:00 2001 From: meskill <8974488+meskill@users.noreply.github.com> Date: Thu, 23 Apr 2026 10:50:32 +0000 Subject: [PATCH 3/3] ci: test coverage --- .github/workflows/ci.yml | 12 ++++++++++-- integration/verify_profiles.sh | 4 ++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e9be8d23..5048ca3bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,14 +170,22 @@ jobs: run: bash integration/complex/run.sh - name: Dry run run: bash integration/dry_run/run.sh - - name: Resharding - run: bash integration/resharding/run.sh - name: Ensure PgDog stopped run: | if pgrep -x pgdog > /dev/null; then killall -TERM pgdog sleep 5 fi + - name: Verify coverage profiles (pgbench) + run: bash integration/verify_profiles.sh pgbench + - name: Verify coverage profiles (go) + run: bash integration/verify_profiles.sh go + - name: Verify coverage profiles (sql) + run: bash integration/verify_profiles.sh sql + - name: Verify coverage profiles (python) + run: bash integration/verify_profiles.sh python + - name: Verify coverage profiles (rust) + run: bash integration/verify_profiles.sh rust - name: Generate integration coverage report run: cargo llvm-cov report --release --package pgdog --lcov --output-path integration.lcov # Requires CODECOV_TOKEN secret for upload diff --git a/integration/verify_profiles.sh b/integration/verify_profiles.sh index f68b66a5a..e10465f8b 100755 --- a/integration/verify_profiles.sh +++ b/integration/verify_profiles.sh @@ -85,8 +85,8 @@ if [ "${CURRENT_COUNT}" = "0" ]; then fi -if [ "${CURRENT_HASH}" = "${PREV_HASH}" ]; then - echo "Note: coverage artifacts unchanged after '${LABEL}'" >&2 +if [ "${CURRENT_HASH}" = "${PREV_HASH}" ] && [ -n "${PREV_HASH}" ]; then + echo "Note: no new coverage profiles generated by '${LABEL}'" >&2 find "${PROFILE_SEARCH_ROOT}" -name '*.profraw' -print || true fi