diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4c57859e..929332ee8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,248 +7,92 @@ on: types: [opened, synchronize, reopened] jobs: - fmt: - runs-on: blacksmith-4vcpu-ubuntu-2404 - steps: - - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - name: Install CMake 3.31 - run: | - sudo apt update && sudo apt install mold -y - sudo apt remove cmake - sudo pip3 install cmake==3.31.6 - cmake --version - - name: Format - run: cargo fmt --all -- --check - - name: Clippy - working-directory: pgdog - run: cargo clippy build: runs-on: blacksmith-4vcpu-ubuntu-2404 steps: - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 + - name: Install CI deps + run: bash integration/ci/install-deps.sh + # Binary cache runs first: its key must be computed against a clean + # tree so it matches what downstream jobs compute. rust-cache runs + # after — it restores target/ which contains generated *.rs under + # target/*/build/*/out/ that would otherwise destabilize the hash. + - name: Compute cache key + id: cache-key + run: echo "key=pgdog-bin-${{ runner.os }}-$(bash integration/ci/cache-key.sh)" >> "$GITHUB_OUTPUT" + - name: Cache pgdog binaries + id: pgdog-bin + uses: useblacksmith/cache@v5 with: - toolchain: stable - override: true - - name: Install CMake 3.31 - run: | - sudo apt update && sudo apt install mold -y - sudo apt remove cmake - sudo pip3 install cmake==3.31.6 - cmake --version - - name: Build - run: cargo build - - name: Check release - run: cargo check --release - tests: - runs-on: blacksmith-4vcpu-ubuntu-2404 - steps: - - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - components: llvm-tools-preview + path: | + target/debug/pgdog + target/release/pgdog + key: ${{ steps.cache-key.outputs.key }} - uses: useblacksmith/rust-cache@v3 + if: steps.pgdog-bin.outputs.cache-hit != 'true' with: - prefix-key: "v1" # Change this when updating tooling - - name: Setup PostgreSQL - run: | - sudo service postgresql start - sudo -u postgres createuser --superuser --login $USER - sudo -u postgres createdb $USER - bash integration/setup.sh || (sudo service postgresql restart && bash integration/setup.sh) - sudo apt update && sudo apt install -y python3-virtualenv mold - sudo gem install bundler - sudo apt remove -y cmake - sudo pip3 install cmake==3.31.6 - cmake --version - cargo install cargo-nextest --version "0.9.78" --locked - bash integration/toxi/setup.sh - - name: Install test dependencies - run: cargo install cargo-nextest --version "0.9.78" --locked - - name: Install coverage tooling - run: cargo install cargo-llvm-cov --locked --version "0.6.10" - - name: Run tests with coverage - env: - RUSTFLAGS: "-C link-dead-code" - run: | - cargo llvm-cov clean --workspace - cargo llvm-cov nextest --lcov --output-path lcov.info --no-fail-fast --test-threads=1 --filter-expr "package(pgdog) | package(pgdog-config) | package(pgdog-vector) | package(pgdog-stats) | package(pgdog-postgres-types)" - - name: Run documentation tests - run: cargo test --doc - # Requires CODECOV_TOKEN secret for upload - - uses: codecov/codecov-action@v4 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - files: lcov.info - flags: unit - fail_ci_if_error: true - integration: + prefix-key: build-v1 + - name: Build (debug) + if: steps.pgdog-bin.outputs.cache-hit != 'true' + run: cargo build --bin pgdog + - name: Build (release) + if: steps.pgdog-bin.outputs.cache-hit != 'true' + run: cargo build --release --bin pgdog + + ci: runs-on: blacksmith-4vcpu-ubuntu-2404 + needs: [build] timeout-minutes: 30 + continue-on-error: ${{ matrix.continue_on_error == true }} + strategy: + fail-fast: false + matrix: + include: + - { name: pgbench, script: integration/pgbench/run.sh } + - { name: schema-sync, script: integration/schema_sync/run.sh } + - { name: go, script: integration/go/run.sh } + - { name: js, script: integration/js/pg_tests/run.sh } + - { name: ruby, script: integration/ruby/run.sh } + - { name: java, script: integration/java/run.sh } + - { name: mirror, script: integration/mirror/run.sh } + - { name: sql, script: integration/sql/run.sh } + - { name: toxi, script: integration/toxi/run.sh } + - { name: rust, script: integration/rust/run.sh } + - { name: python, script: integration/python/run.sh } + - { name: complex, script: integration/complex/run.sh } + - { name: dry-run, script: integration/dry_run/run.sh } + - { name: copy-data, script: integration/copy_data/run.sh } + - { name: load-balancer, script: integration/load_balancer/run.sh } + # plugins/run.sh builds 4 plugin crates with cargo, so it needs + # the workspace target/ cache; the other entries just run the + # cached pgdog binary. + - { name: plugins, script: integration/plugins/run.sh, needs_rust_cache: true, continue_on_error: true } env: - LLVM_PROFILE_FILE: ${{ github.workspace }}/target/llvm-cov-target/profiles/pgdog-%p-%m.profraw + PGDOG_BIN: ${{ github.workspace }}/target/release/pgdog steps: - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 + - name: Install CI deps + run: bash integration/ci/install-deps.sh + - name: Compute cache key + id: cache-key + run: echo "key=pgdog-bin-${{ runner.os }}-$(bash integration/ci/cache-key.sh)" >> "$GITHUB_OUTPUT" + - name: Restore pgdog binaries + uses: useblacksmith/cache/restore@v5 with: - toolchain: stable - override: true - components: llvm-tools-preview - - uses: useblacksmith/rust-cache@v3 + path: | + target/debug/pgdog + target/release/pgdog + key: ${{ steps.cache-key.outputs.key }} + fail-on-cache-miss: true + - name: Restore Rust cache for plugin builds + if: matrix.needs_rust_cache + uses: useblacksmith/rust-cache@v3 with: - prefix-key: release-1 + prefix-key: build-v1 - name: Setup dependencies - run: | - sudo service postgresql start - sudo -u postgres createuser --superuser --login $USER - sudo -u postgres createdb $USER - bash integration/setup.sh || (sudo service postgresql restart && bash integration/setup.sh) - sudo apt update && sudo apt install -y python3-virtualenv mold php-cli php-pgsql curl ca-certificates - # Use a versioned PostgreSQL 18 client so the protocol-version smoke - # test does not depend on whatever libpq the runner image ships. - sudo install -d /usr/share/postgresql-common/pgdg - sudo curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc - . /etc/os-release - sudo sh -c "echo 'deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $VERSION_CODENAME-pgdg main' > /etc/apt/sources.list.d/pgdg.list" - sudo apt update && sudo apt install -y postgresql-client-18 - echo "PSQL_BIN=/usr/lib/postgresql/18/bin/psql" >> "$GITHUB_ENV" - sudo gem install bundler - sudo apt remove -y cmake - sudo pip3 install cmake==3.31.6 - cmake --version - cargo install cargo-nextest --version "0.9.78" --locked - bash integration/toxi/setup.sh - sudo curl -SL https://github.com/docker/compose/releases/download/v2.36.1/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose - - name: Install coverage tooling - run: cargo install cargo-llvm-cov --locked --version "0.6.10" - - name: Prepare instrumented PgDog build - env: - RUSTFLAGS: "-C link-dead-code" - run: | - cargo llvm-cov clean --workspace - mkdir -p target/llvm-cov-target/profiles - cargo llvm-cov run --no-report --release --package pgdog --bin pgdog -- --help - rm -f target/llvm-cov-target/profiles/*.profraw - rm -f target/llvm-cov-target/profiles/.last_snapshot - rm -rf target/llvm-cov-target/reports - BIN_PATH=$(find target/llvm-cov-target -type f -path '*/release/pgdog' | head -n 1) - if [ -z "$BIN_PATH" ]; then - echo "Instrumented PgDog binary not found" >&2 - exit 1 - fi - echo "Using instrumented binary at $BIN_PATH" - echo "PGDOG_BIN=$(realpath "$BIN_PATH")" >> "$GITHUB_ENV" - - name: pgbench - run: bash integration/pgbench/run.sh - - name: schema-sync - run: bash integration/schema_sync/run.sh - - name: Go - run: bash integration/go/run.sh - - name: JavaScript - run: bash integration/js/pg_tests/run.sh - - name: Ruby - run: bash integration/ruby/run.sh - - name: Java - run: bash integration/java/run.sh - - name: Mirror - run: bash integration/mirror/run.sh - - name: SQL - run: bash integration/sql/run.sh - - name: Toxi - run: bash integration/toxi/run.sh - - name: Rust - run: bash integration/rust/run.sh - - name: Stop shared PgDog - run: bash -lc 'source integration/common.sh; stop_pgdog' - - name: Data sync - run: bash integration/copy_data/run.sh - - name: Python - run: bash integration/python/run.sh - - name: Load balancer - run: bash integration/load_balancer/run.sh - - name: More complex stuff - run: bash integration/complex/run.sh - - name: Dry run - run: bash integration/dry_run/run.sh + run: bash integration/ci/setup.sh --with-toxi + - name: Run ${{ matrix.name }} + run: bash ${{ matrix.script }} - name: Ensure PgDog stopped - run: | - if pgrep -x pgdog > /dev/null; then - killall -TERM pgdog - sleep 5 - fi - - name: Generate integration coverage report - run: cargo llvm-cov report --release --package pgdog --lcov --output-path integration.lcov - # Requires CODECOV_TOKEN secret for upload - - uses: codecov/codecov-action@v4 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - files: integration.lcov - flags: integration - fail_ci_if_error: true - plugin-unit-tests: - runs-on: blacksmith-4vcpu-ubuntu-2404 - continue-on-error: true - timeout-minutes: 30 - steps: - - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - uses: useblacksmith/rust-cache@v3 - with: - prefix-key: "plugin-unit-v1" - - name: Install CMake 3.31 - run: | - sudo apt-get install -y mold - sudo apt remove cmake - sudo pip3 install cmake==3.31.6 - cmake --version - - name: Install test dependencies - run: cargo install cargo-nextest --version "0.9.78" --locked - - name: Run plugin unit tests - run: cargo nextest run -E 'package(pgdog-example-plugin)' --no-fail-fast - plugin-integration-tests: - runs-on: blacksmith-4vcpu-ubuntu-2404 - continue-on-error: true - timeout-minutes: 30 - steps: - - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - uses: useblacksmith/rust-cache@v3 - with: - prefix-key: "plugin-integration-v1" - - name: Setup PostgreSQL - run: | - sudo service postgresql start - sudo -u postgres createuser --superuser --login $USER - sudo -u postgres createdb $USER - bash integration/setup.sh || (sudo service postgresql restart && bash integration/setup.sh) - - name: Install dependencies - run: | - sudo apt update && sudo apt install -y python3-virtualenv mold - sudo gem install bundler - sudo apt remove -y cmake - sudo pip3 install cmake==3.31.6 - cmake --version - cargo install cargo-nextest --version "0.9.78" --locked - - name: Build plugin - run: | - cargo build --release - pushd plugins/pgdog-example-plugin - cargo build --release - popd - - name: Run plugin integration tests - run: bash integration/plugins/run.sh + if: always() + run: bash integration/ci/ensure-pgdog-stopped.sh diff --git a/.github/workflows/fmt.yml b/.github/workflows/fmt.yml new file mode 100644 index 000000000..cc9cd10d1 --- /dev/null +++ b/.github/workflows/fmt.yml @@ -0,0 +1,19 @@ +name: fmt +on: + push: + branches: + - main + pull_request: + types: [opened, synchronize, reopened] + +jobs: + fmt: + runs-on: blacksmith-4vcpu-ubuntu-2404 + steps: + - uses: actions/checkout@v4 + - name: Install CI deps + run: bash integration/ci/install-deps.sh + - name: Format + run: cargo fmt --all -- --check + - name: Clippy + run: cargo clippy --all-targets diff --git a/.github/workflows/package-base.yml b/.github/workflows/package-base.yml new file mode 100644 index 000000000..21a677803 --- /dev/null +++ b/.github/workflows/package-base.yml @@ -0,0 +1,118 @@ +name: package-base +on: + workflow_dispatch: + schedule: + # Weekly: Mondays at 06:00 UTC. + - cron: '0 6 * * 1' + +env: + REGISTRY: ghcr.io +jobs: + build: + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - suffix: base-builder + dockerfile: docker/Dockerfile.base-builder + platform: linux/amd64 + runner: blacksmith-4vcpu-ubuntu-2404 + - suffix: base-builder + dockerfile: docker/Dockerfile.base-builder + platform: linux/arm64 + runner: blacksmith-4vcpu-ubuntu-2404-arm + - suffix: base-runtime + dockerfile: docker/Dockerfile.base-runtime + platform: linux/amd64 + runner: blacksmith-4vcpu-ubuntu-2404 + - suffix: base-runtime + dockerfile: docker/Dockerfile.base-runtime + platform: linux/arm64 + runner: blacksmith-4vcpu-ubuntu-2404-arm + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Prepare + run: | + platform='${{ matrix.platform }}' + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push + id: build + uses: docker/build-push-action@v6 + with: + context: . + file: ${{ matrix.dockerfile }} + tags: ${{ env.REGISTRY }}/${{ github.repository }}-${{ matrix.suffix }} + platforms: ${{ matrix.platform }} + outputs: type=image,push-by-digest=true,name-canonical=true,push=true + + - name: Export digest + run: | + mkdir -p ${{ runner.temp }}/digests + digest="${{ steps.build.outputs.digest }}" + touch "${{ runner.temp }}/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.suffix }}-${{ env.PLATFORM_PAIR }} + path: ${{ runner.temp }}/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + runs-on: blacksmith-4vcpu-ubuntu-2404 + needs: + - build + strategy: + fail-fast: false + matrix: + suffix: + - base-builder + - base-runtime + permissions: + contents: read + packages: write + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: ${{ runner.temp }}/digests + pattern: digests-${{ matrix.suffix }}-* + merge-multiple: true + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Create manifest list and push + working-directory: ${{ runner.temp }}/digests + run: | + docker buildx imagetools create -t ${{ env.REGISTRY }}/${{ github.repository }}-${{ matrix.suffix }}:latest \ + $(printf '${{ env.REGISTRY }}/${{ github.repository }}-${{ matrix.suffix }}@sha256:%s ' *) + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ github.repository }}-${{ matrix.suffix }}:latest diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 1ff7523b0..87333be04 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -9,6 +9,8 @@ on: env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} + BUILDER_BASE: ghcr.io/${{ github.repository }}-base-builder:latest + RUNTIME_BASE: ghcr.io/${{ github.repository }}-base-runtime:latest jobs: build: runs-on: ${{ matrix.runner }} @@ -19,7 +21,7 @@ jobs: - platform: linux/amd64 runner: blacksmith-4vcpu-ubuntu-2404 - platform: linux/arm64 - runner: ubuntu-24.04-arm + runner: blacksmith-4vcpu-ubuntu-2404-arm permissions: contents: read packages: write @@ -62,6 +64,9 @@ jobs: tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} labels: ${{ steps.meta.outputs.labels }} platforms: ${{ matrix.platform }} + build-args: | + BUILDER_BASE=${{ env.BUILDER_BASE }} + RUNTIME_BASE=${{ env.RUNTIME_BASE }} outputs: type=image,push-by-digest=true,name-canonical=true,push=true - name: Export digest diff --git a/.github/workflows/plugin-ci.yml b/.github/workflows/plugin-ci.yml new file mode 100644 index 000000000..c69096ae8 --- /dev/null +++ b/.github/workflows/plugin-ci.yml @@ -0,0 +1,22 @@ +name: plugin-ci +on: + push: + branches: + - main + pull_request: + types: [opened, synchronize, reopened] + +jobs: + plugin-unit-tests: + runs-on: blacksmith-4vcpu-ubuntu-2404 + continue-on-error: true + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - name: Install CI deps + run: bash integration/ci/install-deps.sh + - uses: useblacksmith/rust-cache@v3 + with: + prefix-key: "plugin-unit-v1" + - name: Run plugin unit tests + run: cargo nextest run -E 'package(pgdog-example-plugin)' --no-fail-fast diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 000000000..c345c46f9 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,36 @@ +name: tests +on: + push: + branches: + - main + pull_request: + types: [opened, synchronize, reopened] + +jobs: + tests: + runs-on: blacksmith-4vcpu-ubuntu-2404 + steps: + - uses: actions/checkout@v4 + - name: Install CI deps + run: bash integration/ci/install-deps.sh + - uses: useblacksmith/rust-cache@v3 + with: + prefix-key: "v1" # Change this when updating tooling + - name: Setup PostgreSQL + run: bash integration/ci/setup.sh --with-toxi + - name: Run tests with coverage + env: + RUSTFLAGS: "-C link-dead-code" + run: | + cargo llvm-cov clean --workspace + cargo llvm-cov nextest --lcov --output-path lcov.info --no-fail-fast --test-threads=1 --filter-expr "package(pgdog) | package(pgdog-config) | package(pgdog-vector) | package(pgdog-stats) | package(pgdog-postgres-types)" + - name: Run documentation tests + run: cargo test --doc + # Requires CODECOV_TOKEN secret for upload + - uses: codecov/codecov-action@v4 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + files: lcov.info + flags: unit + fail_ci_if_error: true diff --git a/.rwx/integration.yml b/.rwx/integration.yml deleted file mode 100644 index 8051c7704..000000000 --- a/.rwx/integration.yml +++ /dev/null @@ -1,613 +0,0 @@ -on: - github: - pull_request: - init: - commit-sha: ${{ event.git.sha }} - branch: ${{ event.git.branch }} - push: - if: ${{ event.git.branch == "main" }} - init: - commit-sha: ${{ event.git.sha }} - branch: ${{ event.git.branch }} - cli: - init: - commit-sha: ${{ event.git.sha }} - branch: local - -base: - image: ubuntu:24.04 - config: rwx/base 1.0.0 - -concurrency-pools: - - id: pgdogdev/pgdog:integration:${{ init.branch }} - if: ${{ init.branch != "main" && init.branch != "local" }} - capacity: 1 - on-overflow: cancel-running - -aliases: - build-filter: &build-filter - - Cargo.toml - - Cargo.lock - - docker-compose.yml - - examples/demo/** - - integration/** - - pgdog/** - - pgdog-config/** - - pgdog-macros/** - - pgdog-plugin/** - - pgdog-postgres-types/** - - pgdog-stats/** - - pgdog-vector/** - - plugins/** - - scripts/** - postgres-bg-processes: &postgres-bg–processes - - key: postgres - run: | - sudo service postgresql start - sudo -u postgres createuser --superuser --login "$USER" || true - sudo -u postgres createdb "$USER" || true - sudo -u postgres psql -c 'ALTER SYSTEM SET wal_level TO logical;' - sudo -u postgres psql -c 'ALTER SYSTEM SET max_prepared_transactions TO 1000;' - sudo -u postgres psql -c 'ALTER SYSTEM SET max_connections TO 1000;' - sudo -u postgres psql -c 'ALTER SYSTEM SET max_worker_processes TO 64'; - sudo -u postgres psql -c 'ALTER SYSTEM SET max_wal_senders TO 32'; - sudo -u postgres psql -c 'ALTER SYSTEM SET max_replication_slots TO 32'; - sudo service postgresql restart - touch .pg_isready - ready-check: | - [ -f .pg_isready ] && pg_isready -U "$USER" -h 127.0.0.1 - -tasks: - - key: code - call: git/clone 2.0.7 - with: - repository: https://github.com/pgdogdev/pgdog.git - ref: ${{ init.commit-sha }} - submodules: false - preserve-git-dir: true - - - key: rust - call: rust-lang/install 1.0.6 - with: - rust-version: 1.95.0 - - - key: integration-node - call: nodejs/install 1.1.13 - with: - node-version: "20.11.1" - - - key: integration-system-deps - run: | - sudo apt-get update - if [ -e /usr/bin/pg_config.libpq-dev ] && ! sudo dpkg-divert --list /usr/bin/pg_config >/dev/null 2>&1; then - sudo rm -f /usr/bin/pg_config.libpq-dev - fi - sudo apt-get install -y \ - build-essential \ - clang \ - curl \ - default-jdk \ - golang-go \ - lcov \ - libpq-dev \ - libpq5 \ - libssl-dev \ - libyaml-dev \ - lsof \ - mold \ - php-cli \ - php-pgsql \ - pkg-config \ - postgresql \ - postgresql-client \ - psmisc \ - python3 \ - python3-pip \ - python3-venv \ - python3-virtualenv \ - ruby-full - sudo install -d /usr/share/keyrings - curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo gpg --dearmor -o /usr/share/keyrings/postgresql.gpg - echo "deb [signed-by=/usr/share/keyrings/postgresql.gpg] https://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "${VERSION_CODENAME}")-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null - sudo apt-get update - sudo apt-get install -y postgresql-client-18 - sudo apt-get remove -y cmake || true - sudo pip3 install --break-system-packages cmake==3.31.6 - cmake --version - sudo gem install bundler - sudo curl -SL https://github.com/docker/compose/releases/download/v2.36.1/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose - sudo chmod +x /usr/local/bin/docker-compose - sudo apt-get clean - cache: - ttl: 7 days - - - key: cargo-nextest - use: [integration-system-deps, rust] - run: cargo install cargo-nextest --version "0.9.78" --locked - - - key: cargo-llvm-cov - use: [cargo-nextest] - run: cargo install cargo-llvm-cov --locked --version "0.6.10" - - - key: codecov-cli - use: integration-system-deps - run: | - curl -fsSL -o codecov https://cli.codecov.io/v11.2.8/linux/codecov - chmod +x codecov - - - key: integration-build-pgdog-cov - use: [code, rust, integration-system-deps, cargo-nextest, cargo-llvm-cov] - agent: - cpus: 16 - memory: 32gb - tmpfs: true - env: - RUSTFLAGS: "-C link-dead-code" - filter: *build-filter - run: | - cargo llvm-cov clean --workspace - mkdir -p target/llvm-cov-target/profiles - cargo llvm-cov run --no-report --release --package pgdog --bin pgdog -- --help - rm -f target/llvm-cov-target/profiles/*.profraw - rm -f target/llvm-cov-target/profiles/.last_snapshot - rm -rf target/llvm-cov-target/reports - - BIN_PATH=$(find target/llvm-cov-target -type f -path '*/release/pgdog' | head -n 1) - if [ -z "$BIN_PATH" ]; then - echo "Instrumented PgDog binary not found" >&2 - exit 1 - fi - - printf '%s\n' "$(realpath "$BIN_PATH")" > "$RWX_ENV/PGDOG_BIN" - - - key: integration-pgbench - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - timeout: 15m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/pgbench-%p-%m.profraw" - - bash integration/setup.sh - - source integration/common.sh - run_pgdog "$PWD/integration" - wait_for_pgdog - psql "postgres://admin:pgdog@127.0.0.1:6432/admin" -c "SET query_timeout TO 10000" - timeout --signal=TERM --kill-after=30s 10m bash integration/pgbench/dev.sh - timeout --signal=TERM --kill-after=30s 10m bash integration/pgbench/stress.sh - stop_pgdog - - cargo llvm-cov report --release --package pgdog --lcov --output-path pgbench.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-pgbench - path: pgbench.lcov - - - key: integration-schema-sync - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/schema-sync-%p-%m.profraw" - - bash integration/setup.sh - timeout --signal=TERM --kill-after=30s 8m bash integration/schema_sync/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path schema-sync.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-schema-sync - path: schema-sync.lcov - - - key: integration-go - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - timeout: 15m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/go-%p-%m.profraw" - - max_attempts=2 - attempt=1 - while true; do - echo "integration-go attempt ${attempt}/${max_attempts}" - bash integration/setup.sh - if timeout --signal=TERM --kill-after=30s 13m bash integration/go/run.sh; then - break - fi - if [ "${attempt}" -ge "${max_attempts}" ]; then - exit 1 - fi - attempt=$((attempt + 1)) - sleep 5 - done - - cargo llvm-cov report --release --package pgdog --lcov --output-path go.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-go - path: go.lcov - - - key: integration-js - use: [integration-build-pgdog-cov, integration-node] - background-processes: *postgres-bg–processes - timeout: 10m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/js-%p-%m.profraw" - - bash integration/setup.sh - - pushd integration/js/pg_tests - npm pkg set type=module - popd - timeout --signal=TERM --kill-after=30s 6m bash integration/js/pg_tests/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path js.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-js - path: js.lcov - - - key: integration-ruby - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - timeout: 10m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/ruby-%p-%m.profraw" - - cleanup() { - killall -TERM toxiproxy-server 2>/dev/null || true - killall -TERM pgdog 2>/dev/null || true - sleep 1 - killall -KILL toxiproxy-server 2>/dev/null || true - killall -KILL pgdog 2>/dev/null || true - } - trap cleanup EXIT - - bash integration/setup.sh - bash integration/toxi/setup.sh - - export PGDOG_KEEP_RUNNING=1 - timeout --signal=TERM --kill-after=30s 6m bash integration/ruby/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path ruby.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-ruby - path: ruby.lcov - - - key: integration-java - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - timeout: 10m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/java-%p-%m.profraw" - - bash integration/setup.sh - timeout --signal=TERM --kill-after=30s 6m bash integration/java/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path java.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-java - path: java.lcov - - - key: integration-mirror - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - timeout: 10m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/mirror-%p-%m.profraw" - - bash integration/setup.sh - timeout --signal=TERM --kill-after=30s 6m bash integration/mirror/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path mirror.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-mirror - path: mirror.lcov - - - key: integration-sql - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - timeout: 10m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/sql-%p-%m.profraw" - - bash integration/setup.sh - timeout --signal=TERM --kill-after=30s 8m bash integration/sql/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path sql.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-sql - path: sql.lcov - - - key: integration-toxi - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - timeout: 12m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/toxi-%p-%m.profraw" - - cleanup() { - killall -TERM toxiproxy-server 2>/dev/null || true - killall -TERM pgdog 2>/dev/null || true - sleep 1 - killall -KILL toxiproxy-server 2>/dev/null || true - killall -KILL pgdog 2>/dev/null || true - } - trap cleanup EXIT - - bash integration/setup.sh - bash integration/toxi/setup.sh - - TOXI_LOG=/tmp/integration-toxi.log - : > "$TOXI_LOG" - - set +e - timeout --signal=TERM --kill-after=30s 8m bash integration/toxi/run.sh >"$TOXI_LOG" 2>&1 & - suite_pid=$! - - while kill -0 "$suite_pid" 2>/dev/null; do - if grep -Eq '[0-9]+ examples, 0 failures' "$TOXI_LOG"; then - killall -TERM pgdog 2>/dev/null || true - killall -TERM toxiproxy-server 2>/dev/null || true - sleep 2 - killall -KILL pgdog 2>/dev/null || true - killall -KILL toxiproxy-server 2>/dev/null || true - break - fi - sleep 1 - done - - wait "$suite_pid" - suite_status=$? - cat "$TOXI_LOG" - - if [ "$suite_status" -eq 124 ] && grep -Eq '[0-9]+ examples, 0 failures' "$TOXI_LOG"; then - suite_status=0 - fi - set -e - - if [ "$suite_status" -ne 0 ]; then - exit "$suite_status" - fi - - cargo llvm-cov report --release --package pgdog --lcov --output-path toxi.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-toxi - path: toxi.lcov - - - key: integration-rust - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - timeout: 20m - agent: - cpus: 16 - memory: 32gb - tmpfs: true - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/rust-%p-%m.profraw" - - cleanup() { - killall -TERM toxiproxy-server 2>/dev/null || true - killall -TERM pgdog 2>/dev/null || true - sleep 1 - killall -KILL toxiproxy-server 2>/dev/null || true - killall -KILL pgdog 2>/dev/null || true - } - trap cleanup EXIT - - bash integration/setup.sh - bash integration/toxi/setup.sh - - export PGDOG_KEEP_RUNNING=1 - timeout --signal=TERM --kill-after=30s 10m bash integration/rust/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path rust.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-rust - path: rust.lcov - - - key: integration-copy-data - use: integration-build-pgdog-cov - docker: true - background-processes: *postgres-bg–processes - timeout: 20m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/copy-data-%p-%m.profraw" - - bash integration/setup.sh - timeout --signal=TERM --kill-after=30s 18m bash integration/copy_data/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path copy-data.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-copy-data - path: copy-data.lcov - - - key: integration-resharding - use: integration-build-pgdog-cov - docker: true - timeout: 20m - agent: - cpus: 4 - memory: 16gb - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/resharding-%p-%m.profraw" - - cleanup() { - (cd integration/resharding && docker-compose down >/dev/null 2>&1 || true) - killall -TERM pgdog 2>/dev/null || true - sleep 1 - killall -KILL pgdog 2>/dev/null || true - } - trap cleanup EXIT - - timeout --signal=TERM --kill-after=90s 16m bash integration/resharding/dev.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path resharding.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-resharding - path: resharding.lcov - - - key: integration-python - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - timeout: 12m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/python-%p-%m.profraw" - - bash integration/setup.sh - timeout --signal=TERM --kill-after=30s 10m bash integration/python/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path python.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-python - path: python.lcov - - - key: integration-load-balancer - use: integration-build-pgdog-cov - docker: true - timeout: 20m - agent: - cpus: 4 - memory: 16gb - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/load-balancer-%p-%m.profraw" - - cleanup() { - (cd integration/load_balancer && docker-compose down >/dev/null 2>&1 || true) - killall -TERM pgdog 2>/dev/null || true - sleep 1 - killall -KILL pgdog 2>/dev/null || true - } - trap cleanup EXIT - - timeout --signal=TERM --kill-after=30s 16m bash integration/load_balancer/run.sh - cargo llvm-cov report --release --package pgdog --lcov --output-path load-balancer.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-load-balancer - path: load-balancer.lcov - - - key: integration-complex - use: integration-build-pgdog-cov - background-processes: *postgres-bg–processes - timeout: 15m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/complex-%p-%m.profraw" - - bash integration/setup.sh - - pushd integration/python - virtualenv venv - source venv/bin/activate - pip install -r requirements.txt - deactivate - popd - - timeout --signal=TERM --kill-after=30s 10m bash integration/complex/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path complex.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-complex - path: complex.lcov - - - key: integration-dry-run - use: [integration-build-pgdog-cov, integration-node] - background-processes: *postgres-bg–processes - timeout: 10m - run: | - export LLVM_PROFILE_FILE="$PWD/target/llvm-cov-target/profiles/dry-run-%p-%m.profraw" - - bash integration/setup.sh - - pushd integration/dry_run - npm pkg set type=module - popd - timeout --signal=TERM --kill-after=30s 8m bash integration/dry_run/run.sh - - cargo llvm-cov report --release --package pgdog --lcov --output-path dry-run.lcov - outputs: - filesystem: false - artifacts: - - key: lcov-dry-run - path: dry-run.lcov - - - key: integration-lcov - use: integration-system-deps - run: | - lcov \ - -a "$LCOV_PGBENCH" \ - -a "$LCOV_SCHEMA_SYNC" \ - -a "$LCOV_GO" \ - -a "$LCOV_JS" \ - -a "$LCOV_RUBY" \ - -a "$LCOV_JAVA" \ - -a "$LCOV_MIRROR" \ - -a "$LCOV_SQL" \ - -a "$LCOV_TOXI" \ - -a "$LCOV_RUST" \ - -a "$LCOV_COPY_DATA" \ - -a "$LCOV_RESHARDING" \ - -a "$LCOV_PYTHON" \ - -a "$LCOV_LOAD_BALANCER" \ - -a "$LCOV_COMPLEX" \ - -a "$LCOV_DRY_RUN" \ - -o integration.lcov - env: - LCOV_PGBENCH: ${{ tasks.integration-pgbench.artifacts.lcov-pgbench }} - LCOV_SCHEMA_SYNC: ${{ tasks.integration-schema-sync.artifacts.lcov-schema-sync }} - LCOV_GO: ${{ tasks.integration-go.artifacts.lcov-go }} - LCOV_JS: ${{ tasks.integration-js.artifacts.lcov-js }} - LCOV_RUBY: ${{ tasks.integration-ruby.artifacts.lcov-ruby }} - LCOV_JAVA: ${{ tasks.integration-java.artifacts.lcov-java }} - LCOV_MIRROR: ${{ tasks.integration-mirror.artifacts.lcov-mirror }} - LCOV_SQL: ${{ tasks.integration-sql.artifacts.lcov-sql }} - LCOV_TOXI: ${{ tasks.integration-toxi.artifacts.lcov-toxi }} - LCOV_RUST: ${{ tasks.integration-rust.artifacts.lcov-rust }} - LCOV_COPY_DATA: ${{ tasks.integration-copy-data.artifacts.lcov-copy-data }} - LCOV_RESHARDING: ${{ tasks.integration-resharding.artifacts.lcov-resharding }} - LCOV_PYTHON: ${{ tasks.integration-python.artifacts.lcov-python }} - LCOV_LOAD_BALANCER: ${{ tasks.integration-load-balancer.artifacts.lcov-load-balancer }} - LCOV_COMPLEX: ${{ tasks.integration-complex.artifacts.lcov-complex }} - LCOV_DRY_RUN: ${{ tasks.integration-dry-run.artifacts.lcov-dry-run }} - outputs: - artifacts: - - key: integration-lcov - path: integration.lcov - - - key: integration-codecov-upload - use: [code, codecov-cli] - env: - CODECOV_TOKEN: ${{ vaults.pgdog-development.secrets.CODECOV_TOKEN }} - LCOV_FILE: ${{ tasks.integration-lcov.artifacts.integration-lcov }} - run: | - ./codecov --verbose upload-process \ - --disable-search \ - --fail-on-error \ - -F integration \ - -f "$LCOV_FILE" - outputs: - filesystem: false diff --git a/.rwx/rust.yml b/.rwx/rust.yml deleted file mode 100644 index fb298e523..000000000 --- a/.rwx/rust.yml +++ /dev/null @@ -1,349 +0,0 @@ -on: - github: - pull_request: - init: - commit-sha: ${{ event.git.sha }} - branch: ${{ event.git.branch }} - push: - if: ${{ event.git.branch == "main" }} - init: - commit-sha: ${{ event.git.sha }} - branch: ${{ event.git.branch }} - cli: - init: - commit-sha: ${{ event.git.sha }} - branch: local - -base: - image: ubuntu:24.04 - config: rwx/base 1.0.0 - -concurrency-pools: - - id: pgdogdev/pgdog:rust:${{ init.branch }} - if: ${{ init.branch != "main" && init.branch != "local" }} - capacity: 1 - on-overflow: cancel-running - -aliases: - shared-filter: &shared-filter - - "!.claude" - - "!.git" - - "!.gitattributes" - - "!.github" - - "!.gitignore" - - "!.rwx" - - "!AGENTS.md" - - "!CLAUDE.md" - - "!CONTRIBUTING.md" - - "!LICENSE" - - "!README.md" - - "!SECURITY.md" - - "!docs" - -tasks: - - key: code - call: git/clone 2.0.7 - with: - repository: https://github.com/pgdogdev/pgdog.git - ref: ${{ init.commit-sha }} - preserve-git-dir: true - submodules: false - - - key: rust - call: rust-lang/install 1.0.6 - with: - rust-version: 1.95.0 - - - key: system-packages - run: | - sudo apt-get update - if [ -e /usr/bin/pg_config.libpq-dev ] && ! sudo dpkg-divert --list /usr/bin/pg_config >/dev/null 2>&1; then - sudo rm -f /usr/bin/pg_config.libpq-dev - fi - sudo apt-get install -y \ - build-essential \ - clang \ - curl \ - lcov \ - libpq-dev \ - libssl-dev \ - mold \ - pkg-config \ - postgresql \ - postgresql-client \ - psmisc \ - python3-pip \ - python3-venv \ - python3-virtualenv \ - ruby-full - sudo apt-get remove -y cmake || true - sudo pip3 install --break-system-packages cmake==3.31.6 - cmake --version - sudo gem install bundler - sudo apt-get clean - cache: - ttl: 7 days - - - key: cargo-nextest - use: [system-packages, rust] - run: cargo install cargo-nextest --version "0.9.78" --locked - - - key: cargo-llvm-cov - use: [cargo-nextest] - run: cargo install cargo-llvm-cov --locked --version "0.6.10" - - - key: codecov-cli - use: system-packages - run: | - curl -fsSL -o codecov https://cli.codecov.io/v11.2.8/linux/codecov - chmod +x codecov - - - key: cargo-build-workspace - use: [system-packages, code, rust, cargo-nextest] - agent: - cpus: 16 - memory: 32gb - tmpfs: true - run: cargo test --workspace --no-run - outputs: - filesystem: - deduplicate: true - filter: *shared-filter - - - key: cargo-build-workspace-cov - use: [system-packages, code, rust, cargo-nextest, cargo-llvm-cov] - agent: - cpus: 16 - memory: 32gb - tmpfs: true - run: cargo llvm-cov nextest --workspace --no-clean -E 'none()' --no-tests=pass - outputs: - filesystem: - deduplicate: true - filter: ["!**/*.profraw"] - filter: *shared-filter - - - key: cargo-test-pgdog - use: cargo-build-workspace-cov - timeout: 20m - run: | - sudo service postgresql start - sudo -u postgres createuser --superuser --login "$USER" || true - sudo -u postgres createdb "$USER" || true - - bash integration/setup.sh || (sudo service postgresql restart && bash integration/setup.sh) - - cargo llvm-cov nextest \ - --workspace \ - --no-clean \ - --lcov \ - --output-path lcov-pgdog.info \ - --no-fail-fast \ - --test-threads=1 \ - -E 'package(pgdog)' - outputs: - filesystem: false - artifacts: - - key: lcov-pgdog - path: lcov-pgdog.info - filter: *shared-filter - - - key: cargo-test-pgdog-config - use: cargo-build-workspace-cov - run: | - cargo llvm-cov nextest \ - --workspace \ - --no-clean \ - --lcov \ - --output-path lcov-pgdog-config.info \ - --no-fail-fast \ - --test-threads=1 \ - -E 'package(pgdog-config)' - outputs: - filesystem: false - artifacts: - - key: lcov-pgdog-config - path: lcov-pgdog-config.info - filter: *shared-filter - - - key: cargo-test-pgdog-vector - use: cargo-build-workspace-cov - run: | - cargo llvm-cov nextest \ - --workspace \ - --no-clean \ - --lcov \ - --output-path lcov-pgdog-vector.info \ - --no-fail-fast \ - --test-threads=1 \ - -E 'package(pgdog-vector)' - outputs: - filesystem: false - artifacts: - - key: lcov-pgdog-vector - path: lcov-pgdog-vector.info - filter: *shared-filter - - - key: cargo-test-pgdog-stats - use: cargo-build-workspace-cov - run: | - cargo llvm-cov nextest \ - --workspace \ - --no-clean \ - --lcov \ - --output-path lcov-pgdog-stats.info \ - --no-fail-fast \ - --test-threads=1 \ - -E 'package(pgdog-stats)' - outputs: - filesystem: false - artifacts: - - key: lcov-pgdog-stats - path: lcov-pgdog-stats.info - filter: *shared-filter - - - key: cargo-test-pgdog-postgres-types - use: cargo-build-workspace-cov - run: | - cargo llvm-cov nextest \ - --workspace \ - --no-clean \ - --lcov \ - --output-path lcov-pgdog-postgres-types.info \ - --no-fail-fast \ - --test-threads=1 \ - -E 'package(pgdog-postgres-types)' - outputs: - filesystem: false - artifacts: - - key: lcov-pgdog-postgres-types - path: lcov-pgdog-postgres-types.info - filter: *shared-filter - - - key: rust-core-lcov - use: system-packages - run: | - lcov \ - -a "$LCOV_PGDOG" \ - -a "$LCOV_PGDOG_CONFIG" \ - -a "$LCOV_PGDOG_VECTOR" \ - -a "$LCOV_PGDOG_STATS" \ - -a "$LCOV_PGDOG_POSTGRES_TYPES" \ - -o lcov.info - env: - LCOV_PGDOG: ${{ tasks.cargo-test-pgdog.artifacts.lcov-pgdog }} - LCOV_PGDOG_CONFIG: ${{ tasks.cargo-test-pgdog-config.artifacts.lcov-pgdog-config }} - LCOV_PGDOG_VECTOR: ${{ tasks.cargo-test-pgdog-vector.artifacts.lcov-pgdog-vector }} - LCOV_PGDOG_STATS: ${{ tasks.cargo-test-pgdog-stats.artifacts.lcov-pgdog-stats }} - LCOV_PGDOG_POSTGRES_TYPES: ${{ tasks.cargo-test-pgdog-postgres-types.artifacts.lcov-pgdog-postgres-types }} - outputs: - artifacts: - - key: core-lcov - path: lcov.info - - - key: rust-codecov-upload - use: [code, codecov-cli] - env: - CODECOV_TOKEN: ${{ vaults.pgdog-development.secrets.CODECOV_TOKEN }} - LCOV_FILE: ${{ tasks.rust-core-lcov.artifacts.core-lcov }} - run: | - ./codecov --verbose upload-process \ - --disable-search \ - --fail-on-error \ - -F unit \ - -f "$LCOV_FILE" - outputs: - filesystem: false - - - key: rust-fmt-check - use: [code, rust] - run: cargo fmt --all -- --check - outputs: - filesystem: false - filter: *shared-filter - - - key: rust-pgdog-clippy - use: [system-packages, code, rust] - agent: - cpus: 4 - run: cd pgdog && cargo clippy - outputs: - filesystem: false - filter: *shared-filter - - - key: rust-workspace-build - use: [system-packages, code, rust] - agent: - cpus: 8 - run: cargo build - outputs: - filesystem: false - filter: *shared-filter - - - key: rust-workspace-check-release - use: [system-packages, code, rust] - agent: - cpus: 4 - run: cargo check --release - outputs: - filesystem: false - filter: *shared-filter - - - key: rust-doc-tests - use: cargo-build-workspace - run: cargo test --doc - outputs: - filesystem: false - filter: *shared-filter - - - key: rust-plugin-unit - use: cargo-build-workspace - timeout: 30m - run: | - set +e - cargo nextest run -E 'package(pgdog-example-plugin)' --no-fail-fast - status=$? - set -e - - if [ "$status" -ne 0 ]; then - printf '%s\n' \ - "pgdog-example-plugin unit tests failed with exit code $status" \ - > "$RWX_WARNINGS/plugin-unit" - fi - - exit 0 - outputs: - filesystem: false - filter: *shared-filter - - - key: rust-plugin-integration - use: [system-packages, code, rust] - agent: - cpus: 8 - memory: 16gb - timeout: 30m - run: | - sudo service postgresql start - sudo -u postgres createuser --superuser --login "$USER" || true - sudo -u postgres createdb "$USER" || true - - bash integration/setup.sh || (sudo service postgresql restart && bash integration/setup.sh) - - cargo build --release -p pgdog --bin pgdog - export PGDOG_BIN="$PWD/target/release/pgdog" - - set +e - timeout --signal=TERM --kill-after=30s 11m bash integration/plugins/run.sh - status=$? - set -e - - if [ "$status" -ne 0 ]; then - printf '%s\n' \ - "pgdog-example-plugin integration tests failed with exit code $status" \ - > "$RWX_WARNINGS/plugin-integration" - fi - - exit 0 - outputs: - filesystem: false - filter: *shared-filter diff --git a/Dockerfile b/Dockerfile index cf51ae2f2..a73e59d89 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,7 @@ -FROM ubuntu:latest AS builder +ARG BUILDER_BASE=ghcr.io/pgdogdev/pgdog-base-builder:latest +ARG RUNTIME_BASE=ghcr.io/pgdogdev/pgdog-base-runtime:latest -RUN apt update && \ - apt install -y build-essential cmake clang curl pkg-config libssl-dev git mold - -# Install Rust. -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y +FROM ${BUILDER_BASE} AS builder COPY . /build COPY .git /build/.git @@ -20,20 +17,8 @@ RUN source ~/.cargo/env && \ cd .. && \ cargo build --release -p pgdog-primary-only-tables -FROM ubuntu:latest +FROM ${RUNTIME_BASE} ENV RUST_LOG=info -ARG PSQL_VERSION=18 -ENV PSQL_VERSION=${PSQL_VERSION} -RUN apt update && \ - apt install -y curl ca-certificates ssl-cert && \ - update-ca-certificates - -RUN install -d /usr/share/postgresql-common/pgdg && \ - curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc && \ - . /etc/os-release && \ - sh -c "echo 'deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $VERSION_CODENAME-pgdg main' > /etc/apt/sources.list.d/pgdg.list" - -RUN apt update && apt install -y postgresql-client-${PSQL_VERSION} COPY --from=builder /build/target/release/pgdog /usr/local/bin/pgdog COPY --from=builder /build/target/release/libpgdog_primary_only_tables.so /usr/lib/libpgdog_primary_only_tables.so diff --git a/docker/Dockerfile.base-builder b/docker/Dockerfile.base-builder new file mode 100644 index 000000000..93c5935f3 --- /dev/null +++ b/docker/Dockerfile.base-builder @@ -0,0 +1,7 @@ +FROM ubuntu:latest + +RUN apt update && \ + apt install -y build-essential cmake clang curl pkg-config libssl-dev git mold + +# Install Rust. +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y diff --git a/docker/Dockerfile.base-runtime b/docker/Dockerfile.base-runtime new file mode 100644 index 000000000..7ef93c1b7 --- /dev/null +++ b/docker/Dockerfile.base-runtime @@ -0,0 +1,14 @@ +FROM ubuntu:latest +ARG PSQL_VERSION=18 +ENV PSQL_VERSION=${PSQL_VERSION} + +RUN apt update && \ + apt install -y curl ca-certificates ssl-cert && \ + update-ca-certificates + +RUN install -d /usr/share/postgresql-common/pgdg && \ + curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc && \ + . /etc/os-release && \ + sh -c "echo 'deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt $VERSION_CODENAME-pgdg main' > /etc/apt/sources.list.d/pgdg.list" + +RUN apt update && apt install -y postgresql-client-${PSQL_VERSION} diff --git a/integration/ci/apt.sh b/integration/ci/apt.sh new file mode 100755 index 000000000..f3f289ea5 --- /dev/null +++ b/integration/ci/apt.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Thin `apt-get install` wrapper that no-ops on non-Linux platforms (macOS +# developers running these scripts locally already have the deps installed +# via their own package manager). +# +# Usage: integration/ci/apt.sh ... +set -euo pipefail + +if [[ "$(uname -s)" != "Linux" ]]; then + exit 0 +fi + +sudo apt-get install -y --no-install-recommends "$@" diff --git a/integration/ci/cache-key.sh b/integration/ci/cache-key.sh new file mode 100755 index 000000000..0140a721e --- /dev/null +++ b/integration/ci/cache-key.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Print the source hash used to key the pgdog binary cache. Hashes every +# file whose change should invalidate the built `pgdog` binary, and nothing +# else (no `target/`, no examples/tests, no unrelated subcrates). +# +# Keep the file/dir lists in sync with the workspace crates that link into +# `pgdog`. +set -euo pipefail + +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +REPO_ROOT="$( cd "${SCRIPT_DIR}/../.." && pwd )" + +cd "$REPO_ROOT" + +files=( + Cargo.lock + Cargo.toml + .cargo/config.toml + pgdog/Cargo.toml + pgdog/build.rs + pgdog-config/Cargo.toml + pgdog-macros/Cargo.toml + pgdog-plugin/Cargo.toml + pgdog-plugin/build.rs + pgdog-postgres-types/Cargo.toml + pgdog-stats/Cargo.toml + pgdog-vector/Cargo.toml +) + +dirs=( + pgdog/src + pgdog-config/src + pgdog-macros/src + pgdog-plugin/src + pgdog-postgres-types/src + pgdog-stats/src + pgdog-vector/src +) + +{ + for f in "${files[@]}"; do + cksum "$f" + done + for d in "${dirs[@]}"; do + find "$d" -type f -name '*.rs' -print0 | LC_ALL=C sort -z | xargs -0 cksum + done +} | cksum | cut -d' ' -f1 diff --git a/integration/ci/ensure-pgdog-stopped.sh b/integration/ci/ensure-pgdog-stopped.sh new file mode 100755 index 000000000..ed3b46655 --- /dev/null +++ b/integration/ci/ensure-pgdog-stopped.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Make sure no pgdog process is left behind — coverage profile collection +# needs the process to exit cleanly. +set -euo pipefail + +if pgrep -x pgdog > /dev/null; then + killall -TERM pgdog + sleep 5 +fi diff --git a/integration/ci/install-deps.sh b/integration/ci/install-deps.sh new file mode 100755 index 000000000..7fc0aac2c --- /dev/null +++ b/integration/ci/install-deps.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Install the bits that the Blacksmith runner image doesn't already ship. +# Linux/Debian-derivative only — no-op on other platforms. Idempotent. +set -euo pipefail + +if [[ "$(uname -s)" != "Linux" ]]; then + exit 0 +fi +if ! command -v apt-get >/dev/null 2>&1 || ! command -v dpkg >/dev/null 2>&1; then + echo "install-deps.sh requires a Debian/Ubuntu runner (apt-get + dpkg)" >&2 + exit 1 +fi + +NEXTEST_VERSION="${NEXTEST_VERSION:-0.9.78}" +LLVM_COV_VERSION="${LLVM_COV_VERSION:-0.6.10}" +CMAKE_VERSION="${CMAKE_VERSION:-3.31.6}" + +if ! dpkg -s mold >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y --no-install-recommends mold +fi + +# The protocol_version smoke test needs a psql >= 18 client. The runner +# image ships psql 16, so pull the v18 client from PGDG. pg_wrapper picks +# the highest installed major, so /usr/bin/psql will resolve to 18. +if ! dpkg -s postgresql-client-18 >/dev/null 2>&1; then + sudo install -d /usr/share/postgresql-common/pgdg + sudo curl -fsSLo /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \ + https://www.postgresql.org/media/keys/ACCC4CF8.asc + . /etc/os-release + echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main" \ + | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null + sudo apt-get update + sudo apt-get install -y --no-install-recommends postgresql-client-18 +fi + +# Kitware prebuilt cmake pinned to the version pgdog expects. Much faster +# than `pip install cmake` which compiles from source. +if ! cmake --version 2>/dev/null | head -1 | grep -q "$CMAKE_VERSION"; then + curl -fsSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz" \ + | sudo tar xzf - -C /opt + sudo ln -sf "/opt/cmake-${CMAKE_VERSION}-linux-x86_64/bin/cmake" /usr/local/bin/cmake + sudo ln -sf "/opt/cmake-${CMAKE_VERSION}-linux-x86_64/bin/ctest" /usr/local/bin/ctest +fi + +CARGO_BIN="${CARGO_HOME:-$HOME/.cargo}/bin" +mkdir -p "$CARGO_BIN" + +if ! command -v cargo-nextest >/dev/null; then + curl -LsSf "https://get.nexte.st/${NEXTEST_VERSION}/linux" | tar zxf - -C "$CARGO_BIN" +fi + +if ! command -v cargo-llvm-cov >/dev/null; then + curl -LsSf "https://github.com/taiki-e/cargo-llvm-cov/releases/download/v${LLVM_COV_VERSION}/cargo-llvm-cov-x86_64-unknown-linux-gnu.tar.gz" \ + | tar zxf - -C "$CARGO_BIN" +fi diff --git a/integration/ci/prepare-instrumented-pgdog.sh b/integration/ci/prepare-instrumented-pgdog.sh new file mode 100755 index 000000000..eeea163fe --- /dev/null +++ b/integration/ci/prepare-instrumented-pgdog.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Build an llvm-cov-instrumented pgdog binary for the integration job and +# export its path via $GITHUB_ENV so later steps can invoke it. +set -euo pipefail + +export RUSTFLAGS="-C link-dead-code" + +cargo llvm-cov clean --workspace +mkdir -p target/llvm-cov-target/profiles +cargo llvm-cov run --no-report --release --package pgdog --bin pgdog -- --help +rm -f target/llvm-cov-target/profiles/*.profraw +rm -f target/llvm-cov-target/profiles/.last_snapshot +rm -rf target/llvm-cov-target/reports + +BIN_PATH=$(find target/llvm-cov-target -type f -path '*/release/pgdog' | head -n 1) +if [ -z "$BIN_PATH" ]; then + echo "Instrumented PgDog binary not found" >&2 + exit 1 +fi +echo "Using instrumented binary at $BIN_PATH" + +RESOLVED="$(realpath "$BIN_PATH")" +if [[ -n "${GITHUB_ENV:-}" ]]; then + echo "PGDOG_BIN=${RESOLVED}" >> "$GITHUB_ENV" +else + echo "PGDOG_BIN=${RESOLVED}" +fi diff --git a/integration/ci/setup.sh b/integration/ci/setup.sh new file mode 100755 index 000000000..c95b99636 --- /dev/null +++ b/integration/ci/setup.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Bootstrap PostgreSQL + pgdog's test fixtures on a Blacksmith runner. +# Called from workflow jobs after install-deps.sh. +# +# Usage: integration/ci/setup.sh [--with-toxi] +set -euo pipefail + +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +REPO_ROOT="$( cd "${SCRIPT_DIR}/../.." && pwd )" + +# Use whatever pg cluster is installed on the runner unless overridden. +if [[ -z "${PSQL_VERSION:-}" ]]; then + PSQL_VERSION=$(ls -d /etc/postgresql/*/main 2>/dev/null \ + | sed 's|.*/postgresql/\([0-9]\+\)/main|\1|' \ + | sort -n | tail -1 || true) +fi +if [[ -z "${PSQL_VERSION:-}" ]]; then + echo "No PostgreSQL cluster found under /etc/postgresql" >&2 + exit 1 +fi + +with_toxi=false +for arg in "$@"; do + case "$arg" in + --with-toxi) with_toxi=true ;; + *) echo "unknown arg: $arg" >&2; exit 1 ;; + esac +done + +sudo pg_ctlcluster "$PSQL_VERSION" main start + +USER_NAME="$(id -un)" +sudo -u postgres createuser --superuser --login "$USER_NAME" +sudo -u postgres createdb "$USER_NAME" + +bash "${REPO_ROOT}/integration/setup.sh" \ + || (sudo pg_ctlcluster "$PSQL_VERSION" main restart && bash "${REPO_ROOT}/integration/setup.sh") + +if [[ "$with_toxi" == true ]]; then + bash "${REPO_ROOT}/integration/toxi/setup.sh" +fi diff --git a/integration/common.sh b/integration/common.sh index 812acc69f..113db8377 100644 --- a/integration/common.sh +++ b/integration/common.sh @@ -103,6 +103,12 @@ function stop_toxi() { function active_venv() { pushd ${COMMON_DIR}/python - source venv/bin/activate + if [[ ! -f venv/bin/activate ]]; then + virtualenv venv + source venv/bin/activate + pip install -r requirements.txt + else + source venv/bin/activate + fi popd } diff --git a/integration/complex/run.sh b/integration/complex/run.sh index 21747f9ea..41f0993bb 100755 --- a/integration/complex/run.sh +++ b/integration/complex/run.sh @@ -2,6 +2,9 @@ set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +# shutdown.sh and cancel_query use active_venv (python virtualenv). +bash ${SCRIPT_DIR}/../ci/apt.sh python3-virtualenv + pushd ${SCRIPT_DIR} bash shutdown.sh bash passthrough_auth/run.sh diff --git a/integration/dry_run/package.json b/integration/dry_run/package.json index 2865b4940..91c4268c9 100644 --- a/integration/dry_run/package.json +++ b/integration/dry_run/package.json @@ -1,4 +1,5 @@ { + "type": "module", "dependencies": { "@sequelize/postgres": "^7.0.0-alpha.46", "pg": "^8.16.3" diff --git a/integration/js/pg_tests/test/postgres_js.js b/integration/js/pg_tests/test/postgres_js.js index 6b8352a1d..1e3a1f1f6 100644 --- a/integration/js/pg_tests/test/postgres_js.js +++ b/integration/js/pg_tests/test/postgres_js.js @@ -690,7 +690,7 @@ describe("postgres.js unsafe stress test (50k unique statements, 5 clients)", fu assert.strictEqual(completed, TOTAL_QUERIES); // Verify backend prepared statement evictions are happening. - const res = await fetch("http://localhost:9090"); + const res = await fetch("http://127.0.0.1:9090"); const metrics = await res.text(); const evictions = metrics .split("\n") diff --git a/integration/load_balancer/run.sh b/integration/load_balancer/run.sh index e3cbbf88c..1b9e76f05 100755 --- a/integration/load_balancer/run.sh +++ b/integration/load_balancer/run.sh @@ -3,6 +3,10 @@ set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source ${SCRIPT_DIR}/../common.sh +# Uses the legacy standalone `docker-compose` binary; newer Docker installs +# only ship the `docker compose` plugin. +bash ${SCRIPT_DIR}/../ci/apt.sh docker-compose + pushd ${SCRIPT_DIR} export PGUSER=postgres diff --git a/integration/mirror/run.sh b/integration/mirror/run.sh index 3b8ffad56..6f21a8098 100644 --- a/integration/mirror/run.sh +++ b/integration/mirror/run.sh @@ -3,6 +3,12 @@ set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source ${SCRIPT_DIR}/../common.sh +# Runs both ruby and php; install all their build-time deps. +bash ${SCRIPT_DIR}/../ci/apt.sh \ + ruby-dev libyaml-dev libpq-dev build-essential \ + php-cli php-pgsql +command -v bundle >/dev/null || sudo gem install bundler --no-document + run_pgdog $SCRIPT_DIR wait_for_pgdog diff --git a/integration/plugins/run.sh b/integration/plugins/run.sh index 0cf48cfb0..f8bbc01e0 100644 --- a/integration/plugins/run.sh +++ b/integration/plugins/run.sh @@ -3,6 +3,10 @@ set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source ${SCRIPT_DIR}/../common.sh +# dev.sh runs rspec via bundler; native gem extensions need yaml + libpq headers. +bash ${SCRIPT_DIR}/../ci/apt.sh ruby-dev libyaml-dev libpq-dev build-essential +command -v bundle >/dev/null || sudo gem install bundler --no-document + export CARGO_TARGET_DIR=${SCRIPT_DIR}/target pushd ${SCRIPT_DIR}/test-plugins/test-plugin-compatible diff --git a/integration/python/run.sh b/integration/python/run.sh index 833ef5594..89760d383 100644 --- a/integration/python/run.sh +++ b/integration/python/run.sh @@ -3,6 +3,8 @@ set -euo pipefail SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source ${SCRIPT_DIR}/../common.sh +bash ${SCRIPT_DIR}/../ci/apt.sh python3-virtualenv + run_pgdog wait_for_pgdog diff --git a/integration/ruby/run.sh b/integration/ruby/run.sh index 83c730a20..792b35971 100644 --- a/integration/ruby/run.sh +++ b/integration/ruby/run.sh @@ -3,6 +3,10 @@ set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source ${SCRIPT_DIR}/../common.sh +# Native gem extensions (psych, pg) need yaml + libpq headers. +bash ${SCRIPT_DIR}/../ci/apt.sh ruby-dev libyaml-dev libpq-dev build-essential +command -v bundle >/dev/null || sudo gem install bundler --no-document + run_pgdog wait_for_pgdog diff --git a/integration/toxi/run.sh b/integration/toxi/run.sh index 498027451..95a53a32b 100644 --- a/integration/toxi/run.sh +++ b/integration/toxi/run.sh @@ -3,6 +3,10 @@ set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) source ${SCRIPT_DIR}/../common.sh +# Ruby specs; psych + pg need yaml + libpq headers to build. +bash ${SCRIPT_DIR}/../ci/apt.sh ruby-dev libyaml-dev libpq-dev build-essential +command -v bundle >/dev/null || sudo gem install bundler --no-document + run_pgdog wait_for_pgdog