From da8f0d43570a15d4482a945adc6a697b4bcddac5 Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Thu, 16 Jul 2026 12:58:34 +0200 Subject: [PATCH 01/28] [OPIK-6901] [BE] test: buffered cutover deletion-bridge E2E gate + runbook --- .../traces-local-v2-cutover/README.md | 599 +++++++ .../scripts/backfill.sh | 277 ++++ .../000001_backfill_traces_local_v2.sql | 75 + .../000002_delta_and_deletion_replay.sql | 173 +++ .../000003_exchange_and_wrap.sql | 39 + .../000004_rollback_reverse_replay.sql | 37 + ...000004_rollback_stage_a_discard_shadow.sql | 13 + .../000004_rollback_stage_b_exchange_back.sql | 11 + ...0004_rollback_stage_c_promote_original.sql | 16 + .../000005_verify_migration.sql | 187 +++ .../scripts/delta_replay.sh | 46 + .../scripts/estimate.sh | 135 ++ .../scripts/exchange_and_wrap.sh | 184 +++ .../scripts/finalize.sh | 84 + .../scripts/rollback.sh | 131 ++ .../traces-local-v2-cutover/scripts/verify.sh | 136 ++ .../TracesLocalV2CutoverTest.java | 1370 +++++++++++++++++ deployment/docker-compose/docker-compose.yaml | 9 + .../tests/traces-local-v2-cutover/README.md | 109 ++ .../tests/traces-local-v2-cutover/_common.py | 76 + .../traces-local-v2-cutover/delete_traffic.py | 78 + .../traces-local-v2-cutover/live_traffic.py | 82 + .../traces-local-v2-cutover/requirements.txt | 3 + .../traces-local-v2-cutover/seed_history.py | 174 +++ 24 files changed, 4044 insertions(+) create mode 100644 apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md create mode 100755 apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh create mode 100644 apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000001_backfill_traces_local_v2.sql create mode 100644 apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000002_delta_and_deletion_replay.sql create mode 100644 apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000003_exchange_and_wrap.sql create mode 100644 apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_reverse_replay.sql create mode 100644 apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_a_discard_shadow.sql create mode 100644 apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_b_exchange_back.sql create mode 100644 apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql create mode 100644 apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000005_verify_migration.sql create mode 100755 apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/delta_replay.sh create mode 100755 apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/estimate.sh create mode 100755 apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh create mode 100755 apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/finalize.sh create mode 100755 apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh create mode 100755 apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh create mode 100644 apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java create mode 100644 tests_load/tests/traces-local-v2-cutover/README.md create mode 100644 tests_load/tests/traces-local-v2-cutover/_common.py create mode 100644 tests_load/tests/traces-local-v2-cutover/delete_traffic.py create mode 100644 tests_load/tests/traces-local-v2-cutover/live_traffic.py create mode 100644 tests_load/tests/traces-local-v2-cutover/requirements.txt create mode 100644 tests_load/tests/traces-local-v2-cutover/seed_history.py diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md new file mode 100644 index 00000000000..fd6e9104e09 --- /dev/null +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md @@ -0,0 +1,599 @@ +# Buffered cutover runbook — `traces` → partitioned + sharding-ready + +Operator runbook for the buffered cutover of the ClickHouse `traces` table: it migrates the live, unpartitioned +`traces` table to `traces_local_v2` (weekly-partitioned, denullified, `is_deleted`-ready) with **near-zero downtime** +and **zero deletion loss**, then wraps it in a sharding-ready `Distributed` table. + +The mechanism is **backfill + delta + deletion replay + EXCHANGE**, using the ingestion async-insert buffer to absorb +the brief cutover window instead of a dual-write path. + +This runbook is the human-facing artifact; its SQL is validated end-to-end by +[`TracesLocalV2CutoverTest`](../../src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java). Treat +that test as the executable specification of these scripts: if you change the cutover SQL, change it there first and keep +this runbook in sync. + +> **This is not a Liquibase migration.** The backfill / delta / replay / EXCHANGE steps are runbook-driven and paced by +> an operator — they produce sustained I/O and must not run as blocking changesets. `traces_local_v2` (migration 000101) +> and `deletion_events_local` (migration 000096) are already created by Liquibase; everything here operates on those. + +## Why this is not a plain `INSERT ... SELECT` + +A lightweight `DELETE` in ClickHouse flips a hidden row mask; it does **not** bump `last_updated_at` (the +`ReplacingMergeTree` version column). The cutover's delta step re-copies rows that changed during the backfill, but a +lightweight delete leaves no "changed" signal, so the delta is blind to every delete that fires during the +backfill/delta window — those rows stay alive on the new table and the deletion would silently leak across the swap. + +The **deletion-events bridge** closes it: with `traceDeletionEventsCaptureEnabled=true`, every trace delete records its +`(workspace_id, project_id, id)` in `deletion_events_local`; the cutover **replays** those keys as deletes against the +new table before the EXCHANGE. The replay matches the **full key**, not `id` alone — see "Delta and replay correctness". + +> **All user-facing trace deletes route through one captured path.** Single delete, batch delete-by-project, and thread +> deletion all funnel through `TraceService.delete(...)`, which calls `captureDeletions` on both the resolved-project +> and the unresolved (empty-project) branch — so enabling the flag covers every one. The **only** uncaptured +> `DELETE FROM traces` is the retention sweep, which is disabled (see the retention note). Any **new** trace-delete path +> introduced during the migration window must likewise capture, or its deletes would leak across the swap. + +> **Retention sweeps do not run during the cutover.** Data Retention is disabled in every deployment +> (`retention.enabled` defaults to `false`, env `RETENTION_ENABLED`, and has never been enabled), so the retention +> delete path (`TraceDAO.deleteForRetention*`) does not fire. The only deletes during the cutover window are +> **user-initiated**, and those are captured by the bridge (`TraceService`, reason `USER_REQUEST`). The retention path is +> intentionally **not** wired to the bridge. If Data Retention is ever enabled, either pause the retention job for the +> whole backfill→EXCHANGE window, or first wire retention deletes into the bridge (a `RETENTION` reason recorded before +> each `deleteForRetention*` delete). The test still exercises a synthetic large (retention-shape) delete batch, so the +> replay is proven to handle both batch sizes if retention is enabled later. + +## Deletion scenarios and how each is handled + +| Delete timing | Fate | Handling | +|---|---|---| +| Before the backfill | Row masked on the source | `INSERT SELECT` honors `apply_deleted_mask=1` → never copied. No replay. | +| During the backfill, after its row was copied | Delta can't see the mask flip | Captured in the bridge → **replayed** before EXCHANGE. | +| During the delta / buffer window | Same as above | Same bridge, same replay step. | + +## Prerequisites (do not start without these) + +1. **24h UUIDv7 ingestion validation** live for **≥ one full retention cycle**. Without it a stray future-dated UUID + creates an out-of-range weekly partition on the destination. +2. **`traces_local_v2` exists and is empty** (migration 000101). +3. **Successor storage/TTL parity.** `traces_local_v2` must resolve the **same `storage_policy` and TTL-to-cold rules** + as `traces` (tiering is configured per environment, not in the base DDL). If `traces` tiers hot→cold but the + successor does not, the entire backfill lands on the hot volume. `backfill.sh` warns on a `storage_policy` mismatch; + compare TTLs with `SHOW CREATE TABLE traces` vs `traces_local_v2`. +4. **`deletion_events_local` exists** (migration 000096). +5. **`databaseAnalyticsDataModel.traceDeletionEventsCaptureEnabled = true`** deployed and live before the backfill + begins, and kept on for the entire backfill→EXCHANGE window. On docker-compose set + `ANALYTICS_DB_DATA_MODEL_TRACE_DELETION_EVENTS_CAPTURE_ENABLED=true` (the backend service forwards it) and restart the + backend. `backfill.sh` captures the `backfill_start` anchor (a `now64(6)` taken just before the first INSERT) and + prints it — the delta and the replay both key off it. +6. **Cutover buffer knob ready** — `databaseAnalytics.asyncInsertBusyTimeoutMaxMs` (env + `ANALYTICS_DB_ASYNC_INSERT_BUSY_TIMEOUT_MAX_MS`), unset by default so the buffer inherits the + `async_insert_busy_timeout_max_ms=250` carried by `queryParameters`. Raise it to ~10000 for the cutover, then unset it + again. On a staging load test, confirm the adaptive-timeout widening actually takes effect before production. **Also + confirm client/SDK insert timeouts exceed the widened buffer** (~10s) — with `wait_for_async_insert=1` a raised + ceiling blocks each insert until it flushes, so a shorter client timeout would surface as ingestion errors during the + window. +7. **Schema-state flag wired, with a rollout plan** — `databaseAnalyticsDataModel.traceColumnsNonNullable` (env + `ANALYTICS_DB_DATA_MODEL_TRACE_COLUMNS_NON_NULLABLE`, default `false`). The successor's `end_time`/`ttft` are + **non-nullable sentinel** columns, so the app must bind epoch/NaN instead of `null` once they are live — a `null` + bind is rejected. This flag switches that (writes, reads, filters, sorts); it **must be flipped in lockstep with the + EXCHANGE** (see "The final cutover window"). Confirm it is deployable on the target (env passthrough present) and that + you have a fleet-wide rollout mechanism (config push or rolling restart) ready. +8. **Confirm Data Retention is disabled** (`RETENTION_ENABLED=false`, the default). If it is ever enabled, see the + retention note above first. +9. **Sufficient free disk** — the backfill writes a full second physical copy of `traces`, so node free space must clear + **≥ 2× the current `traces` on-disk size** (more counting merge scratch). `estimate.sh` reports headroom and + `backfill.sh` aborts below `--min-free-factor` (default 2.0). On tiered storage this whole-node floor is necessary but + not sufficient — validate per-volume (hot) headroom too, since new parts land hot before they tier. +10. **Schema parity of source and successor** — `traces` and `traces_local_v2` must stay equivalent for as long as both + exist: the same base (stored) columns (which the cutover must copy) and the same materialized columns (which each + table recomputes). Guarded in CI by `TracesLocalV2CutoverTest` — `cutoverCopiesEveryBaseColumn` (a new base column + fails the build until it is in the cutover column list) and `successorMaterializedColumnsMatchSource` (a materialized + column added to one table but not the other fails the build). Re-confirm both are green on the release being + deployed. +11. **Fresh backup / snapshot** of the ClickHouse data node. +12. **Freeze concurrent DDL on `traces` for the window.** Hold any deploy or Liquibase changeset that would `ALTER`, + `RENAME`, or otherwise touch `traces` / `traces_local_v2` for the whole backfill→EXCHANGE window — a schema change + landing mid-cutover races the swap and can corrupt it. The revamp's own migrations (000096/000101) are already + applied; this is about *unrelated* migrations or ad-hoc DDL during the window. +13. Schedule during off-peak hours. + +## The sequence + +1. **Backfill — run [`scripts/backfill.sh`](scripts/backfill.sh)** (preferred). It iterates by week oldest→newest, but + splits each week adaptively into `created_at` sub-windows so no single INSERT exceeds `--max-rows-per-insert` (see + "Batching and throttling"). It reconciles each window on a **dedup-aware** count (`uniqExact` of the dedup key, since + raw `count()` differs between an un-merged source and a destination that deduped versions on insert) and aborts only + on a genuine shortfall in a **settled** window (> 0.01%); a still-live window (its end in the future) legitimately + diverges from concurrent writes/deletes — the delta and replay reconcile it — so that is logged, not fatal. The + reconciliation counts source and destination **together after** each copy (a consistent snapshot): a delete is not + bounded by `created_at`, so it can shrink even a settled window mid-copy, and comparing a stale pre-copy source count + against a fresh post-copy destination count would abort falsely. It is idempotent and resumable (an already-copied + window is skipped), and prints the `backfill_start` anchor for step 2 — captured once and persisted to `--state-file`, + so a resumed run keeps the original anchor rather than minting a later one that would miss early-window deletes. + `--pause-seconds` throttles between windows; `--dry-run` prints the window plan. Preview then run: + ```bash + CLICKHOUSE_HOST= CLICKHOUSE_PASSWORD= ./scripts/backfill.sh --database opik --dry-run + CLICKHOUSE_HOST= CLICKHOUSE_PASSWORD= ./scripts/backfill.sh --database opik --pause-seconds 60 + ``` + It executes the reference statement in + [`000001_backfill_traces_local_v2.sql`](scripts/db-app-analytics/000001_backfill_traces_local_v2.sql) — the script + reads that file and substitutes the window bounds, so the two never drift. +2. **Raise the buffer ceiling** (config, see below), then **[`scripts/delta_replay.sh`](scripts/delta_replay.sh)** + (reference SQL [`000002_delta_and_deletion_replay.sql`](scripts/db-app-analytics/000002_delta_and_deletion_replay.sql)) + — delta-insert (anchored at `backfill_start`), then **deletion replay**. The replay runs with + `lightweight_deletes_sync = 2`, so it returns only once the delete mutation has applied on **every** replica. + clickhouse-client prints each statement's wall time; record the replay's. + ```bash + CLICKHOUSE_HOST= CLICKHOUSE_PASSWORD= ./scripts/delta_replay.sh --database opik --backfill-start '' + ``` +3. **QA — run [`scripts/verify.sh`](scripts/verify.sh)** (see "Verifying the migration"): confirm the copy altered no + data before committing the swap. Run it after step 2 (and it can be re-run after step 4). +4. **[`scripts/exchange_and_wrap.sh`](scripts/exchange_and_wrap.sh)** (reference SQL + [`000003_exchange_and_wrap.sql`](scripts/db-app-analytics/000003_exchange_and_wrap.sql)) — first **gates on a settled + replication state** (empty `replication_queue` on `traces`/`traces_local_v2` and the deletion-replay mutation finished + on `traces_local_v2`, across all replicas via `clusterAllReplicas`) so no replica swaps in an incomplete table + (`--force` overrides); then records and + prints `cutover_start`, runs `EXCHANGE TABLES ... ON CLUSTER` and renames the displaced old data to + `traces_pre_cutover_backup` (see "Naming and the parked backup"), then (unless `--skip-wrap`) the `RENAME` + + `Distributed` wrap. Restore the buffer ceiling and verify. + ```bash + CLICKHOUSE_HOST= CLICKHOUSE_PASSWORD= ./scripts/exchange_and_wrap.sh --database opik --skip-wrap + ``` + +> **HARD PREREQUISITE for the wrap (step 4, part 2): the delete/mutation DAO must target `traces_local` first.** A +> `Distributed` table supports `SELECT` and `INSERT` but **not** mutations. Verified on ClickHouse 26.3: +> - `DELETE FROM ` → `Code 36 BAD_ARGUMENTS: DELETE query is not supported` +> - `ALTER TABLE DELETE` → `Code 48 NOT_IMPLEMENTED: Distributed doesn't support mutations` +> +> So the moment the wrap is applied, **both** the product's delete-by-id (`TraceDAO.DELETE_BY_ID`) **and** the retention +> sweep (`DELETE_FOR_RETENTION`) start returning 500 against `traces`. This is prep work that must ship **before** the +> wrap: point those DAO paths at `traces_local` (reads and inserts can stay on the Distributed `traces`). The `EXCHANGE` +> alone is the data cutover and leaves `traces` a `MergeTree` where deletes still work — so use `exchange_and_wrap.sh +> --skip-wrap` and defer the wrap until the sharding-aware DAO ships. The wrap is the sharding-readiness layer, not the +> cutover. +> +> **Applying the deferred wrap later:** once the sharding-aware DAO has shipped, run +> `exchange_and_wrap.sh --database opik --wrap-only` — it runs the settle gate and applies **only** the `RENAME` + +> `Distributed` wrap on the already-EXCHANGEd `traces` (no second EXCHANGE, no new `cutover_start`). To roll the wrap +> back, use `rollback.sh --stage C`. + +**Dedup note.** After the delta, a row can have two physical versions on `traces_local_v2` (the backfilled one and the +delta one). This is normal — `ReplacingMergeTree` collapses them on merge / under `FINAL` / `LIMIT 1 BY id`, highest +`last_updated_at` winning. Do not "fix" it. + +### The final cutover window (the zero-loss invariant) + +The buffer widening (prereq 6) is what makes the flip lossless, but the guarantee rests on a timing invariant worth +stating precisely. Writes use `async_insert=1, wait_for_async_insert=1`, so a raised `asyncInsertBusyTimeoutMaxMs` parks +each insert (the client blocks) until it flushes — and after the `EXCHANGE` a parked insert flushes into whatever table +is now named `traces`, i.e. the successor. **But the adaptive buffer also flushes on size**, so under load a flush can +still land in the *old* `traces` in the gap between the last delta read and the `EXCHANGE` — and the delta has already +run. The binding constraint is therefore **not** "replay < buffer window"; it is that the **gap between the final delta +and the `EXCHANGE` completing must stay within the buffer hold**. So run the tail as tightly as possible: + +1. Widen the buffer, and **roll out `traceColumnsNonNullable = true` to every backend instance** (see below). +2. Do the QA verify on an **earlier** pass (it can take minutes on a large table — do not let it be the last thing + before the swap). +3. Run a **final** `delta_replay.sh` as the last write-facing step. +4. Run `exchange_and_wrap.sh` **immediately** after it (the settle gate + `EXCHANGE` are fast and metadata-only). +5. Restore the buffer ceiling; parked inserts flush into the successor. + +Keep step 3→4 short. If load is high enough that a size-triggered flush is plausible in that gap, re-run the final delta +once more right before the swap. Everything up to `cutover_start` is covered by the delta+replay; the buffer covers the +instant of the flip. + +**The `traceColumnsNonNullable` flip (mandatory, and why it goes first).** The successor stores `end_time`/`ttft` as +non-nullable epoch/NaN sentinels; the app must bind those sentinels — not `null` — the moment that schema is live under +the name `traces`, or every write of an in-progress trace (no `end_time` yet) is **rejected** by the non-nullable +column. The flag switches the app to sentinel binds (and sentinel→`null` on read). It is a **config** change rolled out +across the fleet (not atomic), unlike the metadata-only `EXCHANGE`, so it cannot be flipped at the same instant. Roll it +out to `true` on **all** instances **before** the `EXCHANGE`: binding the epoch/NaN sentinel into the *still-Nullable* +source column is valid, and reads translate epoch/`null`→`null` either way, so `true` is write-safe on both schemas — +this removes any write-rejection window. The copy machinery already tolerates the resulting NULL/epoch mix (backfill +`coalesce`, verify normalizes both to `0`). The one transient caveat is that "`end_time` is empty"-style **filters** use +sentinel logic against the still-Nullable table during that short pre-swap window; keep the window short and off-peak. +On rollback, after swapping the Nullable original back, revert the flag to `false`. + +## Batching and throttling + +On a large production table a single week can be enormous, so the backfill does **not** run one INSERT per week. Two +independent controls keep each statement safe: + +- **Per-statement row bound (`--max-rows-per-insert`, default 2,000,000).** `backfill.sh` counts each week and, if it + exceeds the bound, halves it in `created_at` time — adaptively, so busy periods split more and quiet ones stay whole — + until every leaf window fits, then inserts each. This bounds each INSERT's **duration**, its **blast radius** on + failure (only that window re-runs), the **part count** it adds to the destination, and gives per-window resume. It is + *not* a memory bound. Smaller values are safer per statement but create more parts (more merge pressure); larger + values create fewer parts but a bigger blast radius. Note the ClickHouse "batch 1k–100k rows" guidance targets + client-side row-by-row inserts; a server-side `INSERT … SELECT` streams and is efficient at far larger sizes, so the + default is millions, not thousands. +- **Per-block memory bound (`--max-insert-block-size`, default 1,048,576 = the ClickHouse default → `SETTINGS + max_insert_block_size`).** An `INSERT … SELECT` streams; ClickHouse forms part-writing blocks capped at the smaller of + this row count and `min_insert_block_size_bytes` (256 MB default). For wide/heavy trace rows the byte cap dominates, + so peak insert memory is a small multiple of ~256 MB regardless of the window size — the statement does not load the + window into memory. Lower this (or `min_insert_block_size_bytes`) on a memory-constrained data node. + +**Throttle** with `--pause-seconds` (recommended 30–60s at peak): it sleeps after each inserted window so background +merges consolidate the new parts before the next window piles on more. + +**Estimate first.** [`scripts/estimate.sh`](scripts/estimate.sh) projects the backfill ETA for a given config: it reads +the live row/byte counts of `traces`, estimates copy throughput with an **on-the-fly read probe** (`SELECT … FORMAT +Null` — it creates no table), derates it by `--write-cost-factor` to account for the copy's unmeasured write/compression +cost, and reports the projected window count, copy time, throttle idle, and total. Run it with the same +`--max-rows-per-insert` / `--pause-seconds` you plan to use: +```bash +CLICKHOUSE_HOST= CLICKHOUSE_PASSWORD= ./scripts/estimate.sh --database opik --max-rows-per-insert 2000000 --pause-seconds 60 +``` +For an exact figure, time one real window with `backfill.sh` and feed its rows/sec back via `--rows-per-sec`. +It is a planning ballpark — real throughput varies with concurrent load, merges and cold-tier reads. + +The **delta-insert** (step 2) covers only writes during the backfill window, not the whole table, so it is normally one +statement (with the same block-size bound); `000002` documents how to split it into two batched passes if a long backfill +made it large. The **deletion replay** is a lightweight `DELETE`, and with retention disabled it is user-scale — a single +mutation; `000002` / `000004` note how to bound it by partition if it is ever large. + +## Why slice by `created_at` (and not `id` or workspace) + +The backfill reads 100% of the table regardless of the slice column — the slice only decides how the work is *batched*, +and it does **not** decide where a row lands on the destination: that is always `toMonday(id_at)`, derived from the row's +`id`, independent of the slice. Three forces pick the slice column, and `created_at` is the only one that satisfies all: + +- **Source read efficiency.** The source `traces` has a **minmax skip index on `created_at`** (migration 000088), so each + week prunes granules cheaply. It has **no `id` skip index**, and `id` is the *trailing* primary-key column + `(workspace_id, project_id, id)` — a bare `id`-range predicate cannot prune the primary index (leading key columns are + free), so `id`-range slicing would **full-scan the table once per week**. +- **Bounded, complete iteration — where the bad-`id` data matters.** A known litellm bug + ([BerriAI/litellm#31294](https://github.com/BerriAI/litellm/pull/31294)) minted some existing traces with UUIDv7 ids + whose embedded timestamp is in the far future (year ~2201), so `id_at` ≠ `created_at` for those rows. 24h UUIDv7 + validation stops *new* offenders but does not fix rows already in the table. This makes `id` **unreliable and + unbounded** as a slice key: an `id`/`id_at`-range loop would have to span from today all the way to ~2201 (thousands of + empty weeks) to cover them. `created_at` is server-stamped and bounded to the real ingestion window, so `backfill.sh` + iterates `toMonday(min/max(created_at))` — a finite, gap-free range — and the bad-`id` rows are still copied (in their + `created_at` week) and still land in their (far-future) `id_at` partition. The bug is an argument *for* `created_at`. +- **Stable membership (data safety).** `created_at` is **immutable across upserts** on `traces` (the merge templates keep + the original), so a row never migrates between weekly slices mid-backfill — none is copied twice or skipped. + `last_updated_at` would *not* be safe here (it moves on every upsert, and is client-settable). + +**On adding an `id` index (and destination write locality).** An `id` skip index on the source is *not* needed — the +delta uses the `created_at`/`last_updated_at` skip indexes, and the replay uses the primary key (full-key match). It +would not rescue `id`-slicing either (the ~2201 span is a *data* problem, not an index one). Destination write locality +is naturally good with `created_at` slicing (`id_at ≈ created_at` once validation holds); slicing by *workspace* would +instead scatter each insert across every weekly partition that workspace spans → a small-part explosion on a 4 TB table. + +**Known issue — far-future partitions from bad ids.** Because the destination partitions by `toMonday(id_at)`, the +bad-`id` rows create far-future weekly partitions on `traces_local_v2` (inherent to the DDL + the bad data, not to the +slice choice). Note the year: the ids' embedded UUIDv7 timestamp is ~2201, but `id_at` is a **32-bit `DateTime`** (max +year 2106), so `UUIDv7ToDateTime` overflows and **wraps ~2201 to ~2065** — that is where the partitions actually land. +Either way it is bounded (few distinct bad timestamps → few extra partitions) and mostly harmless (those partitions +never tier to cold and are skipped by +time-bounded reads); the audit query below finds them regardless of the exact year (it keys on `id_at` being in the +future, not on a specific year). Quantify it before +the cutover and decide whether to remediate: + +```sql +-- rows / distinct far-future partitions the bad ids would create +SELECT count() AS bad_rows, uniqExact(toMonday(id_at)) AS bad_partitions, min(id_at) AS earliest, max(id_at) AS latest +FROM ${ANALYTICS_DB_DATABASE_NAME}.traces +WHERE id_at > now() + INTERVAL 1 DAY; -- outside the 24h validation window +``` + +If the count is material, remediate the source ids (or exclude/quarantine those rows) first; otherwise accept the few +far-future partitions. + +**No explicit `ORDER BY` on the `INSERT ... SELECT`.** Not needed for correctness or reproducibility: the final table +state is a `ReplacingMergeTree` reduction keyed on `(workspace_id, project_id, id)` with `last_updated_at` as the version +— **independent of insert order** — so any run converges to the same live rows; ClickHouse already sorts each insert +block by the destination `ORDER BY`, and since the source shares that key the rows arrive in order anyway; and +reconciliation uses order-independent `uniqExact` of the dedup key. An explicit `ORDER BY` would only add sort cost/memory +on a 4 TB backfill for no gain. + +## Delta and replay correctness + +**Delta anchor — `created_at OR last_updated_at ≥ backfill_start`.** The delta must re-copy everything written during the +(possibly multi-day) backfill: + +- `last_updated_at` is **client-supplied** on the batch-ingest path (`TraceDAO.BATCH_INSERT` binds the request's value, + server time only as a fallback), so `last_updated_at` alone can miss a row whose client stamped it in the past. But + every write path sets **either** a fresh server `created_at` (the batch-ingest path leaves `created_at` to its + `now64()` default) **or** a fresh server `last_updated_at` (the create/update merge paths preserve `created_at` but let + `last_updated_at` default to `now64()`). The **union** therefore catches every physical write, whatever the client sends. +- The anchor is captured **before** the backfill, not at its end — a cutoff taken at the end would miss writes that + landed during the backfill itself. The same `backfill_start` bounds the replay window. + +**Replay matches on two branches — full key, or `(workspace_id, id)` — mirroring the product's two delete paths.** +`TraceService.delete(ids, projectId)` resolves each id's owning project and deletes per project (full key); ids it can't +resolve fall back to a **workspace-scoped** delete — `TraceDAO.DELETE_BY_ID` with the project filter omitted, i.e. +`DELETE … WHERE id IN … AND workspace_id = …` across every project. The bridge records the first with the project and the +second with an **empty `project_id`** (`DeletionEventDAO`: "project_id is empty for workspace-scoped source tables"). The +replay mirrors both: full-key events delete by `(workspace_id, project_id, id)` (exact; prunes on the destination primary +key — correct even though trace ids are not globally unique), and empty-project events delete by `(workspace_id, id)`. +The second branch is a **faithful mirror, not an over-delete**: the workspace-scoped fallback fires only for ids the +resolver found no live row for in any project, and the source delete it replays already removed every `(workspace_id, +id)` row — so the destination ends matching the source. Without it, those deletions would **silently leak** across the +swap. + +**Resurrection guard.** A trace can be deleted and then re-created/updated under the **same id** during the window +(ids are client-supplied; the delete is a mask, and a newer insert wins under `FINAL`). Such an id is bridged as deleted +but is **live again** on the source, and the backfill/delta already copied its live version. So each replay branch also +requires the id is **not currently live on the source** (`AND (…) NOT IN (SELECT … FROM traces WHERE id IN )`, mask-honored) before deleting it — otherwise the replay would drop a row that is live on the source, +silent data loss. This also makes the replay idempotent (it never masks a live-on-source id), so re-running to +convergence is safe. + +The replay runs with `allow_nondeterministic_mutations = 1` because it reads subqueries from `deletion_events_local` and +`traces`; those tables are replicated and identical on every node and the window is fixed, so the subqueries resolve to +the same set on every replica. It also sets `lightweight_deletes_sync = 2` so the statement returns only after the delete +mutation has applied on **every** replica — otherwise the async mutation could still be pending on a replica when the +verify or the EXCHANGE runs, giving a false mismatch or an incomplete cutover. + +## How `backfill.sh` and `000001_backfill_traces_local_v2.sql` relate + +They are **complementary, not alternatives**, and there is **no copy-paste drift**: the script *reads* the `.sql` file. + +- **`000001_...sql` is the single source of the backfill `INSERT` (the "what"):** the exact statement, with `${...}` + placeholders for the database, window bounds and block size. It is read by the driver, not run by hand. +- **`backfill.sh` is the driver (the "how"):** it derives the week range, and for each week reads `000001_...sql`, + substitutes the placeholders, runs it, reconciles, throttles, and is resumable. It embeds no copy of the INSERT. + +**Every SQL operation — happy path and every rollback stage — is run by a driver script; no SQL or `.sql` file is ever +run by hand.** Each `.sql` file is the single source a driver reads: + +| Step | Reference SQL | Driver | +|------|---------------|--------| +| plan — backfill ETA | — | `estimate.sh` | +| 1 — backfill | `000001_backfill_traces_local_v2.sql` | `backfill.sh` | +| 2 — delta + replay | `000002_delta_and_deletion_replay.sql` | `delta_replay.sh` | +| 3 — EXCHANGE + wrap | `000003_exchange_and_wrap.sql` | `exchange_and_wrap.sh` | +| QA — fidelity compare (+ `--drill-down`) | `000005_verify_migration.sql` | `verify.sh` | +| rollback | `000004_rollback_stage_{a,b,c}_*.sql` + `000004_rollback_reverse_replay.sql` | `rollback.sh` | +| finalize — drop parked backup | — | `finalize.sh` | + +Each driver takes the connection from the standard `clickhouse-client` env vars (`CLICKHOUSE_HOST`, `CLICKHOUSE_PORT`, +`CLICKHOUSE_USER`, `CLICKHOUSE_PASSWORD`) and `--database`. + +**The only manual actions are not SQL:** (1) raising/restoring the async-insert buffer ceiling +(`databaseAnalytics.asyncInsertBusyTimeoutMaxMs`) around steps 2–3; (2) flipping +`databaseAnalyticsDataModel.traceColumnsNonNullable` to `true` in lockstep with the EXCHANGE (and back on rollback) — +see "The final cutover window"; and (3) the go/no-go judgement between steps. All three are *backend config* / judgement changes (env + rolling +restart, or a config push) that these DB-facing scripts cannot and should not make. They are deliberately operator-owned; +none involves typing SQL. + +## Naming and the parked backup + +Table names are the primary signal an operator acts on, so they encode which table is which — the safeguard against an +accidental `DROP` of the wrong (irreplaceable) table: + +- **`traces`** — always the live table the app reads/writes (the original before the cutover; the successor after it; + the `Distributed` wrapper after the wrap). +- **`traces_local_v2`** — always **the successor**: created empty by migration 000101, filled by backfill/delta, and — + after a rollback — re-parked as the abandoned successor. It only ever holds "the v2 data," so the `_v2` name is always + truthful. +- **`traces_local`** — the successor's live shard after the wrap (standard `Distributed`-over-`_local` idiom). +- **`traces_pre_cutover_backup`** — **the displaced old original**, produced by renaming it immediately after the + EXCHANGE. This rename is the whole point: leaving the old data under `traces_local_v2` would label the *oldest*, + *sole-backup* copy with a `_v2` suffix that reads as "the newer table" — and, post-wrap, sitting next to the live + `traces_local` it would invite dropping the wrong one. `traces_pre_cutover_backup` says exactly what it is and shares + no stem with the live shard, so neither confusion is possible. + +The one irreversible drop (`finalize.sh`) targets only the parked backup — `traces_pre_cutover_backup` after a +successful cutover, `traces_local_v2` after a rollback — and never the live `traces` or `traces_local`. + +## Rollback + +The full, ready-to-run rollback — including the **reverse deletion replay** so deletes don't resurrect — is pre-written +as one file per stage (`000004_rollback_stage_a_discard_shadow.sql`, `…_stage_b_exchange_back.sql`, +`…_stage_c_promote_original.sql`, and the shared `000004_rollback_reverse_replay.sql`) and driven by +[`scripts/rollback.sh`](scripts/rollback.sh), so no one authors it under pressure. + +**No data loss by construction.** The stages are mutually exclusive, so each lives in its **own file** — no single file +mixes the `TRUNCATE` (stage A only) with the `EXCHANGE`/`DROP` of the others, and running any file does exactly one +stage. No statement drops a data-bearing table: swaps are atomic `EXCHANGE`/`RENAME`, and the only `DROP` targets the +`Distributed` wrapper, which stores no data (it is a routing definition over `traces_local`). Before running, `rollback.sh` +**asserts the live `traces` topology matches the requested stage and aborts otherwise** — so a wrong-stage run (the only +way a `TRUNCATE`/`DROP` could hit the wrong table) makes no change. Every stage lands in the same **canonical state**: +`traces` = the original data (live), `traces_local_v2` = the successor data (parked backup). No leftover `*_new` names. +The parked backup is dropped only later, by `finalize.sh`, after the soak. + +Pick the stage by how far the cutover got (`cutover_start` is the value `exchange_and_wrap.sh` printed): + +- **Stage A — before EXCHANGE:** `./scripts/rollback.sh --database opik --stage A`. Discards the disposable shadow + `traces_local_v2`; the live `traces` was never touched. (Guarded: aborts unless `traces` is still the original schema.) +- **Stage B — after EXCHANGE, before wrap:** `./scripts/rollback.sh --database opik --stage B --cutover-start ''`. + `EXCHANGE` `traces_pre_cutover_backup` back to live `traces`, rename the now-parked successor back to + `traces_local_v2`, then the reverse replay. (Guarded: aborts if `traces` is `Distributed` — use C.) +- **Stage C — after wrap:** `./scripts/rollback.sh --database opik --stage C --cutover-start ''`. Drops the + `Distributed` wrapper, then one atomic `RENAME` promotes the original (`traces_pre_cutover_backup`) back to `traces` + and parks the successor under `traces_local_v2`, then the reverse replay. (Guarded: aborts unless `traces` is + `Distributed`.) + +After a stage B or C rollback, `traces` is the Nullable original again — **revert `traceColumnsNonNullable` to `false`** +(config, operator-owned) so the app resumes binding `null` for absent values and filters use Nullable logic. Leaving it +`true` is write-safe but would make absent-value filters/sorts use sentinel logic against a Nullable column. + +**Point of no return.** The `EXCHANGE` is reversible for as long as the parked backup exists (stage B/C). Dropping that +backup with `finalize.sh` is the one irreversible step, so gate it on an explicit soak: + +- **Soak duration** — keep the parked backup (`traces_pre_cutover_backup` after a successful cutover; `traces_local_v2` + after a rollback) for a defined window (recommend ~2 weeks; it fits well inside the bridge's 2-year TTL) so any latent + read/query regression surfaces while rollback is still an option. +- **Finalize exit criteria** — before dropping: `verify.sh` clean, query p99 within budget over the soak, no + cutover-related incidents open, and (if the wrap was applied) the sharding-aware DAO healthy in production. + +Once those hold, drop the parked backup with [`scripts/finalize.sh`](scripts/finalize.sh) — it auto-detects whichever +parked table is present (`traces_pre_cutover_backup` or `traces_local_v2`), never the live `traces`/`traces_local`. It +is dry-run by default, `--confirm` to drop, refuses if the live `traces` looks empty while the backup does not, and +refuses if both parked names somehow exist (ambiguous — resolve by hand). + +## Deletion bridge lifecycle & future migrations + +`deletion_events_local` is a **shared, long-lived** table (migration `000096`), not per-cutover. It is designed so +each migration sees only its own deletes, regardless of what else it already holds: + +- **Multiplexed by `source_table`** (`traces`, `spans`, …), which is the **leading `ORDER BY` key**. A replay filters + `source_table = '' AND event_time >= `, so it prunes — as a prefix scan — past every other + table's events and every event before its own anchor. A **non-empty bridge is the expected state** for the second and + later migrations; correctness comes from that filter, not from the table being empty (the traces cutover starting + empty was incidental). +- **Bounded** by monthly partitions (`PARTITION BY toYYYYMM(event_time)`) and a **2-year `TTL`**, so it cannot grow + without limit. A cutover only needs events spanning its window (hours–days) plus the soak (~2 weeks), so the TTL has + vast margin; shorten it only if the bridge ever runs hot under heavy delete volume. +- **Captured per source table** by independent knobs — `traceDeletionEventsCaptureEnabled` / + `spanDeletionEventsCaptureEnabled` — so capture is scoped to the table being migrated. + +**Capture is a per-migration, per-table lifecycle** — treat the knob like a valve around each cutover: + +1. Turn capture **on** just before that table's backfill starts (so every in-window delete is recorded). +2. **Keep it on through the soak** — the rollback reverse-replay reads the bridge, so capture must stay live until you + are past the rollback window. +3. Turn it **off after `finalize.sh`** — once the migration is committed and out of rollback range, its capture is an + extra write per delete with no reader. + +Because the knobs are independent, you never need both on at once: e.g. trace capture on for the traces cutover → soak → +finalize → trace capture off; later, span capture on for the spans cutover → soak → finalize → off. + +**For a future migration (e.g. `spans`)**: reuse this exact machinery — the bridge and `SpanService`'s +`SourceTable.SPANS` capture already exist. Build parallel `spans-local-v2-cutover` artifacts mirroring these +(spans schema/columns, `source_table = 'spans'`) rather than generalizing the drivers into one tool: the SQL is +genuinely table-specific, and a parallel directory keeps each migration's runbook self-contained and reviewable. The +only discipline is operational — enable span capture before the span backfill, capture the span `backfill_start` once, +and disable capture after finalize. + +## Per-deployment-variant notes + +| Variant | Strategy | Notes | +|---------|----------|-------| +| Comet SaaS | Buffered cutover (this runbook) | Buffer absorbs the cutover window; bridge active through the soak. | +| On-premise enterprise | Buffered cutover | Same runbook; ships in the same Helm push. | +| Open-source Docker | Brief read-only window | Little data, downtime acceptable. Bridge still ships; the replay is a no-op when there were no concurrent deletes. If the Liquibase ClickHouse extension cannot run `EXCHANGE ON CLUSTER`, use the fallback `RENAME` sequence. | +| AWS SageMaker | Buffered cutover | Runs on its own cadence; the bridge ships ahead of the cutover. | + +## Verifying the migration (QA) + +Prove the copy altered no data by comparing a **normalized fingerprint** of source and destination with +[`scripts/verify.sh`](scripts/verify.sh) (reference query: +[`000005_verify_migration.sql`](scripts/db-app-analytics/000005_verify_migration.sql)). The rows are not byte-identical +after the copy — `end_time` NULL becomes an epoch sentinel, `ttft` NULL becomes NaN, timestamps drop from nanosecond to +microsecond — so both sides are canonicalized to the same value for a faithfully-migrated row before hashing: timestamps +as their microsecond epoch, absent `end_time` as 0, absent `ttft` as the token `nan`, enums/ids via `toString`. Each row +hash includes the `id`; rows are deduped with `FINAL` and the delete mask is honored, so the comparison is of the live, +logical content. The fingerprint intentionally covers only the **copied base columns**: the materialized/derived columns +(`*_length`, `truncated_*`, `output_keys`, `duration`) are recomputed from those bases by identical pinned expressions, +so they cannot diverge unless an expression itself changes — which the gate test's dedicated derived-column parity check +catches directly. Per week it compares `count()` and an order-independent `sum` of the row hashes — together these catch +any changed, missing or extra row (`sum`, unlike `groupBitXor`, does not cancel a colliding pair within a table). The row +hash is `cityHash64`, not `sipHash64`: both sides are hashed live on the same instance, so a fast non-cryptographic +64-bit hash is enough — `sipHash64`'s adversarial-collision resistance would only add CPU (it is the right choice for the +*sharding key* in the wrap, a different job). **This is the exact normalization the gate test asserts** (see below), so +the tool is proven correct, not just plausible. + +```bash +# Full compare, every week, before the EXCHANGE (source=traces, dest=traces_local_v2 successor): +CLICKHOUSE_HOST= CLICKHOUSE_PASSWORD= ./scripts/verify.sh --database opik +# After the EXCHANGE: `traces` is the successor and the old data is parked as traces_pre_cutover_backup: +./scripts/verify.sh --database opik --old-table traces_pre_cutover_backup --new-table traces +``` + +> **The pre-EXCHANGE compare is the gate; the post-EXCHANGE compare has a caveat.** `traces_pre_cutover_backup` is a +> **frozen** snapshot as of `cutover_start`, but live `traces` keeps taking writes the instant the buffer drains — so +> the **current (live) week will legitimately show a mismatch** (the live table is a superset of the frozen backup by +> exactly the post-cutover writes). That is expected, not a leak. To use the post-EXCHANGE compare as a real check, +> either run it **immediately after the swap before writes resume**, or bound it to the **sealed historical weeks** +> (`--to-week `), where a mismatch *would* be a genuine problem. A leak shows up as rows present in the +> backup but absent from `traces`; post-cutover writes are the harmless opposite direction. + +**Feasibility at scale.** A full pass reads every partition (heavy but bounded per week — run off-peak). When that is +infeasible, sample and still get high confidence: +- `--sample-mod N` compares a deterministic 1/N `id` sample — the *same* rows on both sides, so like-for-like. +- `--weeks-stride S` compares every S-th weekly partition (partition-pruned, so genuinely cheaper). +- `--from-week` / `--to-week` bound the range (e.g. verify the most recent weeks fully, older weeks sampled). + +`verify.sh` exits non-zero if any window mismatches and prints the window bounds; re-run with `--drill-down` to list the +keys that differ or exist on one side only (it runs the `drill-down` block of `000005_verify_migration.sql` for each +mismatched window). + +## Verification — the automated test + +`TracesLocalV2CutoverTest` rehearses this exact sequence against a fresh ClickHouse and asserts: + +- **0 deletion leaks** across the EXCHANGE for deletes before backfill, a large retention-shape batch, and single + user-shape ids; +- a **negative control** proving the bridge is load-bearing (the leak reappears when replay is skipped); +- **full-key replay** — a reused id deleted in one project survives in another (no over-delete by id alone); +- **resurrection guard** — an id deleted and then re-created under the same id during the window stays live on the + destination (a naive replay-by-key would drop it — silent data loss); +- **delta completeness** — a row written during the window with a client-backdated `last_updated_at` is still caught (via + the `created_at` arm) and survives the cutover; +- newest-version-wins for concurrent upserts; +- **normalized-fingerprint fidelity** — the deduped, mask-honored, normalized `(count, checksum)` of source and + destination are equal before the swap (the same normalization `verify.sh` uses, so the QA tool is proven correct); +- **derived-column parity** — the recomputed columns (`id_at`, `*_length`, `truncated_*`, `output_keys` exactly; + `duration` within the intended ns→us precision, `NULL`↔`NaN` normalized) match between source and destination, so a + divergent MATERIALIZED expression is caught even though the base-column fingerprint excludes them; +- **schema-parity guards** — the cutover copies every base column of `traces`, and both tables expose the same base and + materialized columns (a future migration that drifts either fails the build); +- `EXCHANGE TABLES ... ON CLUSTER` and the single-shard `Distributed` wrapper both work; +- **reversibility** — rollback at each stage (before EXCHANGE, after EXCHANGE, after wrap) restores the original and + reverse-replays so a post-cutover delete does not resurrect; +- **wrong-stage rollback guard** — the topology signals `rollback.sh` keys on (the `traces` engine and `end_time` + nullability) are distinct in each cutover state, so a mis-targeted stage aborts instead of touching the wrong table; +- the replay wall time is measured and bounded. + +Run it with: `mvn -o test -Dtest=TracesLocalV2CutoverTest` from `apps/opik-backend`. + +## Monitoring and abort criteria + +Watch these for the whole backfill→EXCHANGE window; wire alerts before starting, not during: + +- **Free disk per volume** (`system.disks`) — the backfill is a full second copy; alert well before any volume fills. +- **Active part count / merge backlog** (`system.parts`, `system.merges`) on `traces_local_v2` — a runaway part count + means merges are not keeping up; increase `--pause-seconds`. +- **Replication backlog** (`system.replication_queue`) and **mutations** (`system.mutations` `is_done = 0`) — must trend + to zero; a growing queue means a replica is falling behind. +- **Ingestion latency / error rate** — with the widened buffer, insert latency rises by design (up to the buffer window); + alert on client-side timeouts or ingestion errors, which mean the client timeout is below the buffer. +- **Query p99** on the project traces listing — the backfill competes for I/O; a sustained regression is an abort signal. +- **Deletion-capture health** — capture is best-effort and **swallows** errors (so a bridge hiccup never blocks a user's + delete), so watch the backend logs for `captureDeletions` failures. A silently-dropped capture would leak a delete; + `verify.sh` catches that as a pre-EXCHANGE week mismatch (the row is live on the destination but gone on the source), + so it is an early-warning signal, not a silent hole — but treat repeated failures as an abort signal until capture is + healthy. + +**Roles.** Name an operator (runs the scripts), an independent observer (watches the dashboards), and the person with +authority to call a rollback. **Abort thresholds** (decide the numbers up front): free disk below the per-volume alarm, +query p99 beyond the agreed budget, or replication backlog that will not drain. Aborting before the `EXCHANGE` is +cheap (stage A); the bridge stays enabled so nothing is lost on a retry. + +## Go / No-Go checklist (production cutover) + +- [ ] **Runbook rehearsed on a production-shape staging snapshot** end-to-end; timings recorded. Staging must match + production **topology**, not just data shape — same replica count and tiered-storage policy — since the + multi-replica settle gates, storage/TTL parity, and buffer-flush timing are otherwise untested until production. +- [ ] **Deletion test green** — `TracesLocalV2CutoverTest` passes; **0 deletion leaks** confirmed on staging. +- [ ] **Replay duration fits inside the buffer window with margin** — measured replay wall time < the widened + `asyncInsertBusyTimeoutMaxMs`. +- [ ] **Far-future partitions quantified** — run the bad-`id` audit query above; remediated or explicitly accepted. +- [ ] **`EXCHANGE TABLES ... ON CLUSTER` works end-to-end** — or the fallback `RENAME` sequence is documented for the + variant that needs it. +- [ ] **Async-insert ceiling confirmed** — raising `asyncInsertBusyTimeoutMaxMs` demonstrably widens the adaptive buffer + under load, not just the cap. +- [ ] **Data Retention confirmed disabled** for the cutover window (`RETENTION_ENABLED=false`). +- [ ] **Reconciliation clean** — per-window source/dest counts within 0.01% across the whole backfill. +- [ ] **Replication settled before the EXCHANGE** — `replication_queue` empty and the deletion-replay mutation + `is_done` on **all** replicas (`exchange_and_wrap.sh` gates on this; do not `--force` past it in production). +- [ ] **`traceColumnsNonNullable = true` rolled out to every backend instance before the EXCHANGE** — confirmed live on + the whole fleet (else in-progress-trace writes are rejected the instant the non-nullable schema goes live); revert + plan to `false` ready for rollback. +- [ ] **Schema-parity guards green** — `cutoverCopiesEveryBaseColumn` and `successorMaterializedColumnsMatchSource` pass + on the release, so the cutover copies every base column of `traces` and the two tables' base and materialized + columns match. +- [ ] **Fidelity verified** — `verify.sh` passes (full, or a documented representative sample) between source and + destination before the EXCHANGE. Re-run `delta_replay.sh` then `verify.sh` until it PASSES: while the buffer holds + writes (or, on a rehearsal without it, once traffic is quiescent) the last delta must catch every in-flight write. +- [ ] **`Distributed` wrap gated on app-readiness** — apply the wrap (step 4, part 2) only when the delete/read DAOs are + sharding-aware; otherwise stop after the `EXCHANGE`, since a lightweight `DELETE` against a `Distributed` `traces` + is unsupported and breaks the trace-delete path. +- [ ] **No query-semantics regression** — FINAL / `LIMIT 1 BY` dedup verified; p99 on the project traces listing page within + ±10% of the pre-migration baseline. +- [ ] **Rollback rehearsed at every stage** (before EXCHANGE, after EXCHANGE/before wrap, after wrap) — deletes during + the post-cutover window do not resurrect after the reverse-replay; the parked table is retained for the soak. +- [ ] **Go/No-Go decision recorded** with the staging evidence attached. diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh new file mode 100755 index 00000000000..6d5cda94770 --- /dev/null +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh @@ -0,0 +1,277 @@ +#!/usr/bin/env bash +# +# Backfill driver for the buffered traces cutover (runbook: ../README.md, step 1). +# +# Copies traces -> traces_local_v2 oldest to newest, reconciling and aborting on divergence. It iterates by week (for +# progress and --from-week resume), but each week is further split, adaptively, into time sub-windows so that no single +# INSERT moves more than --max-rows-per-insert rows. On a large production table a whole week can be enormous; bounding +# each statement keeps its duration, its blast radius on failure, and the destination part-count it creates all in +# check. Memory is separately bounded by ClickHouse's block squashing (see --max-insert-block-size below). +# +# Week boundaries are derived from the data (toMonday(min/max(created_at))) — the operator does not hand-write dates. +# Idempotent and resumable: a window whose destination count already matches the source is skipped. +# +# The backfill INSERT is NOT duplicated here: it is read from db-app-analytics/000001_backfill_traces_local_v2.sql +# (the single source), with the ${...} placeholders substituted per window. See README "How backfill.sh and 000001 +# relate", "Why slice by created_at", and "Batching and throttling". +# +# Usage: +# CLICKHOUSE_HOST=... CLICKHOUSE_PASSWORD=... ./backfill.sh --database opik [options] +# +# Connection: passed straight to clickhouse-client via the standard env vars it honors +# (CLICKHOUSE_HOST, CLICKHOUSE_PORT, CLICKHOUSE_USER, CLICKHOUSE_PASSWORD). --database is required. +# +# Options: +# --database NAME analytics database (e.g. opik). Required. +# --dry-run print the window plan and per-window source counts; do not INSERT. +# --from-week N start at week offset N (0-based from the anchor Monday). Default 0. +# --to-week M stop after week offset M (inclusive). Default: last week with data. +# --max-rows-per-insert R upper bound on rows per INSERT statement; a week over this is halved by time until each +# sub-window fits. Default 2000000. Smaller = safer per statement but more parts / merge +# pressure; larger = fewer parts but bigger blast radius. This is a per-statement bound, not +# a memory bound (see --max-insert-block-size). +# --max-insert-block-size N rows per block ClickHouse forms while writing (SETTINGS max_insert_block_size). Peak insert +# memory is a small multiple of the smaller of this and min_insert_block_size_bytes (256 MB +# default), so for wide trace rows the byte bound usually dominates. Default 1048576 (the +# ClickHouse default); lower it on a memory-constrained data node. Applied to the INSERT. +# --divergence P max tolerated |src-dst|/src per window before aborting. Default 0.0001 (0.01%). +# --pause-seconds S sleep S seconds after each inserted window, to let destination merges catch up and bound +# the part count / IO pressure. Default 0. Recommended 30-60 on the ~4 TB table at peak. +# --min-free-factor F abort at startup unless node free disk >= F x the current `traces` on-disk size (the +# backfill writes a full second copy). Default 2.0. Pass 0 to skip the check. This is a +# whole-node floor; on tiered storage validate per-volume (hot) headroom separately. +# --state-file PATH file the captured backfill_start is written to and reused from. On resume the ORIGINAL +# anchor is kept; re-minting a later one would miss deletes that fired during the first run +# against already-copied rows. Default ./traces_cutover_backfill_start. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BACKFILL_SQL="$SCRIPT_DIR/db-app-analytics/000001_backfill_traces_local_v2.sql" + +# Fixed source/destination of this migration. The backfill INSERT itself lives in 000001; these are only for the +# script's own sizing and reconciliation queries. +SRC_TABLE="traces" +DST_TABLE="traces_local_v2" + +DATABASE="" +DRY_RUN=0 +FROM_WEEK=0 +TO_WEEK="" +MAX_ROWS=2000000 # rows: per-statement bound; a week over this is halved in time until each insert fits. Caps + # each INSERT's duration, blast radius and destination part count. NOT a memory bound. +MAX_INSERT_BLOCK_SIZE=1048576 # rows: SETTINGS max_insert_block_size for the INSERT. Peak memory is a small multiple of + # the smaller of this and min_insert_block_size_bytes (256 MB default), which dominates for wide + # trace rows; lower it on a memory-constrained node. 1048576 is the ClickHouse default. +DIVERGENCE="0.0001" # fraction: max tolerated |src-dst|/src per settled window before aborting (0.01%). +PAUSE_SECONDS=0 # seconds: sleep after each inserted window so destination merges catch up. 30-60 at ~4 TB peak. +MIN_FREE_FACTOR="2.0" # multiple of the current traces on-disk size that node free space must clear before starting. +STATE_FILE="./traces_cutover_backfill_start" # backfill_start is persisted here and reused on resume (keeps one anchor). + +# Floor on adaptive splitting: never divide a window shorter than this. Guards against splitting forever on a single +# hot instant; such a window is inserted whole (memory is still bounded by block squashing). +MIN_WINDOW_SECONDS=60 + +while [[ $# -gt 0 ]]; do + case "$1" in + --database) DATABASE="$2"; shift 2 ;; + --dry-run) DRY_RUN=1; shift ;; + --from-week) FROM_WEEK="$2"; shift 2 ;; + --to-week) TO_WEEK="$2"; shift 2 ;; + --max-rows-per-insert) MAX_ROWS="$2"; shift 2 ;; + --max-insert-block-size) MAX_INSERT_BLOCK_SIZE="$2"; shift 2 ;; + --divergence) DIVERGENCE="$2"; shift 2 ;; + --pause-seconds) PAUSE_SECONDS="$2"; shift 2 ;; + --min-free-factor) MIN_FREE_FACTOR="$2"; shift 2 ;; + --state-file) STATE_FILE="$2"; shift 2 ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done + +[[ -n "$DATABASE" ]] || { echo "ERROR: --database is required" >&2; exit 2; } +[[ -f "$BACKFILL_SQL" ]] || { echo "ERROR: cannot find backfill SQL at $BACKFILL_SQL" >&2; exit 2; } + +# Every query runs against the analytics database; --query keeps output scriptable (TSV, no formatting). +ch() { + clickhouse-client --database "$DATABASE" --query "$1" +} + +log() { + echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" +} + +bytes_gib() { + awk -v b="$1" 'BEGIN { printf "%.1f", b / 1073741824 }' +} + +# Capacity pre-flight. The backfill writes a full second physical copy of `traces` (peak ~2x on-disk, more counting +# merge scratch), so abort unless node free space clears --min-free-factor x the current size. This whole-node total is +# a necessary floor, not sufficient on tiered storage: new parts land on the hot volume before they tier, so validate +# per-volume headroom separately. Also warn (not abort) if the successor's storage_policy differs from the source's — a +# mismatch means the copy would not tier the same way and could fill the hot volume even when the node total looks fine. +preflight_capacity() { + local traces_bytes free_bytes need src_policy dst_policy + traces_bytes="$(ch "SELECT sum(bytes_on_disk) FROM system.parts WHERE database = '$DATABASE' AND table = '$SRC_TABLE' AND active")" + free_bytes="$(ch "SELECT sum(free_space) FROM system.disks")" + log "Capacity: $SRC_TABLE on-disk $(bytes_gib "$traces_bytes") GiB, node free $(bytes_gib "$free_bytes") GiB, need >= ${MIN_FREE_FACTOR}x" + if [[ "$MIN_FREE_FACTOR" != "0" ]]; then + need="$(awk -v t="$traces_bytes" -v k="$MIN_FREE_FACTOR" 'BEGIN { printf "%d", t * k }')" + if [[ "$(awk -v f="$free_bytes" -v n="$need" 'BEGIN { print (f < n) ? 1 : 0 }')" == "1" ]]; then + log "ABORT: node free disk $(bytes_gib "$free_bytes") GiB is below ${MIN_FREE_FACTOR}x $SRC_TABLE ($(bytes_gib "$need") GiB). Free space, or pass --min-free-factor 0 to override once per-volume headroom is validated." >&2 + exit 1 + fi + fi + src_policy="$(ch "SELECT storage_policy FROM system.tables WHERE database = '$DATABASE' AND name = '$SRC_TABLE'")" + dst_policy="$(ch "SELECT storage_policy FROM system.tables WHERE database = '$DATABASE' AND name = '$DST_TABLE'")" + if [[ "$src_policy" != "$dst_policy" ]]; then + log "WARNING: storage_policy differs ($SRC_TABLE='$src_policy', $DST_TABLE='$dst_policy'). If $SRC_TABLE tiers to cold and $DST_TABLE does not, the whole backfill lands on the hot volume. Confirm this is intended." >&2 + fi +} + +# Live source rows in [lo, hi). count() honors the deleted-row mask, so masked rows are excluded (they must not copy). +# This is a PHYSICAL row count, used only to size sub-windows against --max-rows-per-insert (not for reconciliation). +count_src() { + ch "SELECT count() + FROM $SRC_TABLE + WHERE created_at >= toDateTime64('$1', 9) + AND created_at < toDateTime64('$2', 9)" +} + +# Distinct LOGICAL rows in [lo, hi), by the ReplacingMergeTree dedup key. Reconciliation must be dedup-aware: raw +# count() differs between an un-merged source and a destination that deduped duplicate versions on insert +# (optimize_on_insert), even for a perfect copy. uniqExact of the key is what FINAL would collapse to on each side. +count_src_uniq() { + ch "SELECT uniqExact(workspace_id, project_id, id) + FROM $SRC_TABLE + WHERE created_at >= toDateTime64('$1', 9) + AND created_at < toDateTime64('$2', 9)" +} + +count_dst_uniq() { + ch "SELECT uniqExact(workspace_id, project_id, id) + FROM $DST_TABLE + WHERE created_at >= toDateTime64('$1', 6) + AND created_at < toDateTime64('$2', 6)" +} + +# Render the reference INSERT for one window by substituting placeholders (pure bash, no envsubst dependency). +run_backfill_window() { + local lo="$1" hi="$2" sql + sql="$(cat "$BACKFILL_SQL")" + sql="${sql//'${ANALYTICS_DB_DATABASE_NAME}'/$DATABASE}" + sql="${sql//'${WINDOW_LO}'/$lo}" + sql="${sql//'${WINDOW_HI}'/$hi}" + sql="${sql//'${MAX_INSERT_BLOCK_SIZE}'/$MAX_INSERT_BLOCK_SIZE}" + clickhouse-client --database "$DATABASE" --multiquery --query "$sql" +} + +# Insert one window whose physical row count is already within the per-statement bound. Reconciliation is dedup-aware +# (uniqExact) and concurrency-aware: a window still receiving writes (its created_at end is in the future) legitimately +# diverges during the copy — the delta-insert and deletion replay reconcile it — so an abort fires only on a genuine +# shortfall in a SETTLED window (a real backfill miss). "Settled" means no new rows by created_at; but a delete is NOT +# bounded by created_at and can mask a row in any window at any time, so the abort compares src and dst counted TOGETHER +# after the copy (a consistent snapshot) — never a stale pre-copy src against a fresh post-copy dst. +# Idempotent/resumable: a window already present on the destination is skipped. +insert_window() { + local label="$1" lo="$2" hi="$3" src dst settled short + src="$(count_src_uniq "$lo" "$hi")" + dst="$(count_dst_uniq "$lo" "$hi")" + + # Resume: destination already has this window (exact, ahead due to concurrent deletes, or within tolerance). + if [[ "$dst" != "0" ]] \ + && [[ "$(awk -v s="$src" -v d="$dst" -v p="$DIVERGENCE" 'BEGIN { print (d >= s || (s - d) / s <= p) ? 1 : 0 }')" == "1" ]]; then + log "$label ($lo .. $hi): already present (src_uniq=$src dst_uniq=$dst), skipping" + return + fi + if [[ "$DRY_RUN" == "1" ]]; then + log "$label ($lo .. $hi): would backfill ~$src rows" + return + fi + + log "$label ($lo .. $hi): backfilling ~$src rows" + run_backfill_window "$lo" "$hi" + + # Recount BOTH sides after the copy: a row deleted on the source between the pre-copy src count and the post-copy dst + # count is masked (so the mask-honoring INSERT never copied it) and would otherwise read as a shortfall and abort a + # settled window falsely. Counting src and dst together after the copy compares like with like. + src="$(count_src_uniq "$lo" "$hi")" + dst="$(count_dst_uniq "$lo" "$hi")" + settled="$(ch "SELECT now() >= toDateTime('$hi', 'UTC')")" + short="$(awk -v s="$src" -v d="$dst" -v p="$DIVERGENCE" 'BEGIN { print (d < s && (s - d) / s > p) ? 1 : 0 }')" + if [[ "$short" == "1" && "$settled" == "1" ]]; then + log "ABORT $label ($lo .. $hi): destination short of a settled window (src_uniq=$src dst_uniq=$dst). Investigate before continuing." >&2 + exit 1 + fi + if [[ "$short" == "1" ]]; then + log "$label ($lo .. $hi): live window (src_uniq=$src dst_uniq=$dst) — the delta-insert will reconcile concurrent writes" + elif [[ "$dst" -gt "$src" ]]; then + log "$label ($lo .. $hi): src_uniq=$src dst_uniq=$dst — concurrent source deletes; the deletion replay will reconcile" + else + log "$label ($lo .. $hi): OK (src_uniq=dst_uniq=$src)" + fi + + if [[ "$PAUSE_SECONDS" != "0" ]]; then + log "pausing ${PAUSE_SECONDS}s for merges to catch up" + sleep "$PAUSE_SECONDS" + fi +} + +# Recursively bound a window to --max-rows-per-insert by halving it in time, then insert each leaf. Adaptive rather than +# fixed sub-windows so it holds under traffic skew (busy periods split more; quiet ones stay whole). +process_range() { + local label="$1" lo="$2" hi="$3" src span mid + src="$(count_src "$lo" "$hi")" + if [[ "$src" == "0" ]]; then + return + fi + span="$(ch "SELECT dateDiff('second', toDateTime('$lo', 'UTC'), toDateTime('$hi', 'UTC'))")" + if [[ "$src" -le "$MAX_ROWS" || "$span" -le "$MIN_WINDOW_SECONDS" ]]; then + insert_window "$label" "$lo" "$hi" + return + fi + mid="$(ch "SELECT toString(addSeconds(toDateTime('$lo', 'UTC'), intDiv(toInt64($span), 2)))")" + log "$label ($lo .. $hi): src=$src > $MAX_ROWS rows, splitting in half at $mid" + process_range "$label" "$lo" "$mid" + process_range "$label" "$mid" "$hi" +} + +# Nothing to do on an empty table (min/max would return the epoch, not a real range). +ROWS="$(ch "SELECT count() FROM $SRC_TABLE")" +if [[ "$ROWS" == "0" ]]; then + log "Source table is empty — nothing to backfill." + exit 0 +fi + +preflight_capacity + +# backfill_start: the single anchor for BOTH the delta-insert and the deletion replay in step 2. Captured BEFORE the +# first INSERT so it covers every write during the (long) backfill, and persisted to --state-file so a resumed run +# reuses the ORIGINAL anchor. Re-minting a later anchor on resume would miss deletes that fired during the first run +# against already-copied rows. The operator MUST record it (also saved to the state file). +if [[ "$DRY_RUN" != "1" ]]; then + if [[ -s "$STATE_FILE" ]]; then + BACKFILL_START="$(cat "$STATE_FILE")" + log "REUSING backfill_start=$BACKFILL_START from $STATE_FILE (resume: original anchor kept)" + else + BACKFILL_START="$(ch "SELECT toString(now64(6))")" + printf '%s' "$BACKFILL_START" > "$STATE_FILE" + log "RECORD backfill_start=$BACKFILL_START (saved to $STATE_FILE; pass this to step 2: 000002_delta_and_deletion_replay.sql)" + fi +fi + +# The anchor is the Monday of the earliest row; the horizon is the Monday after the latest row. All week boundaries are +# computed from the anchor in ClickHouse (addWeeks), so there is no host-side date math or timezone ambiguity. +ANCHOR="$(ch "SELECT toString(toMonday(min(created_at))) FROM $SRC_TABLE")" +HORIZON="$(ch "SELECT toString(addWeeks(toMonday(max(created_at)), 1)) FROM $SRC_TABLE")" +LAST_WEEK="$(ch "SELECT dateDiff('week', toDate('$ANCHOR'), toDate('$HORIZON')) - 1")" +[[ -n "$TO_WEEK" ]] || TO_WEEK="$LAST_WEEK" + +log "Anchor Monday: $ANCHOR | horizon: $HORIZON | weeks: [$FROM_WEEK..$TO_WEEK] | max-rows/insert: $MAX_ROWS | pause: ${PAUSE_SECONDS}s | dry-run: $DRY_RUN" + +for (( week=FROM_WEEK; week<=TO_WEEK; week++ )); do + LO="$(ch "SELECT toString(addWeeks(toDate('$ANCHOR'), $week))") 00:00:00" + HI="$(ch "SELECT toString(addWeeks(toDate('$ANCHOR'), $((week + 1))))") 00:00:00" + process_range "week $week" "$LO" "$HI" +done + +log "Backfill complete for weeks [$FROM_WEEK..$TO_WEEK]. Proceed to step 2 (delta + deletion replay)." diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000001_backfill_traces_local_v2.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000001_backfill_traces_local_v2.sql new file mode 100644 index 00000000000..aad961d0b0c --- /dev/null +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000001_backfill_traces_local_v2.sql @@ -0,0 +1,75 @@ +-- runbook traces-local-v2-cutover — step 1 of 3: backfill (reference statement) +-- +-- This file is the SINGLE source of the backfill INSERT; ../backfill.sh reads it, substitutes the ${...} placeholders +-- (database, window bounds, block size) and runs it once per time sub-window — so the script and this reference never +-- drift. Run the migration through backfill.sh, never this file by hand. WINDOW_LO/WINDOW_HI are a created_at half-open +-- range the driver picks so each INSERT stays under its --max-rows-per-insert bound (see README "Batching and throttling"). +-- +-- Slicing rationale (created_at, not id / not workspace), delta and replay design: see ../../README.md. +-- Notes on the statement: +-- * The SOURCE is sliced by created_at (immutable across upserts, backed by a minmax skip index). The DESTINATION's +-- id_at partition is derived from each row's id independently of the slice. +-- * end_time and ttft are the two denullified columns: coalesce them to their sentinels (epoch / NaN). +-- * is_deleted is omitted so the new column defaults to 0. +-- * apply_deleted_mask stays at its default 1, so rows already lightweight-deleted on the source are skipped. +-- * No explicit ORDER BY: source and destination share ORDER BY (workspace_id, project_id, id), so rows are read in +-- destination order and parts form cleanly (see README "Why slice by created_at"). +-- * SETTINGS max_insert_block_size bounds the rows per part-forming block; peak insert memory is a small multiple of +-- the smaller of that and min_insert_block_size_bytes (256 MB default), which dominates for wide trace rows. + +INSERT INTO ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 ( + id, + workspace_id, + project_id, + name, + start_time, + end_time, + input, + output, + metadata, + tags, + created_at, + last_updated_at, + created_by, + last_updated_by, + error_info, + thread_id, + visibility_mode, + truncation_threshold, + input_slim, + output_slim, + ttft, + source, + environment +) +SELECT + id, + workspace_id, + project_id, + name, + start_time, + coalesce(end_time, toDateTime64('1970-01-01 00:00:00', 6)) AS end_time, + input, + output, + metadata, + tags, + created_at, + last_updated_at, + created_by, + last_updated_by, + error_info, + thread_id, + visibility_mode, + truncation_threshold, + input_slim, + output_slim, + coalesce(ttft, toFloat64('nan')) AS ttft, + source, + environment +FROM ${ANALYTICS_DB_DATABASE_NAME}.traces +WHERE created_at >= toDateTime64('${WINDOW_LO}', 9) + AND created_at < toDateTime64('${WINDOW_HI}', 9) +SETTINGS max_insert_block_size = ${MAX_INSERT_BLOCK_SIZE}; + +-- Per-window reconciliation is automated by backfill.sh (uniqExact of the dedup key, aborting on > 0.01% divergence); +-- fidelity QA across the whole copy is 000005 via verify.sh. Rollback before the EXCHANGE: rollback.sh --stage A. diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000002_delta_and_deletion_replay.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000002_delta_and_deletion_replay.sql new file mode 100644 index 00000000000..dedb4bad996 --- /dev/null +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000002_delta_and_deletion_replay.sql @@ -0,0 +1,173 @@ +-- runbook traces-local-v2-cutover — step 2 of 3: delta-insert + deletion replay +-- Run this only after the whole backfill (step 1) is complete and reconciled. + +-- Step 0: The SQL below (delta-insert + deletion replay) is the single source driven by ../delta_replay.sh, which reads +-- this file, substitutes the placeholders and runs it — never run this file by hand: +-- ../delta_replay.sh --database opik --backfill-start '2025-06-01 12:00:00.000000' +-- The surrounding config operations (buffer raise/restore) and the go/no-go checkpoint stay with the operator, where +-- situational awareness matters most — those are config/judgement, not SQL. clickhouse-client prints each statement's +-- elapsed time, which is the replay measurement in step 5. + +-- Step 1: BACKFILL_START is the timestamp captured BEFORE the backfill began. backfill.sh prints it at startup +-- ("RECORD backfill_start=..."); if you ran the backfill manually, use the now64(6) you captured before the first +-- INSERT. The delta and the replay both key off this single anchor, so writes during the whole backfill window are +-- covered. + +-- Step 2: Raise the async-insert buffer ceiling so the buffer can absorb the cutover window. Set +-- databaseAnalytics.asyncInsertBusyTimeoutMaxMs ~= 10000 (env ANALYTICS_DB_ASYNC_INSERT_BUSY_TIMEOUT_MAX_MS) and roll +-- it out (config push + rolling restart, OR a session-level SET on a dedicated cutover connection). Because +-- async_insert_use_adaptive_busy_timeout=1, this only widens the buffer while rows are queued. VERIFY the widening +-- took effect before proceeding — see README. + +-- Step 3: Delta-insert — re-copy every row written during the backfill window. Anchored on +-- created_at OR last_updated_at >= backfill_start (NOT last_updated_at alone): last_updated_at is client-supplied on the +-- batch-ingest path, so it is not a reliable "changed since" signal by itself. Every trace write sets EITHER a fresh +-- server created_at (batch-ingest path) OR a fresh server last_updated_at (create/update merge paths), so the union is +-- complete. ReplacingMergeTree dedups the re-copied rows against the backfilled ones (newest last_updated_at wins). +-- Uses ${BACKFILL_START}. SETTINGS max_insert_block_size bounds per-block memory as in step 1. +-- BATCHING: the delta covers only writes during the backfill window, not the whole table, so it is normally one +-- statement. If the backfill ran for days on a busy system and the delta is large, run it as two batched passes to keep +-- each INSERT bounded (both columns have a minmax skip index, so each pass prunes): +-- (a) created_at >= backfill_start -- batch by created_at sub-windows +-- (b) last_updated_at >= backfill_start AND created_at < backfill_start -- the updates-to-old-rows arm; batch by +-- last_updated_at sub-windows. (a) ∪ (b) equals the OR below, with no overlap. +INSERT INTO ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 ( + id, + workspace_id, + project_id, + name, + start_time, + end_time, + input, + output, + metadata, + tags, + created_at, + last_updated_at, + created_by, + last_updated_by, + error_info, + thread_id, + visibility_mode, + truncation_threshold, + input_slim, + output_slim, + ttft, + source, + environment +) +SELECT + id, + workspace_id, + project_id, + name, + start_time, + coalesce(end_time, toDateTime64('1970-01-01 00:00:00', 6)) AS end_time, + input, + output, + metadata, + tags, + created_at, + last_updated_at, + created_by, + last_updated_by, + error_info, + thread_id, + visibility_mode, + truncation_threshold, + input_slim, + output_slim, + coalesce(ttft, toFloat64('nan')) AS ttft, + source, + environment +FROM ${ANALYTICS_DB_DATABASE_NAME}.traces +WHERE created_at >= toDateTime64('${BACKFILL_START}', 6) + OR last_updated_at >= toDateTime64('${BACKFILL_START}', 6) +SETTINGS max_insert_block_size = ${MAX_INSERT_BLOCK_SIZE}; + +-- Step 4: Deletion replay — remove from the destination every row that was deleted on the source since backfill_start +-- AND is still deleted there. Two branches, mirroring the product's two delete paths (TraceService.delete): a delete +-- resolves each trace's owning project and deletes per project; ids it cannot resolve fall back to a workspace-scoped +-- delete (TraceDAO DELETE_BY_ID with no project filter). The bridge records the first with the project and the second +-- with an EMPTY project_id (DeletionEventDAO: "project_id is empty for workspace-scoped source tables"). So: +-- * events WITH a project -> match the FULL key (workspace_id, project_id, id). Exact, prunes on the destination +-- primary key, and correct even when an id is reused across projects (ids are not globally unique). +-- * events WITHOUT a project -> match (workspace_id, id). A faithful mirror of the source's workspace-scoped delete. +-- RESURRECTION GUARD (the `NOT IN traces` arm): a trace can be deleted and then re-created/updated under the same id +-- during the window (client-supplied ids; the delete is a mask, a newer insert wins under FINAL). Such an id is bridged +-- as deleted but is LIVE again on the source, and the backfill/delta already copied its live version. Deleting it by key +-- would drop a row that is live on the source — silent data loss. So each branch deletes only ids that are NOT currently +-- live on the source (mask-honored). The `id IN (deleted_ids since anchor)` bound keeps the source lookup pruned to the +-- window's deletes (id skip index), not a full scan. +-- allow_nondeterministic_mutations: a lightweight DELETE with cross-table subqueries is flagged nondeterministic, but +-- deletion_events_local and traces are replicated and identical on every node and the window predicate is fixed, so the +-- subqueries resolve to the same set on every replica. Idempotent (never masks a live-on-source id, so re-runs converge). +-- lightweight_deletes_sync = 2: block until the delete mutation has completed on EVERY replica, not just the one that +-- accepted it. The mutation is otherwise asynchronous, so without this the verify step (and the EXCHANGE) could run +-- against a replica where the mask is not yet applied — a false mismatch, or worse an incomplete cutover. +-- Uses ${BACKFILL_START}. Retention is disabled everywhere (see step 6), so this is user-scale volume — a single +-- mutation. If it is ever large (e.g. retention enabled), bound each mutation by a partition predicate and loop the +-- weeks, e.g. AND toMonday(id_at) = toDate(''). +DELETE FROM ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 +WHERE ( + (workspace_id, project_id, id) IN ( + SELECT + workspace_id, + toFixedString(project_id, 36), + toFixedString(deleted_id, 36) + FROM ${ANALYTICS_DB_DATABASE_NAME}.deletion_events_local + WHERE source_table = 'traces' + AND event_time >= toDateTime64('${BACKFILL_START}', 6) + AND project_id != '' + ) + AND (workspace_id, project_id, id) NOT IN ( + SELECT + workspace_id, + project_id, + id + FROM ${ANALYTICS_DB_DATABASE_NAME}.traces + WHERE id IN ( + SELECT toFixedString(deleted_id, 36) + FROM ${ANALYTICS_DB_DATABASE_NAME}.deletion_events_local + WHERE source_table = 'traces' + AND event_time >= toDateTime64('${BACKFILL_START}', 6) + ) + ) +) +OR ( + (workspace_id, id) IN ( + SELECT + workspace_id, + toFixedString(deleted_id, 36) + FROM ${ANALYTICS_DB_DATABASE_NAME}.deletion_events_local + WHERE source_table = 'traces' + AND event_time >= toDateTime64('${BACKFILL_START}', 6) + AND project_id = '' + ) + AND (workspace_id, id) NOT IN ( + SELECT + workspace_id, + id + FROM ${ANALYTICS_DB_DATABASE_NAME}.traces + WHERE id IN ( + SELECT toFixedString(deleted_id, 36) + FROM ${ANALYTICS_DB_DATABASE_NAME}.deletion_events_local + WHERE source_table = 'traces' + AND event_time >= toDateTime64('${BACKFILL_START}', 6) + ) + ) +) +SETTINGS allow_nondeterministic_mutations = 1, + lightweight_deletes_sync = 2; + +-- Step 5: Measure the replay. Compare its wall time against the buffer window (must fit with margin — acceptance +-- criterion). Re-run steps 3-4 if new rows/deletes accumulated during the replay itself; convergence is fast because +-- the buffer is holding new writes. + +-- Step 6 (retention — see README): Data Retention is disabled in every deployment (RETENTION_ENABLED=false), so the +-- retention delete path does not fire during the cutover. The only deletes in this window are user-initiated, and those +-- ARE captured by the bridge. If retention is ever enabled, pause it for the window (or land retention-path capture). + +-- rollback: none for the delta-insert (it only adds newest versions that ReplacingMergeTree dedups); the replay is +-- idempotent. If aborting the cutover here, TRUNCATE traces_local_v2 (step 1 rollback) and restore the buffer +-- ceiling (step 2, reverse). The live `traces` table is still untouched until the EXCHANGE in step 3. diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000003_exchange_and_wrap.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000003_exchange_and_wrap.sql new file mode 100644 index 00000000000..e89fc94673c --- /dev/null +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000003_exchange_and_wrap.sql @@ -0,0 +1,39 @@ +-- runbook traces-local-v2-cutover — step 3 of 3: EXCHANGE + Distributed wrap (reference statements) +-- +-- ../exchange_and_wrap.sh drives this: it records cutover_start, runs the `exchange` block, and (unless --skip-wrap) +-- the `wrap` block. Run it right after step 2's delta + replay, while the async-insert buffer is still holding writes. +-- Do NOT run this whole file wholesale — the driver runs one marked block at a time. Buffer knob: raise +-- databaseAnalytics.asyncInsertBusyTimeoutMaxMs before the cutover and unset it after (a backend-config action, not SQL). +-- +-- cutover_start is a now64(6) captured RIGHT BEFORE the EXCHANGE; a rollback after this point replays deletes that fired +-- on the new live table since then. exchange_and_wrap.sh captures and prints it; record it for the rollback. + +-- >>> BEGIN exchange +-- The atomic swap: `traces` now refers to the partitioned data. The displaced old data lands under `traces_local_v2` +-- momentarily, then is renamed to `traces_pre_cutover_backup` so its name marks it as the retained pre-cutover backup, +-- not the "v2" successor (rationale: README "Naming and the parked backup"). Requires an Atomic database (default). If +-- the Liquibase ClickHouse extension cannot execute EXCHANGE ON CLUSTER in the downtime-based path, use the fallback +-- RENAME sequence in the README instead. +EXCHANGE TABLES ${ANALYTICS_DB_DATABASE_NAME}.traces AND ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 ON CLUSTER '{cluster}'; + +RENAME TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 TO ${ANALYTICS_DB_DATABASE_NAME}.traces_pre_cutover_backup ON CLUSTER '{cluster}'; +-- >>> END exchange + +-- >>> BEGIN wrap +-- Sharding-ready wrap: rename the partitioned table to *_local and front it with a Distributed table keyed on +-- sipHash64(project_id). Transparent on a single shard; switching on sharding later is config-only. The {cluster} macro +-- (not the literal 'cluster') keeps the DDL portable; it is resolved server-side. +-- HARD PREREQUISITE: a Distributed table supports SELECT and INSERT but NOT mutations — a lightweight DELETE returns +-- "DELETE query is not supported" (code 36) and ALTER ... DELETE returns "Distributed doesn't support mutations" +-- (code 48). So the product's delete-by-id AND retention deletes both break the moment this wrap is applied. Do NOT run +-- the wrap until those DAO paths target `traces_local` (see README "The Distributed wrap"). The EXCHANGE above is the +-- data cutover and leaves `traces` a MergeTree where deletes still work; the wrap is a separate, gated step. +RENAME TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces TO ${ANALYTICS_DB_DATABASE_NAME}.traces_local ON CLUSTER '{cluster}'; + +CREATE TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces ON CLUSTER '{cluster}' AS ${ANALYTICS_DB_DATABASE_NAME}.traces_local + ENGINE = Distributed('{cluster}', '${ANALYTICS_DB_DATABASE_NAME}', 'traces_local', sipHash64(project_id)); +-- >>> END wrap + +-- After the wrap: restore the buffer ceiling (unset asyncInsertBusyTimeoutMaxMs), verify (README "Verifying the +-- migration"), and keep `traces_pre_cutover_backup` (the parked old data) until the soak completes. Rollback: the +-- 000004_rollback_* files via ../rollback.sh. diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_reverse_replay.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_reverse_replay.sql new file mode 100644 index 00000000000..67a6ca52bb8 --- /dev/null +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_reverse_replay.sql @@ -0,0 +1,37 @@ +-- runbook traces-local-v2-cutover — ROLLBACK reverse-replay (driven by ../rollback.sh after stage B or C) +-- +-- Re-applies the deletes that fired on the successor since cutover_start onto the restored original `traces`, so they do +-- not resurrect. Two branches, like the forward replay in 000002: events with a project match the full key; events from +-- the product's workspace-scoped delete fallback (project_id = '') match (workspace_id, id). Shared by stages B and C +-- (run right after the swap/promote), never on its own. If the set is ever large, bound it with +-- AND toMonday(id_at) = toDate('') and loop the weeks. +-- +-- Deliberately NO resurrection guard (the `AND ... NOT IN traces` arm the forward replay in 000002 carries). Do NOT add +-- one here: rollback abandons all post-cutover writes on the successor (they are being discarded) while still honoring +-- post-cutover deletes. `traces` here is the RESTORED ORIGINAL, so a bridged id is present as its pre-cutover version; a +-- liveness guard would spare it and thereby UNDO the user's post-cutover delete (resurrecting stale content). Masking it +-- unconditionally is the correct rollback semantics. +DELETE FROM ${ANALYTICS_DB_DATABASE_NAME}.traces +WHERE (workspace_id, project_id, id) IN ( + SELECT + workspace_id, + toFixedString(project_id, 36), + toFixedString(deleted_id, 36) + FROM ${ANALYTICS_DB_DATABASE_NAME}.deletion_events_local + WHERE source_table = 'traces' + AND event_time >= toDateTime64('${CUTOVER_START}', 6) + AND project_id != '' +) +OR (workspace_id, id) IN ( + SELECT + workspace_id, + toFixedString(deleted_id, 36) + FROM ${ANALYTICS_DB_DATABASE_NAME}.deletion_events_local + WHERE source_table = 'traces' + AND event_time >= toDateTime64('${CUTOVER_START}', 6) + AND project_id = '' +) +-- lightweight_deletes_sync = 2: wait for the mutation on every replica so the restored `traces` is consistent +-- cluster-wide before the rollback is declared done (see 000002 for the rationale). +SETTINGS allow_nondeterministic_mutations = 1, + lightweight_deletes_sync = 2; diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_a_discard_shadow.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_a_discard_shadow.sql new file mode 100644 index 00000000000..e603f190e0b --- /dev/null +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_a_discard_shadow.sql @@ -0,0 +1,13 @@ +-- runbook traces-local-v2-cutover — ROLLBACK stage A: discard the shadow (driven by ../rollback.sh --stage A) +-- +-- Use when the backfill/delta ran but the EXCHANGE did NOT. The live `traces` was never touched by the backfill, so +-- there is nothing to restore; this only discards the disposable shadow copy so a re-attempt starts clean. backfill.sh +-- is idempotent (a re-run skips windows already present), so even this is optional — it just reclaims space. +-- +-- SAFETY: this is the only rollback file containing a TRUNCATE, isolated so it can never run alongside the EXCHANGE/DROP +-- of the other stages. It targets `traces_local_v2` (the disposable shadow successor, only ever holds copied-in data +-- pre-EXCHANGE). rollback.sh asserts the pre-EXCHANGE topology (traces = original schema, not Distributed) before +-- running it, so it cannot fire post-EXCHANGE — by which point the old original has been renamed to +-- `traces_pre_cutover_backup` and `traces_local_v2` no longer exists, so there is nothing here that could touch the +-- backup. +TRUNCATE TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 ON CLUSTER '{cluster}'; diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_b_exchange_back.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_b_exchange_back.sql new file mode 100644 index 00000000000..591e4b3fa0c --- /dev/null +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_b_exchange_back.sql @@ -0,0 +1,11 @@ +-- runbook traces-local-v2-cutover — ROLLBACK stage B: swap the tables back (driven by ../rollback.sh --stage B) +-- +-- Use when the EXCHANGE ran but the wrap did NOT. `traces` holds the successor data and `traces_pre_cutover_backup` +-- parks the original. Swap them back (so `traces` is the original live again), then rename the now-parked successor +-- back to `traces_local_v2` — restoring the canonical state (traces = original live, traces_local_v2 = successor +-- parked), identical to pre-EXCHANGE. Non-destructive. rollback.sh runs the reverse-replay +-- (000004_rollback_reverse_replay.sql) right after this so deletes since cutover_start do not resurrect. rollback.sh +-- asserts the post-EXCHANGE, pre-wrap topology (traces = successor schema, not Distributed) before running it. +EXCHANGE TABLES ${ANALYTICS_DB_DATABASE_NAME}.traces AND ${ANALYTICS_DB_DATABASE_NAME}.traces_pre_cutover_backup ON CLUSTER '{cluster}'; + +RENAME TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces_pre_cutover_backup TO ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 ON CLUSTER '{cluster}'; diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql new file mode 100644 index 00000000000..e58a7e2e1ec --- /dev/null +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql @@ -0,0 +1,16 @@ +-- runbook traces-local-v2-cutover — ROLLBACK stage C: promote the original back (driven by ../rollback.sh --stage C) +-- +-- Use when the wrap ran. Post-wrap topology: `traces` is a Distributed wrapper over `traces_local` (successor data); +-- `traces_pre_cutover_backup` parks the original. Drop the wrapper (it stores no data — a routing definition only), +-- then in one atomic RENAME promote the original back to `traces` and park the successor data back under +-- `traces_local_v2` — ending in the canonical state (traces = original live, traces_local_v2 = successor parked), with +-- no leftover names. +-- +-- rollback.sh runs the reverse-replay (000004_rollback_reverse_replay.sql) right after this so deletes since +-- cutover_start do not resurrect, and asserts the post-wrap topology (traces = Distributed) before running it. +DROP TABLE IF EXISTS ${ANALYTICS_DB_DATABASE_NAME}.traces ON CLUSTER '{cluster}' SYNC; + +RENAME TABLE + ${ANALYTICS_DB_DATABASE_NAME}.traces_pre_cutover_backup TO ${ANALYTICS_DB_DATABASE_NAME}.traces, + ${ANALYTICS_DB_DATABASE_NAME}.traces_local TO ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 + ON CLUSTER '{cluster}'; diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000005_verify_migration.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000005_verify_migration.sql new file mode 100644 index 00000000000..cf147a3de19 --- /dev/null +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000005_verify_migration.sql @@ -0,0 +1,187 @@ +-- runbook traces-local-v2-cutover — QA: normalized fidelity compare of one created_at window (reference statements) +-- +-- Proves the copy altered no data by comparing a NORMALIZED fingerprint of the deduped, live rows on the old-schema and +-- new-schema tables. The rows are not byte-identical (end_time NULL -> epoch sentinel, ttft NULL -> NaN sentinel, +-- timestamps nanosecond -> microsecond), so each side is canonicalized to the same value for a faithfully-migrated row: +-- * timestamps as their microsecond epoch (source ns truncated to us, matching the copy); +-- * absent end_time -> 0 (source NULL; dest epoch); +-- * absent ttft -> the token 'nan' (source NULL; dest NaN); +-- * enums / project_id via toString; id in every row hash so a swap can't cancel; +-- * tags joined with a '\x1f' (ASCII Unit Separator) delimiter: the delimiter is what makes a tag-BOUNDARY change +-- detectable — without it ['a','b'] and ['ab'] both concatenate to 'ab' and hash identically. \x1f is a C0 control +-- char purpose-built as a field separator that real (printable) tag text never contains, so it cannot collide with +-- tag content the way ',' or ' ' could. +-- FINAL collapses ReplacingMergeTree versions to the winner; the default apply_deleted_mask excludes deleted rows. +-- sum() is order-independent (no sort) and, unlike groupBitXor, does not cancel a colliding pair within a table; with +-- count() it detects any changed / missing / extra row. An empty window sums to NULL on the Nullable-typed old side but 0 +-- on the new, so the verdict uses ifNull(_, 0) and a count guard — empty vs empty is a match, empty vs non-empty is not. +-- cityHash64 (not sipHash64): both sides are hashed live on the same instance, so a fast non-cryptographic 64-bit hash is +-- enough — sipHash64's adversarial-collision resistance would only add CPU here, and cross-build portability does not +-- matter because we never compare a stored hash against a later build. Summed 64-bit hashes miss a real difference with +-- probability ~2^-64 per window. Materialized/derived columns and is_deleted are excluded — recomputed, not migrated. +-- +-- ../verify.sh is the single driver: it reads this file and runs the `compare` block once per created_at week (optionally +-- sampled), parsing the single verdict row; with --drill-down it runs the `drill-down` block for a week that reported +-- ok=0. Run QA through verify.sh, never this file by hand. +-- +-- OLD_TABLE is the old-schema table (Nullable, nanosecond) and NEW_TABLE the new-schema one (sentinels, microsecond). +-- Before the EXCHANGE: OLD_TABLE=traces, NEW_TABLE=traces_local_v2 (the successor being built). After it, `traces` is +-- the new schema and the old data is parked as `traces_pre_cutover_backup` — set OLD_TABLE=traces_pre_cutover_backup, +-- NEW_TABLE=traces. SAMPLE_MOD=1 compares every row; SAMPLE_MOD=100 compares a deterministic ~1% id sample (same rows +-- on both sides) when a full pass is infeasible. + +-- >>> BEGIN compare +WITH + src AS ( + SELECT + count() AS c, + sum(cityHash64( + id, + workspace_id, + toString(project_id), + name, + toUnixTimestamp64Micro(toDateTime64(start_time, 6)), + coalesce(toUnixTimestamp64Micro(toDateTime64(end_time, 6)), toInt64(0)), + input, + output, + metadata, + arrayStringConcat(tags, '\x1f'), + toUnixTimestamp64Micro(toDateTime64(created_at, 6)), + toUnixTimestamp64Micro(toDateTime64(last_updated_at, 6)), + created_by, + last_updated_by, + error_info, + thread_id, + toString(visibility_mode), + truncation_threshold, + input_slim, + output_slim, + if(ttft IS NULL, 'nan', toString(ttft)), + toString(source), + toString(environment))) AS h + FROM ${ANALYTICS_DB_DATABASE_NAME}.${OLD_TABLE} FINAL + WHERE created_at >= toDateTime64('${WINDOW_LO}', 9) + AND created_at < toDateTime64('${WINDOW_HI}', 9) + AND cityHash64(id) % ${SAMPLE_MOD} = 0 + ), + dst AS ( + SELECT + count() AS c, + sum(cityHash64( + id, + workspace_id, + toString(project_id), + name, + toUnixTimestamp64Micro(start_time), + toUnixTimestamp64Micro(end_time), + input, + output, + metadata, + arrayStringConcat(tags, '\x1f'), + toUnixTimestamp64Micro(created_at), + toUnixTimestamp64Micro(last_updated_at), + created_by, + last_updated_by, + error_info, + thread_id, + toString(visibility_mode), + truncation_threshold, + input_slim, + output_slim, + if(isNaN(ttft), 'nan', toString(ttft)), + toString(source), + toString(environment))) AS h + FROM ${ANALYTICS_DB_DATABASE_NAME}.${NEW_TABLE} FINAL + WHERE created_at >= toDateTime64('${WINDOW_LO}', 6) + AND created_at < toDateTime64('${WINDOW_HI}', 6) + AND cityHash64(id) % ${SAMPLE_MOD} = 0 + ) +SELECT + src.c AS src_rows, + dst.c AS dst_rows, + ifNull(src.h, 0) AS src_checksum, + ifNull(dst.h, 0) AS dst_checksum, + (src.c = dst.c AND ifNull(src.h, 0) = ifNull(dst.h, 0)) AS ok +FROM src, dst; +-- >>> END compare + +-- >>> BEGIN drill-down +-- Lists up to 100 keys that differ or exist on one side only, for a window the compare reported as ok=0. +-- join_use_nulls = 1 is required for correctness: by default ClickHouse fills an unmatched FULL OUTER JOIN side with the +-- column's DEFAULT (0 for the UInt64 hash), not NULL — which would make a row missing on one side indistinguishable from +-- a real hash of 0 and leave the `IS NULL` predicates below dead. With it, the absent side is NULL, so `src_hash IS NULL +-- OR dst_hash IS NULL` correctly flags a missing row and prints it as NULL. +SELECT + key, + src_hash, + dst_hash +FROM ( + SELECT + (workspace_id, project_id, id) AS key, + cityHash64( + id, + workspace_id, + toString(project_id), + name, + toUnixTimestamp64Micro(toDateTime64(start_time, 6)), + coalesce(toUnixTimestamp64Micro(toDateTime64(end_time, 6)), toInt64(0)), + input, + output, + metadata, + arrayStringConcat(tags, '\x1f'), + toUnixTimestamp64Micro(toDateTime64(created_at, 6)), + toUnixTimestamp64Micro(toDateTime64(last_updated_at, 6)), + created_by, + last_updated_by, + error_info, + thread_id, + toString(visibility_mode), + truncation_threshold, + input_slim, + output_slim, + if(ttft IS NULL, 'nan', toString(ttft)), + toString(source), + toString(environment)) AS src_hash + FROM ${ANALYTICS_DB_DATABASE_NAME}.${OLD_TABLE} FINAL + WHERE created_at >= toDateTime64('${WINDOW_LO}', 9) + AND created_at < toDateTime64('${WINDOW_HI}', 9) + AND cityHash64(id) % ${SAMPLE_MOD} = 0 +) AS s +FULL OUTER JOIN ( + SELECT + (workspace_id, project_id, id) AS key, + cityHash64( + id, + workspace_id, + toString(project_id), + name, + toUnixTimestamp64Micro(start_time), + toUnixTimestamp64Micro(end_time), + input, + output, + metadata, + arrayStringConcat(tags, '\x1f'), + toUnixTimestamp64Micro(created_at), + toUnixTimestamp64Micro(last_updated_at), + created_by, + last_updated_by, + error_info, + thread_id, + toString(visibility_mode), + truncation_threshold, + input_slim, + output_slim, + if(isNaN(ttft), 'nan', toString(ttft)), + toString(source), + toString(environment)) AS dst_hash + FROM ${ANALYTICS_DB_DATABASE_NAME}.${NEW_TABLE} FINAL + WHERE created_at >= toDateTime64('${WINDOW_LO}', 6) + AND created_at < toDateTime64('${WINDOW_HI}', 6) + AND cityHash64(id) % ${SAMPLE_MOD} = 0 +) AS d USING (key) +WHERE src_hash != dst_hash + OR src_hash IS NULL + OR dst_hash IS NULL +LIMIT 100 +SETTINGS join_use_nulls = 1; +-- >>> END drill-down diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/delta_replay.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/delta_replay.sh new file mode 100755 index 00000000000..cf4421a9a46 --- /dev/null +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/delta_replay.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# +# Driver for step 2 of the buffered traces cutover: delta-insert + deletion replay (runbook: ../README.md). +# +# Reads db-app-analytics/000002_delta_and_deletion_replay.sql (the single source), substitutes the placeholders and runs +# it. Run it after backfill.sh, then verify.sh, then exchange_and_wrap.sh. +# +# Connection: clickhouse-client env vars (CLICKHOUSE_HOST, CLICKHOUSE_PORT, CLICKHOUSE_USER, CLICKHOUSE_PASSWORD). +# +# Options: +# --database NAME analytics database (e.g. opik). Required. +# --backfill-start TS the anchor printed by backfill.sh ("RECORD backfill_start=..."). Required. +# --max-insert-block-size N SETTINGS max_insert_block_size for the delta INSERT. Default 1048576. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SQL_FILE="$SCRIPT_DIR/db-app-analytics/000002_delta_and_deletion_replay.sql" + +DATABASE="" +BACKFILL_START="" +MAX_INSERT_BLOCK_SIZE=1048576 + +while [[ $# -gt 0 ]]; do + case "$1" in + --database) DATABASE="$2"; shift 2 ;; + --backfill-start) BACKFILL_START="$2"; shift 2 ;; + --max-insert-block-size) MAX_INSERT_BLOCK_SIZE="$2"; shift 2 ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done + +[[ -n "$DATABASE" ]] || { echo "ERROR: --database is required" >&2; exit 2; } +[[ -n "$BACKFILL_START" ]] || { echo "ERROR: --backfill-start is required (printed by backfill.sh)" >&2; exit 2; } +[[ -f "$SQL_FILE" ]] || { echo "ERROR: cannot find $SQL_FILE" >&2; exit 2; } + +echo "Reminder: raise databaseAnalytics.asyncInsertBusyTimeoutMaxMs before this step (backend config, not SQL) and" +echo "restore it after the EXCHANGE." + +sql="$(cat "$SQL_FILE")" +sql="${sql//'${ANALYTICS_DB_DATABASE_NAME}'/$DATABASE}" +sql="${sql//'${BACKFILL_START}'/$BACKFILL_START}" +sql="${sql//'${MAX_INSERT_BLOCK_SIZE}'/$MAX_INSERT_BLOCK_SIZE}" +clickhouse-client --database "$DATABASE" --multiquery --query "$sql" + +echo "Delta + deletion replay complete. Run verify.sh before the EXCHANGE." diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/estimate.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/estimate.sh new file mode 100755 index 00000000000..36fbaf92531 --- /dev/null +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/estimate.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# +# Ballpark ETA for the backfill — the dominant, longest-running step (runbook: ../README.md, "Batching and throttling"). +# +# It reads the live size of `traces`, estimates this instance's copy throughput with an on-the-fly READ probe (no table +# is created — a bounded `SELECT ... FORMAT Null` that reads and decompresses a sample, timed), and combines them with +# the batch/throttle config to project how long backfill.sh will take. The number is a planning ballpark, not a +# guarantee. +# +# The probe measures READ+decompress throughput only. A real `INSERT ... SELECT` also pays write+compression (ZSTD on the +# wide text columns is the bottleneck) and background merges, so the copy is slower than a bare read — that gap is folded +# in by --write-cost-factor. For an exact figure, time one real window with backfill.sh and pass its rows/sec via +# --rows-per-sec. +# +# Connection: clickhouse-client env vars (CLICKHOUSE_HOST, CLICKHOUSE_PORT, CLICKHOUSE_USER, CLICKHOUSE_PASSWORD). +# +# Options: +# --database NAME analytics database (e.g. opik). Required. +# --max-rows-per-insert R the value you will pass to backfill.sh; sets how many windows the copy splits into. +# Default 2000000 (matches backfill.sh). +# --pause-seconds S backfill.sh --pause-seconds; added once per window as merge-catch-up idle time. Default 0. +# --probe-rows N rows to read in the throughput probe (SELECT ... LIMIT N FORMAT Null). Larger = steadier +# estimate but a heavier probe. Default 200000. Ignored if --rows-per-sec is given. +# --write-cost-factor F multiplier applied to the read-probe time to account for the unmeasured write+compression +# +merge cost of a real copy. Default 2.5 (wide ZSTD-compressed rows are write-bound). Set 1 +# to report the raw read-only floor. Ignored if --rows-per-sec is given. +# --rows-per-sec R skip the probe and use this measured COPY throughput directly (e.g. from a real backfill +# window). When set, --write-cost-factor is not applied. + +set -euo pipefail + +DATABASE="" +MAX_ROWS=2000000 +PAUSE_SECONDS=0 +PROBE_ROWS=200000 +WRITE_COST_FACTOR=2.5 +ROWS_PER_SEC="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --database) DATABASE="$2"; shift 2 ;; + --max-rows-per-insert) MAX_ROWS="$2"; shift 2 ;; + --pause-seconds) PAUSE_SECONDS="$2"; shift 2 ;; + --probe-rows) PROBE_ROWS="$2"; shift 2 ;; + --write-cost-factor) WRITE_COST_FACTOR="$2"; shift 2 ;; + --rows-per-sec) ROWS_PER_SEC="$2"; shift 2 ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done + +[[ -n "$DATABASE" ]] || { echo "ERROR: --database is required" >&2; exit 2; } + +ch() { + clickhouse-client --database "$DATABASE" --query "$1" +} + +# Physical rows to copy (count() honors the deleted-row mask, so masked rows are excluded — as the backfill excludes +# them) and the projected window count: each week splits until every sub-window is <= MAX_ROWS, so a week of `cnt` rows +# yields ~ceil(cnt / MAX_ROWS) inserts. Both come from one grouped scan. +read -r TOTAL_ROWS EST_WINDOWS WEEKS <<< "$(ch " + SELECT + sum(cnt), + sum(if(cnt = 0, 0, toUInt64(ceil(cnt / $MAX_ROWS)))), + count() + FROM ( + SELECT + toMonday(created_at) AS wk, + count() AS cnt + FROM traces + GROUP BY wk + ) + FORMAT TSV +")" + +if [[ -z "$TOTAL_ROWS" || "$TOTAL_ROWS" == "0" ]]; then + echo "Source table 'traces' is empty — nothing to backfill." + exit 0 +fi + +SIZE="$(ch "SELECT formatReadableSize(sum(bytes_on_disk)) FROM system.parts WHERE database = '$DATABASE' AND table = 'traces' AND active")" + +# Disk headroom. The backfill writes a full second physical copy of `traces` (peak ~2x on-disk, more counting merge +# scratch), so free space must clear that before starting. This is a whole-node total-space floor; on tiered storage +# validate per-volume (hot) headroom too, since new parts land on the hot volume before they tier. +TRACES_BYTES="$(ch "SELECT sum(bytes_on_disk) FROM system.parts WHERE database = '$DATABASE' AND table = 'traces' AND active")" +FREE_BYTES="$(ch "SELECT sum(free_space) FROM system.disks")" +awk -v t="$TRACES_BYTES" -v f="$FREE_BYTES" 'BEGIN { + g = 1073741824 + printf "Disk: traces on-disk %.1f GiB, node free %.1f GiB, ~2x needed %.1f GiB%s\n", + t/g, f/g, t*2/g, (f < t*2 ? " *** below 2x — free space before backfilling ***" : "") +}' + +# Effective COPY throughput. If the caller measured a real one, use it as-is. Otherwise probe READ throughput with an +# on-the-fly SELECT ... FORMAT Null (bounded by LIMIT, reads and decompresses ~PROBE_ROWS rows, no table created) and +# derate it by --write-cost-factor to approximate the copy's added write/merge cost. +FACTOR_NOTE="" +if [[ -z "$ROWS_PER_SEC" ]]; then + PROBE_ACTUAL="$(awk -v a="$PROBE_ROWS" -v b="$TOTAL_ROWS" 'BEGIN { print (a < b) ? a : b }')" + echo "Probing read throughput with a $PROBE_ACTUAL-row SELECT ... FORMAT Null (no table created)..." + ELAPSED="$(clickhouse-client --database "$DATABASE" --time --query \ + "SELECT * FROM traces LIMIT $PROBE_ROWS FORMAT Null" 2>&1 1>/dev/null)" + READ_RPS="$(awk -v r="$PROBE_ACTUAL" -v t="$ELAPSED" 'BEGIN { print (t > 0) ? r / t : 0 }')" + [[ "$(awk -v v="$READ_RPS" 'BEGIN { print (v > 0) ? 1 : 0 }')" == "1" ]] || { + echo "ERROR: probe measured 0 rows/sec (elapsed='$ELAPSED'). Pass --rows-per-sec." >&2 + exit 1 + } + ROWS_PER_SEC="$(awk -v r="$READ_RPS" -v f="$WRITE_COST_FACTOR" 'BEGIN { print r / f }')" + echo "Read throughput: ~$(printf '%.0f' "$READ_RPS") rows/sec ($PROBE_ACTUAL rows in ${ELAPSED}s)." + FACTOR_NOTE=" (read ${READ_RPS%.*}/s derated by write-cost-factor ${WRITE_COST_FACTOR})" +fi + +# ETA = copy time + total throttle idle. Throttle idle is one --pause-seconds per window (a fresh run inserts every +# window; a resumed run inserts fewer, so this is an upper bound). +awk -v rows="$TOTAL_ROWS" -v windows="$EST_WINDOWS" -v weeks="$WEEKS" -v rps="$ROWS_PER_SEC" \ + -v pause="$PAUSE_SECONDS" -v maxrows="$MAX_ROWS" -v size="$SIZE" -v note="$FACTOR_NOTE" ' +function hms(s, h, m) { + h = int(s / 3600); s -= h * 3600 + m = int(s / 60); s -= m * 60 + return sprintf("%dh %dm %ds", h, m, int(s)) +} +BEGIN { + copy = rows / rps + idle = windows * pause + total = copy + idle + printf "\n" + printf "Backfill estimate for %s (%s rows across %d weeks)\n", size, rows, weeks + printf " config: max-rows-per-insert=%d, pause-seconds=%d\n", maxrows, pause + printf " windows: ~%d inserts\n", windows + printf " copy rate: ~%.0f rows/sec%s\n", rps, note + printf " copy time: %s\n", hms(copy) + printf " throttle: %s (%d windows x %ds)\n", hms(idle), windows, pause + printf " TOTAL ETA: %s\n", hms(total) + printf "\nBallpark only. The copy rate is derived from a read probe + write-cost-factor; for accuracy, time one real\n" + printf "window with backfill.sh and pass its rows/sec via --rows-per-sec.\n" +}' diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh new file mode 100755 index 00000000000..3576bc17bab --- /dev/null +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +# +# Driver for step 3 of the buffered traces cutover: EXCHANGE + Distributed wrap (runbook: ../README.md). +# +# Captures and prints cutover_start (needed by rollback.sh if you roll back after this), runs the `exchange` block of +# db-app-analytics/000003_exchange_and_wrap.sql, then the `wrap` block unless --skip-wrap. Run it right after the delta + +# replay + verify, while the async-insert buffer is still holding writes. +# +# Guarded like rollback.sh: it asserts the live `traces` topology matches the requested action before touching anything, +# so a re-run cannot silently swap the tables back, and a partial EXCHANGE (swap done, post-swap RENAME not) is detected +# with the command to finish it. +# +# Connection: clickhouse-client env vars (CLICKHOUSE_HOST, CLICKHOUSE_PORT, CLICKHOUSE_USER, CLICKHOUSE_PASSWORD). +# +# Options: +# --database NAME analytics database (e.g. opik). Required. +# --skip-wrap stop after the EXCHANGE (the data cutover); do not create the Distributed wrapper. Use this until +# the delete/read DAOs are sharding-aware — a lightweight DELETE against a Distributed table is +# unsupported. +# --wrap-only run ONLY the Distributed wrap on the already-EXCHANGEd `traces`, skipping the EXCHANGE (and the +# cutover_start capture — the data cutover already happened). This is the deferred second half of a +# prior `--skip-wrap` run: apply it once the delete/read DAOs are sharding-aware. Mutually exclusive +# with --skip-wrap. The replication-settle gate still runs first. +# --force skip the replication-settle gate. By default the swap aborts while any replica still +# has replication-queue backlog or an unfinished mutation on traces / traces_local_v2, since a +# behind replica would swap in an incomplete table. Use only if settlement is confirmed out of band. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SQL_FILE="$SCRIPT_DIR/db-app-analytics/000003_exchange_and_wrap.sql" + +DATABASE="" +SKIP_WRAP=0 +WRAP_ONLY=0 +FORCE=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --database) DATABASE="$2"; shift 2 ;; + --skip-wrap) SKIP_WRAP=1; shift ;; + --wrap-only) WRAP_ONLY=1; shift ;; + --force) FORCE=1; shift ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done + +[[ -n "$DATABASE" ]] || { echo "ERROR: --database is required" >&2; exit 2; } +[[ -f "$SQL_FILE" ]] || { echo "ERROR: cannot find $SQL_FILE" >&2; exit 2; } +[[ "$SKIP_WRAP" == "1" && "$WRAP_ONLY" == "1" ]] && { echo "ERROR: --skip-wrap and --wrap-only are mutually exclusive" >&2; exit 2; } + +ch() { + clickhouse-client --database "$DATABASE" --query "$1" +} + +# Single scalar (empty string if the object does not exist). +traces_engine() { + ch "SELECT engine FROM system.tables WHERE database = '$DATABASE' AND name = '$1'" +} +traces_endtime_type() { + ch "SELECT type FROM system.columns WHERE database = '$DATABASE' AND table = '$1' AND name = 'end_time'" +} + +# Topology precondition. The EXCHANGE must run ONLY against the pre-EXCHANGE estate (traces = original schema). Running +# it a second time would silently swap the tables back — the successor gets parked and the old original goes live again, +# with no error — so this refuses instead. It also detects the split state where the EXCHANGE committed but the +# post-swap RENAME did not, and prints the one command that completes it. Signals are the same ones rollback.sh keys on: +# the `traces` engine and its end_time nullability (original = Nullable; successor = non-Nullable; wrapped = Distributed). +assert_pre_exchange_topology() { + local engine end_time + engine="$(traces_engine traces)" + end_time="$(traces_endtime_type traces)" + [[ -n "$engine" ]] || { echo "ERROR: no 'traces' table found in database '$DATABASE'." >&2; exit 1; } + + if [[ "$engine" == "Distributed" ]]; then + echo "ERROR: 'traces' is already a Distributed wrapper — the cutover and the wrap already ran. Nothing to EXCHANGE. To roll back, use rollback.sh --stage C." >&2 + exit 1 + fi + if [[ "$end_time" != Nullable* ]]; then + # traces already holds the successor schema, so the EXCHANGE has run. + if [[ -n "$(traces_engine traces_local_v2)" ]]; then + echo "ERROR: the EXCHANGE already ran (traces holds the successor schema) but 'traces_local_v2' still exists —" >&2 + echo " the post-swap RENAME did not complete. Finish it, then continue (e.g. --wrap-only or rollback.sh):" >&2 + echo " clickhouse-client --database $DATABASE --query \"RENAME TABLE $DATABASE.traces_local_v2 TO $DATABASE.traces_pre_cutover_backup ON CLUSTER '{cluster}'\"" >&2 + else + echo "ERROR: the EXCHANGE already ran (traces is the successor; old data parked as traces_pre_cutover_backup)." >&2 + echo " Do NOT re-run it — a second EXCHANGE would swap the tables back. Apply the deferred wrap with --wrap-only, or roll back with rollback.sh --stage B." >&2 + fi + exit 1 + fi + [[ -n "$(traces_engine traces_local_v2)" ]] || { echo "ERROR: successor 'traces_local_v2' not found; run the backfill + delta first." >&2; exit 1; } +} + +# --wrap-only precondition: traces must be the post-EXCHANGE successor MergeTree (not the original, not already wrapped). +assert_pre_wrap_topology() { + local engine end_time + engine="$(traces_engine traces)" + end_time="$(traces_endtime_type traces)" + [[ -n "$engine" ]] || { echo "ERROR: no 'traces' table found in database '$DATABASE'." >&2; exit 1; } + if [[ "$engine" == "Distributed" ]]; then + echo "ERROR: --wrap-only: 'traces' is already a Distributed wrapper (the wrap already ran). Nothing to do." >&2 + exit 1 + fi + [[ "$end_time" != Nullable* ]] || { + echo "ERROR: --wrap-only expects the post-EXCHANGE state (traces = successor schema), but traces has Nullable end_time (the EXCHANGE has not run). Run without --wrap-only first." >&2 + exit 1 + } +} + +# Pre-EXCHANGE gate: the swap is metadata-only and near-instant, but each replica reads its own local parts afterwards. +# If a replica is still fetching backfilled parts (replication_queue) or has not finished the deletion-replay mutation +# (system.mutations), swapping now would make that replica serve an incomplete table. Both are checked across ALL +# replicas via clusterAllReplicas, so a single connection sees the whole cluster's backlog. Aborts unless --force. +assert_replication_settled() { + local cluster queue mutations + cluster="$(ch "SELECT getMacro('cluster')")" + [[ -n "$cluster" ]] || { echo "ERROR: could not resolve the '{cluster}' macro (getMacro('cluster') was empty). Pass --force only if you have confirmed replication settlement out of band." >&2; exit 1; } + + queue="$(ch "SELECT count() + FROM clusterAllReplicas('$cluster', system.replication_queue) + WHERE database = '$DATABASE' + AND table IN ('traces', 'traces_local_v2')")" + mutations="$(ch "SELECT count() + FROM clusterAllReplicas('$cluster', system.mutations) + WHERE database = '$DATABASE' + AND table = 'traces_local_v2' + AND is_done = 0")" + + if [[ "$queue" != "0" || "$mutations" != "0" ]]; then + echo "ERROR: replication not settled across cluster '$cluster' — replication_queue=$queue, unfinished mutations=$mutations." >&2 + echo " Wait for both to reach 0 (parts fetched, deletion replay applied everywhere) before the EXCHANGE, or pass --force to override." >&2 + exit 1 + fi + echo "Replication settled across cluster '$cluster' (replication_queue=0, mutations done)." +} + +# Topology precondition first (independent of --force, which only bypasses the replication-settle gate). +if [[ "$WRAP_ONLY" == "1" ]]; then + assert_pre_wrap_topology +else + assert_pre_exchange_topology +fi + +if [[ "$FORCE" == "1" ]]; then + echo "WARNING: --force set; skipping the replication-settle gate." +else + assert_replication_settled +fi + +# Extract one `-- >>> BEGIN ` .. `-- >>> END ` block from the reference SQL (exact-line markers). +extract() { + awk -v begin="-- >>> BEGIN $1" -v end="-- >>> END $1" '$0 == begin {f = 1; next} $0 == end {f = 0} f' "$SQL_FILE" +} + +run_block() { + local sql + sql="$(extract "$1")" + sql="${sql//'${ANALYTICS_DB_DATABASE_NAME}'/$DATABASE}" + clickhouse-client --database "$DATABASE" --multiquery --query "$sql" +} + +if [[ "$WRAP_ONLY" == "1" ]]; then + # Deferred second half: the EXCHANGE already happened in a prior --skip-wrap run, so `traces` is the live + # partitioned data. Do not re-EXCHANGE (that would swap the parked original back in) and do not capture a new + # cutover_start (the data cutover is already done). Just apply the Distributed wrap. + run_block wrap + echo "Distributed wrap done: 'traces' fronts 'traces_local' via sipHash64(project_id). (EXCHANGE was a prior step.)" + exit 0 +fi + +CUTOVER_START="$(clickhouse-client --database "$DATABASE" --query "SELECT toString(now64(6))")" +echo "RECORD cutover_start=$CUTOVER_START (pass to rollback.sh --cutover-start if you roll back after this point)" + +run_block exchange +echo "EXCHANGE done: 'traces' is now the partitioned data; the old data is parked as 'traces_pre_cutover_backup'." + +if [[ "$SKIP_WRAP" == "1" ]]; then + echo "Skipping the Distributed wrap (--skip-wrap). The data cutover is complete." +else + run_block wrap + echo "Distributed wrap done: 'traces' fronts 'traces_local' via sipHash64(project_id)." +fi + +echo "Restore databaseAnalytics.asyncInsertBusyTimeoutMaxMs to default, verify, and keep traces_pre_cutover_backup for the soak." diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/finalize.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/finalize.sh new file mode 100755 index 00000000000..b261dbe6a9f --- /dev/null +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/finalize.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# +# Drops the parked backup once the cutover (or a rollback) has soaked and the live `traces` is confirmed healthy +# (runbook: ../README.md). This is the ONLY script that drops a data-bearing table, so it is guarded and defaults to a +# dry run. +# +# The parked backup's NAME depends on how the estate got here, and the two never co-exist: +# * after a successful cutover -> the old original is parked as `traces_pre_cutover_backup` (the live successor is +# `traces`, or `traces_local` behind the Distributed wrapper). Dropping it commits to +# the new layout. +# * after a rollback -> the abandoned successor is parked as `traces_local_v2` (the original is live as +# `traces`). Dropping it abandons the migration. +# This detects whichever parked table is present and drops it — it never targets the live `traces` or the live +# `traces_local` shard. It refuses if the live `traces` is empty while the backup is not (the live table may be +# unhealthy and the "backup" the only copy), and if BOTH parked names exist (an ambiguous, unexpected state that a human +# must resolve). +# +# Connection: clickhouse-client env vars (CLICKHOUSE_HOST, CLICKHOUSE_PORT, CLICKHOUSE_USER, CLICKHOUSE_PASSWORD). +# +# Options: +# --database NAME analytics database (e.g. opik). Required. +# --confirm actually drop; without it, prints what would be dropped and exits (dry run). + +set -euo pipefail + +DATABASE="" +CONFIRM=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --database) DATABASE="$2"; shift 2 ;; + --confirm) CONFIRM=1; shift ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done + +[[ -n "$DATABASE" ]] || { echo "ERROR: --database is required" >&2; exit 2; } + +ch() { + clickhouse-client --database "$DATABASE" --query "$1" +} + +exists() { + ch "SELECT count() FROM system.tables WHERE database = '$DATABASE' AND name = '$1'" +} + +[[ "$(exists traces)" != "0" ]] || { echo "ERROR: live 'traces' table not found in '$DATABASE'." >&2; exit 1; } + +# Detect the parked backup by name: traces_pre_cutover_backup (post-successful-cutover) or traces_local_v2 +# (post-rollback). They never co-exist in a clean flow; if both are present the estate is ambiguous — refuse. +HAS_PRECUTOVER="$([[ "$(exists traces_pre_cutover_backup)" != "0" ]] && echo 1 || echo 0)" +HAS_V2="$([[ "$(exists traces_local_v2)" != "0" ]] && echo 1 || echo 0)" + +if [[ "$HAS_PRECUTOVER" == "1" && "$HAS_V2" == "1" ]]; then + echo "ERROR: both 'traces_pre_cutover_backup' and 'traces_local_v2' exist — ambiguous state." >&2 + echo " Expected exactly one parked backup. Investigate and drop the correct one by hand." >&2 + exit 1 +elif [[ "$HAS_PRECUTOVER" == "1" ]]; then + BACKUP="traces_pre_cutover_backup" +elif [[ "$HAS_V2" == "1" ]]; then + BACKUP="traces_local_v2" +else + echo "Nothing to finalize: no parked backup ('traces_pre_cutover_backup' or 'traces_local_v2') exists." + exit 0 +fi + +LIVE_ROWS="$(ch "SELECT count() FROM traces")" +BACKUP_ROWS="$(ch "SELECT count() FROM $BACKUP")" + +# Refuse the dangerous case: a live table that looks empty while the backup holds data. +if [[ "$LIVE_ROWS" == "0" && "$BACKUP_ROWS" != "0" ]]; then + echo "ERROR: live 'traces' is empty but '$BACKUP' has $BACKUP_ROWS rows. Refusing to drop the backup —" >&2 + echo " verify the live table is the healthy one before finalizing." >&2 + exit 1 +fi + +echo "Live 'traces': $LIVE_ROWS rows. Parked '$BACKUP': $BACKUP_ROWS rows." +if [[ "$CONFIRM" != "1" ]]; then + echo "DRY RUN: would DROP TABLE $DATABASE.$BACKUP. Re-run with --confirm to drop it." + exit 0 +fi + +ch "DROP TABLE IF EXISTS $BACKUP ON CLUSTER '{cluster}' SYNC" +echo "Dropped $DATABASE.$BACKUP. The cutover is finalized." diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh new file mode 100755 index 00000000000..2daf5d5f47d --- /dev/null +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# +# Driver for rolling the buffered traces cutover back (runbook: ../README.md). +# +# Runs the db-app-analytics/000004_rollback_* file(s) that match how far the cutover got. Pick the stage by the last +# step that completed: +# --stage A backfill/delta ran but the EXCHANGE did not — discard the shadow (live `traces` is untouched). +# --stage B the EXCHANGE ran but not the wrap — swap the tables back, then reverse-replay. +# --stage C the wrap ran — drop the wrapper, promote the parked original, then reverse-replay. +# Stages B and C need --cutover-start (printed by exchange_and_wrap.sh) to bound the reverse-replay. Keep the deletion +# bridge enabled through the rollback so no delete is lost. +# +# SAFETY: the stages are mutually exclusive and each lives in its OWN file, so no single file mixes a TRUNCATE with an +# EXCHANGE/DROP — running any file does exactly one stage. Before running, this asserts the live `traces` topology matches +# the requested stage and aborts otherwise, so a wrong-stage run cannot destroy data. No data-bearing table is dropped; +# every stage ends in the canonical state (traces = original data live, traces_local_v2 = successor data parked). The +# parked backup is dropped only later by finalize.sh, after the soak. +# +# Connection: clickhouse-client env vars (CLICKHOUSE_HOST, CLICKHOUSE_PORT, CLICKHOUSE_USER, CLICKHOUSE_PASSWORD). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SQL_DIR="$SCRIPT_DIR/db-app-analytics" + +DATABASE="" +STAGE="" +CUTOVER_START="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --database) DATABASE="$2"; shift 2 ;; + --stage) STAGE="$2"; shift 2 ;; + --cutover-start) CUTOVER_START="$2"; shift 2 ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done + +[[ -n "$DATABASE" ]] || { echo "ERROR: --database is required" >&2; exit 2; } +case "$STAGE" in + A|B|C) ;; + *) echo "ERROR: --stage must be A, B or C" >&2; exit 2 ;; +esac + +ch() { + clickhouse-client --database "$DATABASE" --query "$1" +} + +# Single scalar (or empty string if the object does not exist). Used by the topology guards below. +traces_engine() { + ch "SELECT engine FROM system.tables WHERE database = '$DATABASE' AND name = '$1'" +} +traces_endtime_type() { + ch "SELECT type FROM system.columns WHERE database = '$DATABASE' AND table = '$1' AND name = 'end_time'" +} + +# The migration walks traces through three shapes; a stage is only valid in one of them: +# pre-EXCHANGE -> traces is a *MergeTree with Nullable end_time (the original schema) -> stage A +# post-EXCHANGE -> traces is a *MergeTree with non-Nullable end_time (the successor schema) -> stage B +# post-wrap -> traces is a Distributed table -> stage C +# Asserting the shape makes a wrong-stage run (which is where a TRUNCATE/DROP would be catastrophic) abort with no change. +assert_topology() { + local engine end_time + engine="$(traces_engine traces)" + end_time="$(traces_endtime_type traces)" + [[ -n "$engine" ]] || { echo "ERROR: no 'traces' table found in database '$DATABASE'." >&2; exit 1; } + + case "$STAGE" in + A) + [[ "$engine" != "Distributed" && "$end_time" == Nullable* ]] || { + echo "ERROR: stage A expects the pre-EXCHANGE state (traces = original schema), but traces is engine='$engine' end_time='$end_time'." >&2 + echo " The EXCHANGE has already run — truncating the shadow now would destroy the parked original. Use stage B or C." >&2 + exit 1 + } + [[ -n "$(traces_engine traces_local_v2)" ]] || { echo "ERROR: shadow table 'traces_local_v2' not found; nothing to discard." >&2; exit 1; } + ;; + B) + [[ "$engine" != "Distributed" ]] || { + echo "ERROR: stage B expects the post-EXCHANGE, pre-wrap state, but traces is Distributed (the wrap ran). Use stage C." >&2 + exit 1 + } + [[ "$end_time" != Nullable* ]] || { + echo "ERROR: stage B expects traces to hold the successor schema, but end_time is Nullable (the EXCHANGE has not run). Nothing to roll back; use stage A to discard the shadow." >&2 + exit 1 + } + [[ -n "$(traces_engine traces_pre_cutover_backup)" ]] || { echo "ERROR: 'traces_pre_cutover_backup' (parked original) not found; cannot swap back." >&2; exit 1; } + ;; + C) + [[ "$engine" == "Distributed" ]] || { + echo "ERROR: stage C expects the post-wrap state (traces = Distributed), but traces is engine='$engine'. The wrap was not applied — use stage B." >&2 + exit 1 + } + [[ -n "$(traces_engine traces_local)" ]] || { echo "ERROR: 'traces_local' (successor data) not found; topology is not a clean post-wrap state." >&2; exit 1; } + [[ -n "$(traces_engine traces_pre_cutover_backup)" ]] || { echo "ERROR: 'traces_pre_cutover_backup' (parked original) not found; topology is not a clean post-wrap state." >&2; exit 1; } + ;; + esac +} + +# Run one rollback .sql file wholesale, substituting the placeholders. Each file is exactly one stage's statements. +run_file() { + local file="$SQL_DIR/$1" sql + [[ -f "$file" ]] || { echo "ERROR: cannot find $file" >&2; exit 2; } + sql="$(cat "$file")" + sql="${sql//'${ANALYTICS_DB_DATABASE_NAME}'/$DATABASE}" + sql="${sql//'${CUTOVER_START}'/$CUTOVER_START}" + clickhouse-client --database "$DATABASE" --multiquery --query "$sql" +} + +assert_topology + +case "$STAGE" in + A) + run_file 000004_rollback_stage_a_discard_shadow.sql + echo "Stage A done: shadow discarded. Live 'traces' was untouched." + ;; + B) + [[ -n "$CUTOVER_START" ]] || { echo "ERROR: --cutover-start is required for stage B" >&2; exit 2; } + run_file 000004_rollback_stage_b_exchange_back.sql + run_file 000004_rollback_reverse_replay.sql + echo "Stage B done: tables swapped back and deletes since cutover_start re-applied." + ;; + C) + [[ -n "$CUTOVER_START" ]] || { echo "ERROR: --cutover-start is required for stage C" >&2; exit 2; } + run_file 000004_rollback_stage_c_promote_original.sql + run_file 000004_rollback_reverse_replay.sql + echo "Stage C done: wrapper dropped, original promoted, deletes since cutover_start re-applied." + ;; +esac + +echo "Now in the canonical state: traces = original data (live), traces_local_v2 = successor data (parked)." +echo "Verify (README 'Verifying the migration'), then drop the parked backup with finalize.sh once healthy." diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh new file mode 100755 index 00000000000..90681d5af3b --- /dev/null +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# +# Fidelity QA driver for the buffered traces cutover (runbook: ../README.md, "Verifying the migration"). +# +# Compares the migrated data on the old-schema and new-schema tables, week by week (created_at), using a NORMALIZED +# fingerprint so sentinel/precision differences (end_time NULL<->epoch, ttft NULL<->NaN, ns<->us) do not count as +# changes. For each week it reads one (row count, checksum) verdict per side; a mismatch means that week's live, deduped +# content differs. With --drill-down, a mismatched week is followed by a per-key listing of the rows that differ. Exits +# non-zero if any window mismatched. +# +# The compare and drill-down SQL are NOT duplicated here: both are read from db-app-analytics/000005_verify_migration.sql +# (the single source, and the exact normalization the gate test asserts). See README "Verifying the migration". +# +# Feasibility on a large table: full mode reads every partition (heavy but bounded per week; run off-peak). --sample-mod +# compares a deterministic id sample (same rows on both sides); --weeks-stride compares every Nth week; --from/--to-week +# limit the range. Comparing a representative subset gives high confidence when a full pass is infeasible. +# +# Usage: +# CLICKHOUSE_HOST=... CLICKHOUSE_PASSWORD=... ./verify.sh --database opik [options] +# +# Options: +# --database NAME analytics database (e.g. opik). Required. +# --old-table NAME old-schema table (Nullable, nanosecond). Default traces. After the EXCHANGE: traces_pre_cutover_backup. +# --new-table NAME new-schema table (sentinels, microsecond). Default traces_local_v2. After the EXCHANGE: traces. +# --sample-mod N compare a deterministic 1/N id sample (same ids on both sides). Default 1 (every row). +# --from-week N start at week offset N (0-based from the anchor Monday). Default 0. +# --to-week M stop after week offset M (inclusive). Default: last week with data. +# --weeks-stride S compare every S-th week (S>1 samples partitions for a quick pass). Default 1. +# --drill-down on a mismatched week, also print up to 100 keys that differ or exist on one side only. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VERIFY_SQL="$SCRIPT_DIR/db-app-analytics/000005_verify_migration.sql" + +DATABASE="" +OLD_TABLE="traces" # old-schema side; becomes traces_pre_cutover_backup after the EXCHANGE (see --old-table) +NEW_TABLE="traces_local_v2" # new-schema side (the successor being built); becomes traces after the EXCHANGE +SAMPLE_MOD=1 # 1 = every row; N compares a deterministic 1/N id sample, identical on both sides +FROM_WEEK=0 +TO_WEEK="" +WEEKS_STRIDE=1 # 1 = every week; S skips to every S-th weekly partition for a quick, pruned pass +DRILL_DOWN=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --database) DATABASE="$2"; shift 2 ;; + --old-table) OLD_TABLE="$2"; shift 2 ;; + --new-table) NEW_TABLE="$2"; shift 2 ;; + --sample-mod) SAMPLE_MOD="$2"; shift 2 ;; + --from-week) FROM_WEEK="$2"; shift 2 ;; + --to-week) TO_WEEK="$2"; shift 2 ;; + --weeks-stride) WEEKS_STRIDE="$2"; shift 2 ;; + --drill-down) DRILL_DOWN=1; shift ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done + +[[ -n "$DATABASE" ]] || { echo "ERROR: --database is required" >&2; exit 2; } +[[ -f "$VERIFY_SQL" ]] || { echo "ERROR: cannot find verify SQL at $VERIFY_SQL" >&2; exit 2; } + +ch() { + clickhouse-client --database "$DATABASE" --query "$1" +} + +log() { + echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" +} + +# Extract one `-- >>> BEGIN ` .. `-- >>> END ` block from the reference SQL (exact-line markers), and +# substitute this window's placeholders. +render_block() { + local block="$1" lo="$2" hi="$3" sql + sql="$(awk -v begin="-- >>> BEGIN $block" -v end="-- >>> END $block" \ + '$0 == begin {f = 1; next} $0 == end {f = 0} f' "$VERIFY_SQL")" + sql="${sql//'${ANALYTICS_DB_DATABASE_NAME}'/$DATABASE}" + sql="${sql//'${OLD_TABLE}'/$OLD_TABLE}" + sql="${sql//'${NEW_TABLE}'/$NEW_TABLE}" + sql="${sql//'${WINDOW_LO}'/$lo}" + sql="${sql//'${WINDOW_HI}'/$hi}" + sql="${sql//'${SAMPLE_MOD}'/$SAMPLE_MOD}" + printf '%s' "$sql" +} + +# Verdict TSV row for one window: src_rows dst_rows src_checksum dst_checksum ok +compare_window() { + clickhouse-client --database "$DATABASE" --multiquery --query "$(render_block compare "$1" "$2")" +} + +# Per-key differences for one window (only run on a mismatch, under --drill-down). +drill_down_window() { + clickhouse-client --database "$DATABASE" --multiquery --query "$(render_block drill-down "$1" "$2")" +} + +ROWS="$(ch "SELECT count() FROM $OLD_TABLE")" +if [[ "$ROWS" == "0" ]]; then + log "Old-schema table '$OLD_TABLE' is empty — nothing to verify." + exit 0 +fi + +# Week range from the old table's created_at (bounded and real; covers rows whose id_at is far-future from the bad-id bug +# but whose created_at is real). Same anchor math as backfill.sh. +ANCHOR="$(ch "SELECT toString(toMonday(min(created_at))) FROM $OLD_TABLE")" +HORIZON="$(ch "SELECT toString(addWeeks(toMonday(max(created_at)), 1)) FROM $OLD_TABLE")" +LAST_WEEK="$(ch "SELECT dateDiff('week', toDate('$ANCHOR'), toDate('$HORIZON')) - 1")" +[[ -n "$TO_WEEK" ]] || TO_WEEK="$LAST_WEEK" + +log "Verify: $OLD_TABLE vs $NEW_TABLE | weeks [$FROM_WEEK..$TO_WEEK] stride $WEEKS_STRIDE | sample 1/$SAMPLE_MOD" + +mismatches=0 +checked=0 +for (( week=FROM_WEEK; week<=TO_WEEK; week+=WEEKS_STRIDE )); do + LO="$(ch "SELECT toString(addWeeks(toDate('$ANCHOR'), $week))") 00:00:00" + HI="$(ch "SELECT toString(addWeeks(toDate('$ANCHOR'), $((week + 1))))") 00:00:00" + + read -r src_rows dst_rows src_checksum dst_checksum ok <<< "$(compare_window "$LO" "$HI")" + checked=$((checked + 1)) + if [[ "$ok" == "1" ]]; then + log "week $week ($LO .. $HI): OK (rows=$src_rows)" + else + mismatches=$((mismatches + 1)) + log "MISMATCH week $week ($LO .. $HI): src_rows=$src_rows dst_rows=$dst_rows src_checksum=$src_checksum dst_checksum=$dst_checksum" >&2 + if [[ "$DRILL_DOWN" == "1" ]]; then + log " differing keys (key, src_hash, dst_hash; NULL = missing on that side):" >&2 + drill_down_window "$LO" "$HI" >&2 + else + log " re-run with --drill-down to list the differing keys for this window" >&2 + fi + fi +done + +if [[ "$mismatches" != "0" ]]; then + log "FAILED: $mismatches of $checked windows mismatched." >&2 + exit 1 +fi +log "PASSED: all $checked windows match (sample 1/$SAMPLE_MOD)." diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java new file mode 100644 index 00000000000..0eb7f3fbdc7 --- /dev/null +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java @@ -0,0 +1,1370 @@ +package com.comet.opik.infrastructure; + +import com.comet.opik.api.resources.utils.ClickHouseContainerUtils; +import com.comet.opik.api.resources.utils.MigrationUtils; +import com.comet.opik.domain.IdGenerator; +import com.comet.opik.domain.TestIdGeneratorFactory; +import com.comet.opik.infrastructure.db.TransactionTemplateAsync; +import com.comet.opik.utils.ClickHouseDateTimeFormat; +import com.comet.opik.utils.template.TemplateUtils; +import io.r2dbc.spi.Statement; +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.testcontainers.clickhouse.ClickHouseContainer; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.lifecycle.Startables; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static com.comet.opik.api.resources.utils.ClickHouseContainerUtils.DATABASE_NAME; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * End-to-end validation of the buffered cutover that migrates {@code traces} to its partitioned, sharding-ready + * successor {@code traces_local_v2}. It rehearses the full sequence against a fresh ClickHouse in raw SQL — the same + * steps an operator runs from the {@code data-migrations/traces-local-v2-cutover} runbook — and pins the properties + * the cutover's correctness depends on. + * + *

Deletions must survive the swap (the core property). A lightweight DELETE flips a hidden row mask; it does + * not bump {@code last_updated_at} (the {@code ReplacingMergeTree} version column), so the version-based delta-insert is + * blind to deletes that land while the table is being copied — the already-copied row stays alive on the destination + * and the deletion would leak across the swap. The deletion-events bridge closes this: every delete is recorded in + * {@code deletion_events_local} and replayed against the destination before the swap. The test exercises: + *

    + *
  • rows deleted before the backfill — excluded from {@code INSERT SELECT} by the + * {@code apply_deleted_mask = 1} default, so they never reach the destination;
    • + *
  • rows deleted during the backfill (a large retention-shape batch and single user-shape ids) — the test + * asserts the leak is real (still alive on the destination after the delta-insert: the negative control that proves + * the bridge is load-bearing), that the replay masks them, and that there are zero leaks after the swap.
    • + *
+ * + *

Replay matches the full key {@code (workspace_id, project_id, id)}. Trace ids are not globally unique — + * imported or crafted rows can reuse an id across projects — so replaying by {@code id} alone would over-delete a live + * row that merely shares the id in another project. The bridge captures the resolved {@code (workspace_id, project_id)}, + * so the replay deletes by the full key, which is also the destination primary key (so the mutation prunes on it). A + * reused id deleted in one project and surviving in another exercises this. + * + *

The delta is anchored on {@code created_at OR last_updated_at >= backfill_start}. {@code last_updated_at} is + * client-supplied on the batch-ingest path, so it is not a reliable "changed since" signal on its own; and a cutoff + * taken at backfill end would miss writes that landed during the (long) backfill. But every trace write sets either a + * fresh server {@code created_at} (batch-ingest path) or a fresh server {@code last_updated_at} (create/update merge + * paths), so the union, anchored before the backfill, catches every row written during the window. Both arms are + * covered: a normal upsert (new {@code last_updated_at}) and a row created during the window with a client-backdated + * {@code last_updated_at} that only the {@code created_at} arm catches. + * + *

It also confirms {@code EXCHANGE TABLES ... ON CLUSTER} on the single-shard cluster, the sharding-ready + * {@code Distributed} wrapper reading transparently on one shard, newest-version-wins for concurrent upserts, and it + * measures the replay wall time so the runbook can size it against the ingestion buffer window. Finally it proves the + * cutover is reversible: the post-wrap rollback drops the wrapper, promotes the parked old data back to {@code traces}, + * and reverse-replays so a post-cutover delete does not resurrect. + * + *

Dedicated, non-reused containers are required because the cutover ends in a destructive {@code EXCHANGE} + + * {@code RENAME} of the live {@code traces} table, which must never touch a container shared with other suites. Runs + * raw SQL over {@link TransactionTemplateAsync} with no Dropwizard app, mirroring {@link TracesLocalV2PartitioningTest}. + * + *

Why raw SQL and not the production DAOs. The cutover orchestration this validates (backfill + * {@code INSERT SELECT}, delta, replay, {@code EXCHANGE}, wrap) is operator SQL that no DAO owns, and it needs the + * destructive-safe containers above, which the shared app-harness cannot provide. The seeding, delete and + * bridge-capture helpers mirror the production write shapes ({@code TraceDAO}, {@code TraceService} delete, + * {@code DeletionEventDAO}) and reproduce the two version-stamp regimes the delta relies on (fresh server + * {@code created_at} vs client {@code last_updated_at}); the DAOs' own semantics are covered by their dedicated suites + * (e.g. {@code TraceDeletionEventTest}). + * + *

Scope: this gate validates the cutover SQL logic, not the driver scripts. The safety guards in the runbook's + * bash drivers — {@code backfill.sh}'s reconciliation abort, {@code rollback.sh}'s wrong-stage topology assertions, + * {@code exchange_and_wrap.sh}'s replication-settle gate, {@code finalize.sh}'s empty-live refusal — are exercised by the + * OPIK-6901 staging dry-run, not by this test (which runs the SQL those scripts wrap, directly). This test asserts the + * logic is correct when invoked; the staging rehearsal asserts the scripts invoke it safely. + */ +@Slf4j +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TracesLocalV2CutoverTest { + + /** + * A fixed historical Monday the seeded rows are minted at week offsets from, so the backfill can slice the source + * by whole {@code created_at} weeks deterministically. Far in the past and never {@code now}-derived, so nothing + * drifts across a week boundary mid-run. It intentionally overlaps the anchor another suite + * ({@code TracesLocalV2PartitioningTest}) uses, which is safe: this suite runs on its own dedicated, non-reused + * containers (see the container fields below), so its data never shares a ClickHouse instance with any other suite. + */ + private static final LocalDate ANCHOR_MONDAY = LocalDate.of(2025, 3, 3); + + /** + * A client-backdated version stamp, well before {@code backfill_start}. A row written during the window carrying + * this as its {@code last_updated_at} can only be caught by the delta's {@code created_at} arm. + */ + private static final Instant BACKDATED = LocalDate.of(2020, 1, 1).atStartOfDay().toInstant(ZoneOffset.UTC); + + /** Rows spread across three consecutive weekly partitions, so the backfill runs as three weekly batches. */ + private static final int SEED_WEEKS = 3; + private static final int SURVIVORS_PER_WEEK = 40; + private static final int PRE_EXISTING_DELETED_PER_WEEK = 15; + private static final int RETENTION_DELETED_PER_WEEK = 80; + private static final int USER_DELETED_PER_WEEK = 5; + private static final int DELTA_UPSERTS = 20; + private static final int DELTA_LATE_CREATED = 10; + + private static final int REPLAY_BUDGET_SECONDS = 60; + + private static final String[] FIDELITY_SOURCES = {"sdk", "experiment", "playground", "optimization", "evaluator"}; + private static final String[] FIDELITY_ENVIRONMENTS = {"production", "staging", "dev", ""}; + + /** + * The stored (non-materialized) columns the cutover copies, one per line. Both INSERT clauses are built from this + * list, and {@link #cutoverCopiesEveryBaseColumn()} asserts it equals the live base columns of {@code traces} — so a + * base column added by a future migration cannot be silently left uncopied (the fidelity fingerprint, which lists a + * fixed set, would not catch that on its own). A new column here without a matching SELECT entry fails arity at run. + */ + private static final String COPIED_COLUMNS = """ + id, + workspace_id, + project_id, + name, + start_time, + end_time, + input, + output, + metadata, + tags, + created_at, + last_updated_at, + created_by, + last_updated_by, + error_info, + thread_id, + visibility_mode, + truncation_threshold, + input_slim, + output_slim, + ttft, + source, + environment"""; + + /** + * The SELECT projection the backfill and delta share: the {@link #COPIED_COLUMNS} columns, with the two denullified + * columns coalesced to their sentinels (end_time → epoch, ttft → NaN). The two INSERT-SELECTs differ only in their + * WHERE clause, so the projection is defined once here. A column added to {@link #COPIED_COLUMNS} but not here (or + * vice versa) fails arity at run. + */ + private static final String COPIED_SELECT = """ + id, + workspace_id, + project_id, + name, + start_time, + coalesce(end_time, toDateTime64('1970-01-01 00:00:00', 6)) AS end_time, + input, + output, + metadata, + tags, + created_at, + last_updated_at, + created_by, + last_updated_by, + error_info, + thread_id, + visibility_mode, + truncation_threshold, + input_slim, + output_slim, + coalesce(ttft, toFloat64('nan')) AS ttft, + source, + environment"""; + + private static final IdGenerator ID_GENERATOR = TestIdGeneratorFactory.create(); + + private final Network network = Network.newNetwork(); + private final GenericContainer zookeeperContainer = ClickHouseContainerUtils.newZookeeperContainer(false, + network); + private final ClickHouseContainer clickHouseContainer = ClickHouseContainerUtils + .newClickHouseContainer(false, network, zookeeperContainer); + + private final TransactionTemplateAsync template; + + { + Startables.deepStart(zookeeperContainer, clickHouseContainer).join(); + MigrationUtils.runClickhouseDbMigration(clickHouseContainer); + template = TransactionTemplateAsync.create( + ClickHouseContainerUtils.newDatabaseAnalyticsFactory(clickHouseContainer, DATABASE_NAME).build()); + } + + /** + * Each rollback stage ends in the canonical state (traces = original schema, traces_local_v2 = successor), which is + * also the pre-EXCHANGE baseline — so truncating the three tables (and dropping any stray wrap/rename artifacts) is a + * clean reset that keeps the tests independent of each other and of run order. The stray artifacts are the + * post-wrap {@code traces_local} shard and the post-EXCHANGE {@code traces_pre_cutover_backup} — a test that fails + * mid-cutover (after the swap/wrap, before its rollback) would otherwise leak one of them into the next test. + */ + @BeforeEach + void resetTables() { + execute("DROP TABLE IF EXISTS traces_local ON CLUSTER '{cluster}' SYNC", _ -> { + }); + execute("DROP TABLE IF EXISTS traces_pre_cutover_backup ON CLUSTER '{cluster}' SYNC", _ -> { + }); + execute("TRUNCATE TABLE IF EXISTS traces", _ -> { + }); + execute("TRUNCATE TABLE IF EXISTS traces_local_v2", _ -> { + }); + execute("TRUNCATE TABLE IF EXISTS deletion_events_local", _ -> { + }); + } + + @Test + void bufferedCutoverPreservesEveryDeletionAcrossExchange() { + var workspaceId = UUID.randomUUID().toString(); + var projectId = ID_GENERATOR.generateId(); + var otherProjectId = ID_GENERATOR.generateId(); + + var survivors = mintIds(SURVIVORS_PER_WEEK); + var preExistingDeleted = mintIds(PRE_EXISTING_DELETED_PER_WEEK); + var retentionDeleted = mintIds(RETENTION_DELETED_PER_WEEK); + var userDeleted = mintIds(USER_DELETED_PER_WEEK); + // Deletes the delete-by-ids path could not resolve to a project — captured in the bridge with an empty + // project_id. The replay must still remove them (matched by (workspace_id, id)) or they leak across the swap. + var unresolvedDeleted = mintIds(USER_DELETED_PER_WEEK); + // One id reused across two projects: deleted in projectId, must survive in otherProjectId (full-key replay). + var reusedInstant = weekInstant(0, 1); + var reusedId = ID_GENERATOR.generateId(reusedInstant); + var reused = List.of(CategorizedId.builder().id(reusedId).createdAt(reusedInstant).build()); + + // Seed the live table across the weekly partitions. created_at drives the backfill slice; id (a UUIDv7 minted + // at the same week) drives the destination id_at partition, independently of the slice. + var allSeeded = new ArrayList(); + allSeeded.addAll(survivors); + allSeeded.addAll(preExistingDeleted); + allSeeded.addAll(retentionDeleted); + allSeeded.addAll(userDeleted); + allSeeded.addAll(unresolvedDeleted); + seedTraces(allSeeded, workspaceId, projectId); + seedTraces(reused, workspaceId, projectId); + seedTraces(reused, workspaceId, otherProjectId); + // Every migrated column populated with distinct values at ns precision (+ some NULL end_time/ttft), so the + // fidelity fingerprint below actually exercises every column and the ns->us truncation. + var fidelityIds = seedFidelityCohort(workspaceId, projectId); + + // Pre-existing deletes: removed before the backfill starts, and NOT recorded in the bridge — INSERT SELECT + // honors the mask and never copies them, so no replay is involved. + lightweightDelete(idStrings(preExistingDeleted), workspaceId); + + // Anchor for BOTH the delta and the replay window, captured BEFORE the backfill so it covers the whole run. + var backfillStart = nowMicros(); + + // Weekly-batched backfill, the same INSERT SELECT the runbook runs (sentinel coalescing for the denullified + // columns; is_deleted omitted so it defaults to 0). + for (int week = 0; week < SEED_WEEKS; week++) { + backfillWeek(week); + } + + // Guard: masked rows did not ride across the copy. + assertThat(liveCount("traces_local_v2", idStrings(preExistingDeleted), workspaceId)) + .as("pre-existing masked rows must not be copied by the backfill") + .isZero(); + assertThat(liveCount("traces_local_v2", idStrings(survivors), workspaceId)) + .as("all survivors backfilled") + .isEqualTo(survivors.size()); + + // Deletes during the backfill/delta window. + // Retention-shape: bridge INSERT first (before the LWD), then one large lightweight DELETE. + recordDeletionEvents(idStrings(retentionDeleted), workspaceId, projectId.toString(), "retention"); + lightweightDelete(idStrings(retentionDeleted), workspaceId); + // User-shape: single-id deletes. + recordDeletionEvents(idStrings(userDeleted), workspaceId, projectId.toString(), "user_request"); + lightweightDelete(idStrings(userDeleted), workspaceId); + // Unresolved-project deletes: captured with an empty project_id (delete-by-ids couldn't resolve the project). + recordDeletionEvents(idStrings(unresolvedDeleted), workspaceId, "", "user_request"); + lightweightDelete(idStrings(unresolvedDeleted), workspaceId); + // Reused-id delete scoped to projectId only — the copy under otherProjectId must survive. + recordDeletionEvents(Set.of(reusedId.toString()), workspaceId, projectId.toString(), "user_request"); + lightweightDeleteScoped(Set.of(reusedId.toString()), workspaceId, projectId); + // Concurrent upserts: a newer version of a subset of survivors — caught by the delta's last_updated_at arm. + var deltaUpserted = survivors.subList(0, DELTA_UPSERTS); + insertRows(deltaUpserted, workspaceId, projectId, deltaName(), _ -> Instant.now()); + // Rows created during the window with a client-backdated last_updated_at — caught ONLY by the created_at arm. + var deltaLateCreated = mintNowIds(DELTA_LATE_CREATED); + insertRows(deltaLateCreated, workspaceId, projectId, "late", _ -> BACKDATED); + + // Delta-insert: created_at OR last_updated_at since backfill_start (see class Javadoc). + deltaInsert(backfillStart); + + // Negative control — before replay, the during-backfill deletes have leaked onto the destination: still fully + // alive there, because the delta-insert cannot see a lightweight delete. This is what the bridge exists to fix. + // Includes the unresolved (empty-project) deletes, which only the replay's (workspace_id, id) branch catches. + var leakedIds = union(union(idStrings(retentionDeleted), idStrings(userDeleted)), idStrings(unresolvedDeleted)); + assertThat(liveCount("traces_local_v2", leakedIds, workspaceId)) + .as("negative control: without replay, during-backfill deletes leak across the copy") + .isEqualTo(leakedIds.size()); + // Both delta arms worked: the backdated-last_updated_at rows were caught via created_at. + assertThat(liveCount("traces_local_v2", idStrings(deltaLateCreated), workspaceId)) + .as("delta created_at arm caught rows written during the window with a backdated last_updated_at") + .isEqualTo(deltaLateCreated.size()); + + // Deletion replay: read the bridge for the window and re-issue the deletes against the destination, matched on + // the full key so a reused id in another project is untouched. + var replayMillis = replayDeletions(backfillStart); + log.info("Deletion replay covered {} ids in {} ms", leakedIds.size() + 1, replayMillis); + assertThat(replayMillis) + .as("replay wall time smoke bound — the runbook compares this against the buffer window") + .isLessThan(REPLAY_BUDGET_SECONDS * 1_000L); + + // After replay, the leak is closed on the destination, before the swap. + assertThat(liveCount("traces_local_v2", leakedIds, workspaceId)) + .as("replay masks every bridged deletion on the destination") + .isZero(); + + // The all-column fidelity cohort was copied intact (its content is checked by the fingerprint below). + assertThat(liveCount("traces_local_v2", Set.copyOf(fidelityIds), workspaceId)) + .as("every fidelity-cohort row (all columns populated, ns created_at) is backfilled") + .isEqualTo(fidelityIds.size()); + + // Fidelity QA: before the swap, the deduped, mask-honored, NORMALIZED content of source and destination must be + // identical. This is the same normalized fingerprint verify.sh computes per week for production QA; asserting it + // here also proves the normalization (NULL/epoch and NULL/NaN sentinels, ns->us precision) is correct — a wrong + // normalization would fail even on this faithfully-migrated data. + assertThat(fingerprint("traces_local_v2", Shape.NEW, workspaceId)) + .as("normalized (count, checksum) fingerprint matches between source and destination") + .isEqualTo(fingerprint("traces", Shape.OLD, workspaceId)); + + // Derived/materialized columns are recomputed by each table's own DDL, so the base-column fingerprint above does + // not cover them. Assert the successor's expressions yield the SAME values as the source's on the fidelity + // cohort: the deterministic ones (lengths, truncated_*, output_keys) exactly, and duration within the intended + // ns->us precision (source computes from nanosecond timestamps and is NULL when unset; the successor computes + // from the microsecond copy and is NaN when unset). + assertThat(derivedFingerprint("traces_local_v2", workspaceId)) + .as("deterministic derived columns match after the copy (no MATERIALIZED-expression drift)") + .isEqualTo(derivedFingerprint("traces", workspaceId)); + assertThat(durationMismatches(workspaceId)) + .as("duration matches within the ns->us truncation, NULL<->NaN normalized") + .isZero(); + + // The atomic swap: EXCHANGE TABLES ... ON CLUSTER on the single-shard cluster. Record the instant just before it + // as the rollback's reverse-replay window start (a post-cutover delete after this must not resurrect on rollback). + var cutoverStart = nowMicros(); + exchangeTables(); + + // Post-EXCHANGE, `traces` is the partitioned successor. Assert zero deletion leaks. + assertThat(liveCount("traces", idStrings(survivors), workspaceId)) + .as("every survivor is present after the cutover") + .isEqualTo(survivors.size()); + assertThat(liveCount("traces", idStrings(deltaLateCreated), workspaceId)) + .as("rows created during the window (backdated last_updated_at) survive the cutover") + .isEqualTo(deltaLateCreated.size()); + assertThat(liveCount("traces", idStrings(preExistingDeleted), workspaceId)) + .as("pre-existing deletions stay deleted after the cutover") + .isZero(); + assertThat(liveCount("traces", idStrings(retentionDeleted), workspaceId)) + .as("retention-shape deletions do not leak across the EXCHANGE") + .isZero(); + assertThat(liveCount("traces", idStrings(userDeleted), workspaceId)) + .as("user-shape deletions do not leak across the EXCHANGE") + .isZero(); + assertThat(liveCount("traces", idStrings(unresolvedDeleted), workspaceId)) + .as("unresolved (empty-project) deletions do not leak across the EXCHANGE") + .isZero(); + + // Full-key replay: the reused id is gone under the deleted project but alive under the other project. + assertThat(liveCountScoped("traces", Set.of(reusedId.toString()), workspaceId, projectId)) + .as("reused id is deleted under its own project") + .isZero(); + assertThat(liveCountScoped("traces", Set.of(reusedId.toString()), workspaceId, otherProjectId)) + .as("reused id survives under the other project — replay did not over-delete by id alone") + .isEqualTo(1L); + + // Newest-version-wins: the delta upserts are the surviving version after ReplacingMergeTree dedup. + assertThat(newestNames("traces", idStrings(deltaUpserted), workspaceId)) + .as("delta upserts win under FINAL dedup after the cutover") + .containsOnly(deltaName()); + + // Sharding-ready wrap: RENAME to *_local, front it with a Distributed table keyed on project_id. + wrapInDistributed(); + assertThat(liveCount("traces", idStrings(survivors), workspaceId)) + .as("the single-shard Distributed wrapper reads transparently") + .isEqualTo(survivors.size()); + assertThat(liveCount("traces", leakedIds, workspaceId)) + .as("deletions stay deleted when read through the Distributed wrapper") + .isZero(); + assertThat(liveCountScoped("traces", Set.of(reusedId.toString()), workspaceId, otherProjectId)) + .as("reused id still readable under the other project through the Distributed wrapper") + .isEqualTo(1L); + + // Rollback (Stage C) — the wrap is reversible without resurrecting post-cutover deletes. A sharding-aware app + // deletes on the local table, so simulate a post-wrap delete on `traces_local` and record it in the bridge with + // an empty project (the unresolved case), then roll back: drop the Distributed wrapper, promote the parked old + // data back to `traces`, and reverse-replay from cutover_start. + var postWrapDeleted = Set.of(survivors.getFirst().id().toString()); + recordDeletionEvents(postWrapDeleted, workspaceId, "", "user_request"); + execute("DELETE FROM traces_local WHERE workspace_id = :workspace_id AND id IN :ids", + statement -> statement.bind("workspace_id", workspaceId).bind("ids", postWrapDeleted)); + rollbackAfterWrap(cutoverStart); + + assertThat(isDistributed("traces")) + .as("rollback drops the Distributed wrapper; `traces` is a regular table again") + .isFalse(); + assertThat(liveCount("traces", postWrapDeleted, workspaceId)) + .as("post-wrap delete does not resurrect on the rolled-back table") + .isZero(); + assertThat(liveCount("traces", idStrings(survivors.subList(1, survivors.size())), workspaceId)) + .as("all other survivors are intact after rollback") + .isEqualTo(survivors.size() - 1); + assertThat(tableExists("traces_local_v2")) + .as("rollback ends in the canonical state: successor data parked as traces_local_v2") + .isTrue(); + assertThat(tableExists("traces_local")) + .as("no leftover sharding table after rollback") + .isFalse(); + } + + /** + * Rollback stage A (000004_rollback_stage_a): aborting before the EXCHANGE only discards the shadow — the live + * {@code traces} table, which the backfill never writes to, must be byte-for-byte untouched. + */ + @Test + void rollbackBeforeExchangeDiscardsShadowAndLeavesLiveUntouched() { + var workspaceId = UUID.randomUUID().toString(); + var projectId = ID_GENERATOR.generateId(); + var survivors = mintIds(SURVIVORS_PER_WEEK); + seedTraces(survivors, workspaceId, projectId); + + for (int week = 0; week < SEED_WEEKS; week++) { + backfillWeek(week); + } + assertThat(liveCount("traces_local_v2", idStrings(survivors), workspaceId)) + .as("shadow was backfilled before the abort") + .isEqualTo(survivors.size()); + var liveBefore = fingerprint("traces", Shape.OLD, workspaceId); + + rollbackDiscardShadow(); + + assertThat(liveCount("traces_local_v2", idStrings(survivors), workspaceId)) + .as("stage A discards the shadow copy") + .isZero(); + assertThat(fingerprint("traces", Shape.OLD, workspaceId)) + .as("stage A leaves the live table untouched") + .isEqualTo(liveBefore); + } + + /** + * Rollback stage B (000004_rollback_stage_b + reverse_replay): aborting after the EXCHANGE but before the wrap swaps + * the tables back and reverse-replays, so a delete that landed on the successor after cutover_start does not + * resurrect on the restored original. Exercises the reverse-replay's FULL-KEY branch (the comprehensive test covers + * the empty-project branch in stage C). + */ + @Test + void rollbackAfterExchangeSwapsBackWithoutResurrectingDeletes() { + var workspaceId = UUID.randomUUID().toString(); + var projectId = ID_GENERATOR.generateId(); + var survivors = mintIds(SURVIVORS_PER_WEEK); + var windowDeleted = mintIds(USER_DELETED_PER_WEEK); + seedTraces(survivors, workspaceId, projectId); + seedTraces(windowDeleted, workspaceId, projectId); + + var backfillStart = nowMicros(); + for (int week = 0; week < SEED_WEEKS; week++) { + backfillWeek(week); + } + // A delete during the window: bridged, applied to the source, then reconciled onto the destination by the replay. + recordDeletionEvents(idStrings(windowDeleted), workspaceId, projectId.toString(), "user_request"); + lightweightDelete(idStrings(windowDeleted), workspaceId); + deltaInsert(backfillStart); + replayDeletions(backfillStart); + + var cutoverStart = nowMicros(); + exchangeTables(); + assertThat(liveCount("traces", idStrings(survivors), workspaceId)) + .as("survivors present on the successor after the EXCHANGE") + .isEqualTo(survivors.size()); + assertThat(liveCount("traces", idStrings(windowDeleted), workspaceId)) + .as("window deletes did not leak across the EXCHANGE") + .isZero(); + + // Post-cutover delete on the new live table (a MergeTree post-EXCHANGE, so a lightweight DELETE works), captured + // with its project — the reverse-replay's full-key branch. + var postCutoverDeleted = Set.of(survivors.getFirst().id().toString()); + recordDeletionEvents(postCutoverDeleted, workspaceId, projectId.toString(), "user_request"); + lightweightDelete(postCutoverDeleted, workspaceId); + + rollbackExchangeBack(cutoverStart); + + assertThat(isDistributed("traces")) + .as("stage B restores a regular table") + .isFalse(); + assertThat(liveCount("traces", postCutoverDeleted, workspaceId)) + .as("post-cutover delete does not resurrect after the swap-back") + .isZero(); + assertThat(liveCount("traces", idStrings(windowDeleted), workspaceId)) + .as("window deletes stay deleted after the swap-back") + .isZero(); + assertThat(liveCount("traces", idStrings(survivors.subList(1, survivors.size())), workspaceId)) + .as("all other survivors are intact after the swap-back") + .isEqualTo(survivors.size() - 1); + assertThat(tableExists("traces_local_v2")) + .as("canonical state: successor parked as traces_local_v2") + .isTrue(); + assertThat(tableExists("traces_local")) + .as("no leftover sharding table after stage B") + .isFalse(); + } + + /** + * rollback.sh refuses a wrong-stage run by reading two signals off the live {@code traces} — its engine and its + * {@code end_time} nullability — and aborting unless they match the requested stage (the guard that stops a stage-A + * {@code TRUNCATE} from destroying the parked original once the EXCHANGE has run). This drives the DB through the + * three cutover states and asserts those signals are distinct in each, so the guard can always tell which stage is + * valid. It validates the signals the guard reads, not the bash parsing itself — the script's own execution is + * covered by the staging dry-run. + */ + @Test + void rollbackTopologySignalsDistinguishEveryStage() { + var workspaceId = UUID.randomUUID().toString(); + var projectId = ID_GENERATOR.generateId(); + seedTraces(mintIds(SURVIVORS_PER_WEEK), workspaceId, projectId); + for (int week = 0; week < SEED_WEEKS; week++) { + backfillWeek(week); + } + + // Pre-EXCHANGE — original schema: a MergeTree with Nullable end_time. Only stage A is valid. + assertThat(tableEngine("traces")) + .as("pre-EXCHANGE traces is a MergeTree, not Distributed") + .doesNotContain("Distributed"); + assertThat(columnType("traces", "end_time")) + .as("pre-EXCHANGE end_time is Nullable (original schema)") + .startsWith("Nullable"); + + var cutoverStart = nowMicros(); + exchangeTables(); + + // Post-EXCHANGE — successor schema under `traces`: still a MergeTree, but end_time is non-Nullable. Stage B is + // valid; a stage-A run must now abort, since its guard requires Nullable end_time. + assertThat(tableEngine("traces")) + .as("post-EXCHANGE traces is still a MergeTree") + .doesNotContain("Distributed"); + assertThat(columnType("traces", "end_time")) + .as("post-EXCHANGE end_time is non-Nullable (successor schema)") + .doesNotContain("Nullable"); + + wrapInDistributed(); + + // Post-wrap — Distributed wrapper: only stage C is valid. + assertThat(tableEngine("traces")) + .as("post-wrap traces is a Distributed wrapper") + .isEqualTo("Distributed"); + + // Restore the canonical baseline (traces = original, traces_local_v2 = successor) so @BeforeEach's reset — which + // assumes a regular `traces` — works for the next test. No deletes were bridged, so the reverse-replay is a no-op. + rollbackAfterWrap(cutoverStart); + } + + /** + * A trace deleted and then re-created/updated under the SAME id during the window is bridged as deleted but is live + * again on the source (ids are client-supplied; the newer insert wins under FINAL). The replay's resurrection guard + * must keep it on the destination — deleting it by key would drop a row that is live on the source (silent data + * loss). Mirrors the delete_traffic + live_traffic overlap the local rehearsal produces. With the guard removed this + * test fails (the resurrected rows come back zero). + * + *

The replay is run twice to also pin its idempotence: the runbook has the operator re-run delta+replay to + * convergence, so a second replay must not change the result — in particular it must not eventually drop the + * resurrected (live-on-source) rows. + * + *

Parameterized over the bridge capture shape so both replay branches carry the guard: a delete captured + * WITH its project exercises the full-key branch, one captured with an empty project (the workspace-scoped delete + * fallback) exercises the {@code (workspace_id, id)} branch. Both branches must spare a resurrected id. + */ + @ParameterizedTest(name = "resurrection guard holds on the {0}-project replay branch") + @ValueSource(booleans = {true, false}) + void deleteThenResurrectSurvivesTheReplay(boolean resolvedProject) { + var workspaceId = UUID.randomUUID().toString(); + var projectId = ID_GENERATOR.generateId(); + // Capture shape selects the replay branch: the resolved project (full key) or an empty project (workspace-scoped). + var captureProject = resolvedProject ? projectId.toString() : ""; + var survivors = mintIds(SURVIVORS_PER_WEEK); + var resurrected = mintIds(3); // deleted then re-created under the same id + var stayDeleted = mintIds(3); // deleted and NOT re-created + seedTraces(survivors, workspaceId, projectId); + seedTraces(resurrected, workspaceId, projectId); + seedTraces(stayDeleted, workspaceId, projectId); + + var backfillStart = nowMicros(); + for (int week = 0; week < SEED_WEEKS; week++) { + backfillWeek(week); + } + + // During the window: delete both cohorts (bridged), then re-create the resurrected cohort under the same ids + // with a fresh last_updated_at — the newer version wins under FINAL, so they are live again on the source (caught + // by the delta's last_updated_at arm since their created_at stays historical). + recordDeletionEvents(idStrings(resurrected), workspaceId, captureProject, "user_request"); + lightweightDelete(idStrings(resurrected), workspaceId); + recordDeletionEvents(idStrings(stayDeleted), workspaceId, captureProject, "user_request"); + lightweightDelete(idStrings(stayDeleted), workspaceId); + insertRows(resurrected, workspaceId, projectId, "resurrected", _ -> Instant.now()); + + deltaInsert(backfillStart); + // Run the replay twice: it must be idempotent (re-runnable to convergence) and must not drop the resurrected + // live-on-source rows on the second pass. + replayDeletions(backfillStart); + replayDeletions(backfillStart); + + assertThat(liveCount("traces_local_v2", idStrings(resurrected), workspaceId)) + .as("resurrection guard is idempotent: a deleted-then-recreated id stays live after a repeated replay") + .isEqualTo(resurrected.size()); + assertThat(liveCount("traces_local_v2", idStrings(stayDeleted), workspaceId)) + .as("a deleted-and-not-recreated id is removed from the destination") + .isZero(); + assertThat(liveCount("traces_local_v2", idStrings(survivors), workspaceId)) + .as("untouched survivors are intact") + .isEqualTo(survivors.size()); + } + + /** + * Schema-drift guard. The cutover copies a fixed column list, and the fidelity fingerprint also lists fixed + * columns — so a base column added to {@code traces} by a future migration would be silently left uncopied, with no + * existing check failing. This asserts the cutover's {@link #COPIED_COLUMNS} equals the live stored columns of + * {@code traces}, and that {@code traces_local_v2} mirrors them plus only the {@code is_deleted} meta-column. Adding + * a stored column to either table fails this until it is added to {@code COPIED_COLUMNS} (and thus to the copy). + */ + @Test + void cutoverCopiesEveryBaseColumn() { + var tracesBase = baseColumns("traces"); + var successorBase = baseColumns("traces_local_v2"); + var copied = Arrays.stream(COPIED_COLUMNS.split(",")) + .map(String::trim) + .filter(column -> !column.isEmpty()) + .collect(Collectors.toUnmodifiableSet()); + + assertThat(copied) + .as("cutover COPIED_COLUMNS must equal the stored (non-materialized) columns of traces") + .isEqualTo(tracesBase); + assertThat(successorBase) + .as("traces_local_v2 stored columns = traces stored columns + the is_deleted meta-column") + .isEqualTo(union(tracesBase, Set.of("is_deleted"))); + } + + /** + * Materialized-column parity guard, the complement to {@link #cutoverCopiesEveryBaseColumn()}. The backfill does not + * copy materialized columns (the destination recomputes them), so they are outside the copy guard — but the two + * tables must still expose the SAME materialized columns for as long as both exist, or a materialized column added + * to one by a future migration and not the other leaves post-cutover queries referencing a column the live table + * lacks. This checks presence; their values are covered by {@link #derivedFingerprint} / {@link #durationMismatches}. + */ + @Test + void successorMaterializedColumnsMatchSource() { + assertThat(materializedColumns("traces_local_v2")) + .as("traces_local_v2 must expose exactly the same MATERIALIZED columns as traces") + .isEqualTo(materializedColumns("traces")); + } + + /** Stored (physically materialized) columns of a table — excludes {@code MATERIALIZED} / {@code ALIAS} columns. */ + private Set baseColumns(String table) { + return columnNames(table, "default_kind NOT IN ('MATERIALIZED', 'ALIAS')"); + } + + /** MATERIALIZED (recomputed, not stored-from-insert) columns of a table. */ + private Set materializedColumns(String table) { + return columnNames(table, "default_kind = 'MATERIALIZED'"); + } + + /** Column names of a table filtered by a {@code system.columns} predicate. */ + private Set columnNames(String table, String defaultKindPredicate) { + var sql = """ + SELECT name + FROM system.columns + WHERE database = :db + AND table = :t + AND %s + """.formatted(defaultKindPredicate); + return template.stream(connection -> Flux.from(connection.createStatement(sql) + .bind("db", DATABASE_NAME) + .bind("t", table) + .execute()) + .flatMap(result -> result.map((row, ignored) -> row.get("name", String.class)))) + .collectList().block().stream().collect(Collectors.toUnmodifiableSet()); + } + + // --- cutover steps (mirror the runbook SQL) ------------------------------------------------------------------ + + /** + * The runbook's backfill INSERT SELECT for one week. Columns map by name; {@code end_time} and {@code ttft} are the + * two denullified columns, coalesced to their sentinels (epoch / NaN); {@code is_deleted} is omitted so the new + * column defaults to 0. {@code apply_deleted_mask} stays at its default 1, so masked source rows are skipped. + */ + private void backfillWeek(int week) { + var weekLo = ClickHouseDateTimeFormat.formatMicros(weekInstant(week, 0)); + var weekHi = ClickHouseDateTimeFormat.formatMicros(weekInstant(week + 1, 0)); + execute(""" + INSERT INTO traces_local_v2 ( + %s + ) + SELECT + %s + FROM traces + WHERE created_at >= parseDateTime64BestEffort(:week_lo, 9) + AND created_at < parseDateTime64BestEffort(:week_hi, 9) + SETTINGS max_insert_block_size = 100000 + """.formatted(COPIED_COLUMNS, COPIED_SELECT), + statement -> statement.bind("week_lo", weekLo).bind("week_hi", weekHi)); + } + + /** + * The delta-insert: re-copy every row written during the backfill window. Anchored on + * {@code created_at OR last_updated_at >= backfill_start} so it is complete regardless of the client-supplied + * {@code last_updated_at} on the batch-ingest path (see class Javadoc). + */ + private void deltaInsert(String backfillStart) { + execute(""" + INSERT INTO traces_local_v2 ( + %s + ) + SELECT + %s + FROM traces + WHERE created_at >= parseDateTime64BestEffort(:backfill_start, 6) + OR last_updated_at >= parseDateTime64BestEffort(:backfill_start, 6) + SETTINGS max_insert_block_size = 100000 + """.formatted(COPIED_COLUMNS, COPIED_SELECT), + statement -> statement.bind("backfill_start", backfillStart)); + } + + /** + * Reads the bridge for the cutover window and removes the captured deletes from the destination in a single + * mutation (mirrors 000002). Two branches, because the delete-by-ids path does not always resolve a trace's project: + * events WITH a project match the full key {@code (workspace_id, project_id, id)} (exact; a reused id in another + * project is untouched); events captured WITHOUT a project match {@code (workspace_id, id)} — otherwise those + * deletions silently leak across the swap. Each branch also requires the id is NOT currently live on the source + * (the resurrection guard), so a deleted-then-recreated id is not dropped. Returns the wall time so the runbook can + * size it against the buffer window. + */ + private long replayDeletions(String backfillStart) { + var start = System.nanoTime(); + // allow_nondeterministic_mutations: a lightweight DELETE with a cross-table subquery is flagged + // nondeterministic, but deletion_events_local is replicated and identical on every node and the window + // predicate is fixed, so the subquery resolves to the same set on every replica. lightweight_deletes_sync = 2 + // waits for the mutation on every replica before returning, so verify/EXCHANGE never race an un-applied mask. + execute(""" + DELETE FROM traces_local_v2 + WHERE ( + (workspace_id, project_id, id) IN ( + SELECT + workspace_id, + toFixedString(project_id, 36), + toFixedString(deleted_id, 36) + FROM deletion_events_local + WHERE source_table = 'traces' + AND event_time >= parseDateTime64BestEffort(:backfill_start, 6) + AND project_id != '' + ) + AND (workspace_id, project_id, id) NOT IN ( + SELECT + workspace_id, + project_id, + id + FROM traces + WHERE id IN ( + SELECT toFixedString(deleted_id, 36) + FROM deletion_events_local + WHERE source_table = 'traces' + AND event_time >= parseDateTime64BestEffort(:backfill_start, 6) + ) + ) + ) + OR ( + (workspace_id, id) IN ( + SELECT + workspace_id, + toFixedString(deleted_id, 36) + FROM deletion_events_local + WHERE source_table = 'traces' + AND event_time >= parseDateTime64BestEffort(:backfill_start, 6) + AND project_id = '' + ) + AND (workspace_id, id) NOT IN ( + SELECT + workspace_id, + id + FROM traces + WHERE id IN ( + SELECT toFixedString(deleted_id, 36) + FROM deletion_events_local + WHERE source_table = 'traces' + AND event_time >= parseDateTime64BestEffort(:backfill_start, 6) + ) + ) + ) + SETTINGS allow_nondeterministic_mutations = 1, + lightweight_deletes_sync = 2 + """, statement -> statement.bind("backfill_start", backfillStart)); + return (System.nanoTime() - start) / 1_000_000L; + } + + /** + * The atomic swap (000003 exchange block): EXCHANGE puts the successor under {@code traces} and the old data under + * {@code traces_local_v2}, then a RENAME moves the old data to {@code traces_pre_cutover_backup} so its name says it + * is the retained pre-cutover backup, not the "v2" successor. + */ + private void exchangeTables() { + execute("EXCHANGE TABLES traces AND traces_local_v2 ON CLUSTER '{cluster}'", _ -> { + }); + execute("RENAME TABLE traces_local_v2 TO traces_pre_cutover_backup ON CLUSTER '{cluster}'", _ -> { + }); + } + + private void wrapInDistributed() { + execute("RENAME TABLE traces TO traces_local ON CLUSTER '{cluster}'", _ -> { + }); + execute(("CREATE TABLE traces ON CLUSTER '{cluster}' AS traces_local " + + "ENGINE = Distributed('{cluster}', '" + DATABASE_NAME + "', 'traces_local', sipHash64(project_id))"), + _ -> { + }); + } + + /** Rollback stage A (000004_rollback_stage_a): discard the disposable shadow; the live `traces` is untouched. */ + private void rollbackDiscardShadow() { + execute("TRUNCATE TABLE traces_local_v2 ON CLUSTER '{cluster}'", _ -> { + }); + } + + /** + * Rollback stage B (000004_rollback_stage_b + reverse_replay): swap the original (parked as + * {@code traces_pre_cutover_backup}) back to live {@code traces}, rename the now-parked successor back to + * {@code traces_local_v2}, then reverse-replay so a delete on the successor since {@code cutoverStart} does not + * resurrect on the restored original. + */ + private void rollbackExchangeBack(String cutoverStart) { + execute("EXCHANGE TABLES traces AND traces_pre_cutover_backup ON CLUSTER '{cluster}'", _ -> { + }); + execute("RENAME TABLE traces_pre_cutover_backup TO traces_local_v2 ON CLUSTER '{cluster}'", _ -> { + }); + reverseReplay(cutoverStart); + } + + /** + * Rollback stage C (000004_rollback_stage_c + reverse_replay): drop the Distributed wrapper (no data), then in one + * atomic RENAME promote the parked original (from {@code traces_pre_cutover_backup}) back to {@code traces} and park + * the successor data back under {@code traces_local_v2}, ending in the canonical state with no leftover names. Then + * reverse-replay. + */ + private void rollbackAfterWrap(String cutoverStart) { + execute("DROP TABLE traces ON CLUSTER '{cluster}' SYNC", _ -> { + }); + execute(""" + RENAME TABLE + traces_pre_cutover_backup TO traces, + traces_local TO traces_local_v2 + ON CLUSTER '{cluster}' + """, _ -> { + }); + reverseReplay(cutoverStart); + } + + /** + * The shared reverse-replay (000004_rollback_reverse_replay): re-apply the deletes captured since + * {@code cutoverStart} onto the restored original, so they do not resurrect. Two branches — full key for events with + * a project, {@code (workspace_id, id)} for the workspace-scoped (empty-project) fallback. Unlike the forward replay + * it carries NO resurrection guard by design: rollback abandons post-cutover writes while honoring post-cutover + * deletes, so a bridged id is masked unconditionally (a guard would undo the user's delete). See the .sql header. + */ + private void reverseReplay(String cutoverStart) { + execute(""" + DELETE FROM traces + WHERE (workspace_id, project_id, id) IN ( + SELECT + workspace_id, + toFixedString(project_id, 36), + toFixedString(deleted_id, 36) + FROM deletion_events_local + WHERE source_table = 'traces' + AND event_time >= parseDateTime64BestEffort(:cutover_start, 6) + AND project_id != '' + ) + OR (workspace_id, id) IN ( + SELECT + workspace_id, + toFixedString(deleted_id, 36) + FROM deletion_events_local + WHERE source_table = 'traces' + AND event_time >= parseDateTime64BestEffort(:cutover_start, 6) + AND project_id = '' + ) + SETTINGS allow_nondeterministic_mutations = 1, + lightweight_deletes_sync = 2 + """, statement -> statement.bind("cutover_start", cutoverStart)); + } + + private boolean isDistributed(String table) { + return "Distributed".equals(tableEngine(table)); + } + + /** The table's engine (e.g. {@code ReplicatedReplacingMergeTree}, {@code Distributed}) from {@code system.tables}. */ + private String tableEngine(String table) { + return template.nonTransaction(connection -> Mono.from(connection.createStatement( + "SELECT engine FROM system.tables WHERE database = :db AND name = :t") + .bind("db", DATABASE_NAME) + .bind("t", table) + .execute()) + .flatMap(result -> Mono.from(result.map((row, ignored) -> row.get("engine", String.class))))) + .block(); + } + + /** A column's declared type (e.g. {@code Nullable(DateTime64(9, 'UTC'))}) from {@code system.columns}. */ + private String columnType(String table, String column) { + return template.nonTransaction(connection -> Mono.from(connection.createStatement( + "SELECT type FROM system.columns WHERE database = :db AND table = :t AND name = :c") + .bind("db", DATABASE_NAME) + .bind("t", table) + .bind("c", column) + .execute()) + .flatMap(result -> Mono.from(result.map((row, ignored) -> row.get("type", String.class))))) + .block(); + } + + private boolean tableExists(String table) { + return Boolean.TRUE.equals(template.nonTransaction(connection -> Mono.from(connection.createStatement( + "SELECT count() FROM system.tables WHERE database = :db AND name = :t") + .bind("db", DATABASE_NAME) + .bind("t", table) + .execute()) + .flatMap(result -> Mono.from(result.map((row, ignored) -> row.get(0, Long.class) > 0)))) + .block()); + } + + // --- seeding / mutation helpers ------------------------------------------------------------------------------ + + private void seedTraces(List ids, String workspaceId, UUID projectId) { + insertRows(ids, workspaceId, projectId, "seed", CategorizedId::createdAt); + } + + /** + * Batch-insert rows following the {@code TraceDAO.BATCH_INSERT} shape: {@code created_at} is the row's minted time, + * {@code last_updated_at} is whatever {@code lastUpdatedAt} yields (server-now for upserts, a backdated stamp to + * exercise the delta's {@code created_at} arm). + */ + private void insertRows(List ids, String workspaceId, UUID projectId, String name, + Function lastUpdatedAt) { + var sql = TemplateUtils.getBatchSql(""" + INSERT INTO traces ( + id, + workspace_id, + project_id, + name, + created_at, + last_updated_at + ) + FORMAT Values + , + :workspace_id, + :project_id, + :name, + :created_at, + :last_updated_at + ) + , + }> + ; + """, ids.size()).render(); + execute(sql, statement -> { + statement.bind("workspace_id", workspaceId).bind("project_id", projectId).bind("name", name); + for (int i = 0; i < ids.size(); i++) { + statement.bind("id" + i, ids.get(i).id()) + .bind("created_at" + i, ClickHouseDateTimeFormat.formatMicros(ids.get(i).createdAt())) + .bind("last_updated_at" + i, + ClickHouseDateTimeFormat.formatMicros(lastUpdatedAt.apply(ids.get(i)))); + } + }); + } + + /** + * Seeds a small cohort with EVERY migrated column populated with distinct, varied values — at nanosecond + * {@code created_at} precision, and a share of NULL {@code end_time} / {@code ttft}. The fingerprint is + * workspace-scoped, so these rows make it sensitive to every column and to the ns->us truncation: an all-default row + * would hash-match on both sides even if the copy dropped a column. They are ordinary survivors (historical + * created_at, never deleted). Inline literals (not binds) keep array/enum/NULL formatting reliable. Returns the ids. + */ + private List seedFidelityCohort(String workspaceId, UUID projectId) { + var ids = new ArrayList(); + var rows = new StringBuilder(); + int n = SEED_WEEKS * 3; + for (int i = 0; i < n; i++) { + var createdAt = weekInstant(i % SEED_WEEKS, i + 1).plusNanos(i * 137L + 3); // sub-microsecond ns remainder + var id = ID_GENERATOR.generateId(createdAt).toString(); + ids.add(id); + var createdNs = ClickHouseDateTimeFormat.formatNanos(createdAt); + var endTime = (i % 3 == 0) + ? "NULL" + : "toDateTime64('" + ClickHouseDateTimeFormat.formatNanos(createdAt.plusMillis(50L + i)) + "', 9)"; + var ttft = (i % 4 == 0) ? "NULL" : String.valueOf(0.01 * (i + 1)); + var errorInfo = (i % 7 == 0) ? "{\"type\":\"Err" + i + "\"}" : ""; + var threadId = (i % 2 == 0) ? "" : "thread-" + i; + rows.append(i == 0 ? "" : ",\n") + .append("('").append(id).append("','").append(workspaceId).append("','").append(projectId) + .append("','seed-fidelity',") + .append("toDateTime64('").append(createdNs).append("', 9),") // start_time + .append(endTime).append(",") + .append("'in-").append(i).append("','out-").append(i).append("',") + .append("'{\"model\":\"m").append(i).append("\",\"n\":").append(i).append("}',") // metadata + .append("['tag").append(i).append("','g").append(i % 4).append("'],") // tags + .append("toDateTime64('").append(createdNs).append("', 9),") // created_at (ns) + .append("toDateTime64('").append(ClickHouseDateTimeFormat.formatMicros(createdAt)).append("', 6),") + .append("'user").append(i % 5).append("','user").append((i + 1) % 5).append("',") // *_by + .append("'").append(errorInfo).append("','").append(threadId).append("',") + .append("'").append(i % 9 == 0 ? "hidden" : "default").append("',") + .append(10001 + (i % 2) * 10000).append(",") // truncation_threshold + .append("'slim-in-").append(i).append("','slim-out-").append(i).append("',") + .append(ttft).append(",") + .append("'").append(FIDELITY_SOURCES[i % FIDELITY_SOURCES.length]).append("',") + .append("'").append(FIDELITY_ENVIRONMENTS[i % FIDELITY_ENVIRONMENTS.length]).append("')"); + } + execute("INSERT INTO traces (id, workspace_id, project_id, name, start_time, end_time, input, output, metadata, " + + "tags, created_at, last_updated_at, created_by, last_updated_by, error_info, thread_id, " + + "visibility_mode, truncation_threshold, input_slim, output_slim, ttft, source, environment) VALUES " + + rows, _ -> { + }); + return ids; + } + + private void lightweightDelete(Set ids, String workspaceId) { + execute(""" + DELETE FROM traces + WHERE workspace_id = :workspace_id + AND id IN :ids + """, + statement -> statement.bind("workspace_id", workspaceId).bind("ids", ids)); + } + + private void lightweightDeleteScoped(Set ids, String workspaceId, UUID projectId) { + execute(""" + DELETE FROM traces + WHERE workspace_id = :workspace_id + AND project_id = :project_id + AND id IN :ids + """, + statement -> statement + .bind("workspace_id", workspaceId) + .bind("project_id", projectId) + .bind("ids", ids)); + } + + /** + * Batch INSERT into the bridge, mirroring {@code DeletionEventDAO}'s write shape. {@code projectId} is a string so a + * caller can pass {@code ""} to reproduce an unresolved delete (the delete-by-ids path records an empty project when + * it cannot resolve a trace's project). + */ + private void recordDeletionEvents(Set ids, String workspaceId, String projectId, String reason) { + var idList = List.copyOf(ids); + var sql = TemplateUtils.getBatchSql(""" + INSERT INTO deletion_events_local ( + source_table, + workspace_id, + project_id, + deleted_id, + deletion_reason + ) + FORMAT Values + , + :reason + ) + , + }> + ; + """, idList.size()).render(); + execute(sql, statement -> { + statement.bind("workspace_id", workspaceId).bind("project_id", projectId).bind("reason", reason); + for (int i = 0; i < idList.size(); i++) { + statement.bind("deleted_id" + i, idList.get(i)); + } + }); + } + + // --- query helpers ------------------------------------------------------------------------------------------- + + /** Distinct live (mask-honored) ids from {@code table} within {@code ids} — collapses ReplacingMergeTree versions. */ + private long liveCount(String table, Set ids, String workspaceId) { + if (ids.isEmpty()) { + return 0L; + } + var sql = """ + SELECT uniqExact(id) AS c + FROM %s + WHERE workspace_id = :workspace_id + AND id IN :ids + """.formatted(table); + return template + .nonTransaction(connection -> Mono + .from(connection.createStatement(sql) + .bind("workspace_id", workspaceId) + .bind("ids", ids) + .execute()) + .flatMap(result -> Mono.from(result.map((row, ignored) -> row.get("c", Long.class))))) + .block(); + } + + private long liveCountScoped(String table, Set ids, String workspaceId, UUID projectId) { + var sql = """ + SELECT uniqExact(id) AS c + FROM %s + WHERE workspace_id = :workspace_id + AND project_id = :project_id + AND id IN :ids + """.formatted(table); + return template + .nonTransaction(connection -> Mono + .from(connection.createStatement(sql) + .bind("workspace_id", workspaceId) + .bind("project_id", projectId) + .bind("ids", ids) + .execute()) + .flatMap(result -> Mono.from(result.map((row, ignored) -> row.get("c", Long.class))))) + .block(); + } + + private Set newestNames(String table, Set ids, String workspaceId) { + var sql = """ + SELECT name + FROM %s FINAL + WHERE workspace_id = :workspace_id + AND id IN :ids + """.formatted(table); + return template.stream(connection -> Flux.from(connection.createStatement(sql) + .bind("workspace_id", workspaceId) + .bind("ids", ids) + .execute()) + .flatMap(result -> result.map((row, ignored) -> row.get("name", String.class)))) + .collectList().block().stream().collect(Collectors.toUnmodifiableSet()); + } + + private String nowMicros() { + return template.nonTransaction(connection -> Mono.from(connection.createStatement( + "SELECT toString(now64(6)) AS n") + .execute()) + .flatMap(result -> Mono.from(result.map((row, ignored) -> row.get("n", String.class))))) + .block(); + } + + /** + * A migration schema shape. OLD is the source layout (Nullable end_time/ttft, nanosecond timestamps); NEW is the + * successor layout (epoch / NaN sentinels, microsecond timestamps). The per-row hash normalizes each shape to the + * same canonical value for a faithfully-migrated row, so equal fingerprints prove no data was altered. + */ + private enum Shape { + OLD, + NEW + } + + @Builder(toBuilder = true) + private record Fingerprint(long count, long checksum) { + } + + /** + * Order-independent (count, checksum) fingerprint of the deduped, mask-honored, normalized rows for a workspace. + * {@code FINAL} collapses ReplacingMergeTree versions to the winner; the default {@code apply_deleted_mask} excludes + * lightweight-deleted rows; the per-row {@code cityHash64} canonicalizes the two schema shapes so a faithful copy + * hashes identically. {@code sum} needs no sort (bounded memory) and, unlike {@code groupBitXor}, does not cancel a + * colliding pair within a table; with {@code id} in every row hash, a changed, missing or extra row flips the + * aggregate. Materialized/derived columns and {@code is_deleted} are excluded — they are recomputed, not migrated + * data; their expression parity is checked separately by {@link #derivedFingerprint} and {@link #durationMismatches}. + */ + private Fingerprint fingerprint(String table, Shape shape, String workspaceId) { + var hash = rowHash(shape == Shape.OLD ? OLD_HASH_OVERRIDES : NEW_HASH_OVERRIDES); + var sql = """ + SELECT + count() AS c, + sum(%s) AS h + FROM %s FINAL + WHERE workspace_id = :workspace_id + """.formatted(hash, table); + return template.nonTransaction(connection -> Mono.from(connection.createStatement(sql) + .bind("workspace_id", workspaceId) + .execute()) + .flatMap(result -> Mono.from(result.map((row, ignored) -> Fingerprint.builder() + .count(row.get("c", Long.class)) + .checksum(row.get("h", Long.class)) + .build())))) + .block(); + } + + /** + * (count, checksum) over the DETERMINISTIC derived columns of the fidelity cohort — {@code id_at} (the partition + * key), the three {@code *_length}s, {@code truncated_input} / {@code truncated_output} and {@code output_keys}. + * Each is the same MATERIALIZED expression over faithfully-copied base columns on both tables, so equal fingerprints + * prove the successor's expressions did not drift from the source's. {@code duration} is checked separately + * ({@link #durationMismatches}) because its value legitimately differs by up to the ns-to-us truncation. + */ + private Fingerprint derivedFingerprint(String table, String workspaceId) { + var sql = """ + SELECT + count() AS c, + sum(cityHash64( + id, + id_at, + input_length, + output_length, + metadata_length, + truncated_input, + truncated_output, + toString(output_keys))) AS h + FROM %s FINAL + WHERE workspace_id = :workspace_id + AND name = 'seed-fidelity' + """.formatted(table); + return template.nonTransaction(connection -> Mono.from(connection.createStatement(sql) + .bind("workspace_id", workspaceId) + .execute()) + .flatMap(result -> Mono.from(result.map((row, ignored) -> Fingerprint.builder() + .count(row.get("c", Long.class)) + .checksum(row.get("h", Long.class)) + .build())))) + .block(); + } + + /** + * Count of fidelity-cohort rows whose {@code duration} disagrees between source and destination beyond the intended + * ns-to-us truncation. The source computes duration from nanosecond timestamps and is {@code NULL} when unset; the + * successor computes it from the microsecond copy and is {@code NaN} when unset. So a faithful row is unset on both + * (source NULL, dest NaN) or set on both within one microsecond (0.001 ms); anything else is a real divergence. + */ + private long durationMismatches(String workspaceId) { + var sql = """ + SELECT count() AS c + FROM ( + SELECT id, duration AS d FROM traces FINAL + WHERE workspace_id = :workspace_id AND name = 'seed-fidelity' + ) AS s + INNER JOIN ( + SELECT id, duration AS d FROM traces_local_v2 FINAL + WHERE workspace_id = :workspace_id AND name = 'seed-fidelity' + ) AS t USING (id) + WHERE NOT ( + (isNaN(t.d) AND s.d IS NULL) + OR (NOT isNaN(t.d) AND s.d IS NOT NULL AND abs(s.d - t.d) <= 0.0015) + ) + """; + return template.nonTransaction(connection -> Mono.from(connection.createStatement(sql) + .bind("workspace_id", workspaceId) + .execute()) + .flatMap(result -> Mono.from(result.map((row, ignored) -> row.get("c", Long.class))))) + .block(); + } + + // Canonical per-row hash, BUILT from COPIED_COLUMNS ({@link #rowHash}) so it covers every copied column by + // construction: a column added to COPIED_COLUMNS (which cutoverCopiesEveryBaseColumn pins to the live schema) is + // automatically hashed and can never be silently left value-unverified. Each column hashes as-is unless it needs + // shape-specific normalization, supplied by these override maps: timestamps as their microsecond epoch (ns + // truncated to us, matching the copy); absent end_time as 0 (source NULL / dest epoch) and absent ttft as 'nan' + // (source NULL / dest NaN); enums and project_id via toString; tags joined on the \x1f unit separator. A future + // denullified column needs a matching override in both maps; without one it still hashes as-is (included, just not + // normalized), and a wrong sentinel there makes dest != source so the fidelity assertion still catches it. + private static final Map OLD_HASH_OVERRIDES = Map.ofEntries( + Map.entry("project_id", "toString(project_id)"), + Map.entry("start_time", "toUnixTimestamp64Micro(toDateTime64(start_time, 6))"), + Map.entry("end_time", "coalesce(toUnixTimestamp64Micro(toDateTime64(end_time, 6)), toInt64(0))"), + Map.entry("created_at", "toUnixTimestamp64Micro(toDateTime64(created_at, 6))"), + Map.entry("last_updated_at", "toUnixTimestamp64Micro(toDateTime64(last_updated_at, 6))"), + Map.entry("tags", "arrayStringConcat(tags, '\\x1f')"), + Map.entry("visibility_mode", "toString(visibility_mode)"), + Map.entry("ttft", "if(ttft IS NULL, 'nan', toString(ttft))"), + Map.entry("source", "toString(source)"), + Map.entry("environment", "toString(environment)")); + + private static final Map NEW_HASH_OVERRIDES = Map.ofEntries( + Map.entry("project_id", "toString(project_id)"), + Map.entry("start_time", "toUnixTimestamp64Micro(start_time)"), + Map.entry("end_time", "toUnixTimestamp64Micro(end_time)"), + Map.entry("created_at", "toUnixTimestamp64Micro(created_at)"), + Map.entry("last_updated_at", "toUnixTimestamp64Micro(last_updated_at)"), + Map.entry("tags", "arrayStringConcat(tags, '\\x1f')"), + Map.entry("visibility_mode", "toString(visibility_mode)"), + Map.entry("ttft", "if(isNaN(ttft), 'nan', toString(ttft))"), + Map.entry("source", "toString(source)"), + Map.entry("environment", "toString(environment)")); + + /** + * The per-row fidelity hash for a shape, generated from {@link #COPIED_COLUMNS} in order so every copied column is + * hashed. Each column contributes its {@code overrides} expression, or the bare column name when no normalization is + * needed. Argument order matches on both shapes (both iterate COPIED_COLUMNS), so a faithfully-migrated row hashes + * identically under {@link #OLD_HASH_OVERRIDES} and {@link #NEW_HASH_OVERRIDES}. + */ + private static String rowHash(Map overrides) { + var args = Arrays.stream(COPIED_COLUMNS.split(",")) + .map(String::trim) + .filter(column -> !column.isEmpty()) + .map(column -> overrides.getOrDefault(column, column)) + .collect(Collectors.joining(",\n ")); + return "cityHash64(\n " + args + ")"; + } + + // --- primitives ---------------------------------------------------------------------------------------------- + + private void execute(String sql, Consumer binder) { + template.nonTransaction(connection -> { + var statement = connection.createStatement(sql); + binder.accept(statement); + return Mono.from(statement.execute()).flatMap(result -> Mono.from(result.getRowsUpdated())).then(); + }).block(); + } + + private List mintIds(int perWeek) { + var ids = new ArrayList(); + for (int week = 0; week < SEED_WEEKS; week++) { + for (int i = 0; i < perWeek; i++) { + var createdAt = weekInstant(week, i + 1); + ids.add(CategorizedId.builder().id(ID_GENERATOR.generateId(createdAt)).createdAt(createdAt).build()); + } + } + return ids; + } + + /** Ids created "now" — used for rows written during the window, so their created_at is >= backfill_start. */ + private List mintNowIds(int count) { + var ids = new ArrayList(); + for (int i = 0; i < count; i++) { + var createdAt = Instant.now(); + ids.add(CategorizedId.builder().id(ID_GENERATOR.generateId(createdAt)).createdAt(createdAt).build()); + } + return ids; + } + + private static Set idStrings(List ids) { + return ids.stream().map(id -> id.id().toString()).collect(Collectors.toUnmodifiableSet()); + } + + private static Set union(Set a, Set b) { + var union = new ArrayList<>(a); + union.addAll(b); + return Set.copyOf(union); + } + + private static String deltaName() { + return "delta-upserted"; + } + + /** A within-day offset so ids/created_at in the same week are distinct but stay inside their weekly partition. */ + private Instant weekInstant(int weekOffset, int minuteOffset) { + return ANCHOR_MONDAY.plusWeeks(weekOffset).atTime(1, 0).plusSeconds(minuteOffset).toInstant(ZoneOffset.UTC); + } + + @Builder(toBuilder = true) + private record CategorizedId(UUID id, Instant createdAt) { + } +} diff --git a/deployment/docker-compose/docker-compose.yaml b/deployment/docker-compose/docker-compose.yaml index de2ab8a0330..27e59b40093 100644 --- a/deployment/docker-compose/docker-compose.yaml +++ b/deployment/docker-compose/docker-compose.yaml @@ -173,6 +173,15 @@ services: TOGGLE_OLLIE_ENABLED: ${TOGGLE_OLLIE_ENABLED:-"false"} ANALYTICS_DB_READ_ONLY_FREEFORM_SQL_USER: ${ANALYTICS_DB_READ_ONLY_FREEFORM_SQL_USER:-comet_readonly_freeform_sql_user} ANALYTICS_DB_READ_ONLY_FREEFORM_SQL_PASS: ${ANALYTICS_DB_READ_ONLY_FREEFORM_SQL_PASS:-opik} + # Traces cutover knobs (data-migrations/traces-local-v2-cutover). Default off / unset; the operator sets these + # for the backfill-to-cutover window. + ANALYTICS_DB_DATA_MODEL_TRACE_DELETION_EVENTS_CAPTURE_ENABLED: ${ANALYTICS_DB_DATA_MODEL_TRACE_DELETION_EVENTS_CAPTURE_ENABLED:-false} + ANALYTICS_DB_DATA_MODEL_SPAN_DELETION_EVENTS_CAPTURE_ENABLED: ${ANALYTICS_DB_DATA_MODEL_SPAN_DELETION_EVENTS_CAPTURE_ENABLED:-false} + ANALYTICS_DB_ASYNC_INSERT_BUSY_TIMEOUT_MAX_MS: ${ANALYTICS_DB_ASYNC_INSERT_BUSY_TIMEOUT_MAX_MS:-} + # Flip TRACE_COLUMNS_NON_NULLABLE to true in lockstep with the EXCHANGE (the successor's end_time/ttft are + # non-nullable sentinel columns; a null bind would be rejected). The SPAN sibling is for the later spans cutover. + ANALYTICS_DB_DATA_MODEL_TRACE_COLUMNS_NON_NULLABLE: ${ANALYTICS_DB_DATA_MODEL_TRACE_COLUMNS_NON_NULLABLE:-false} + ANALYTICS_DB_DATA_MODEL_SPAN_COLUMNS_NON_NULLABLE: ${ANALYTICS_DB_DATA_MODEL_SPAN_COLUMNS_NON_NULLABLE:-false} JAVA_OPTS: "-Dliquibase.propertySubstitutionEnabled=true -XX:+UseG1GC -XX:MaxRAMPercentage=80.0" REDIS_URL: redis://:opik@redis:6379/ OPIK_OTEL_SDK_ENABLED: ${OPIK_OTEL_SDK_ENABLED:-false} diff --git a/tests_load/tests/traces-local-v2-cutover/README.md b/tests_load/tests/traces-local-v2-cutover/README.md new file mode 100644 index 00000000000..349fb329900 --- /dev/null +++ b/tests_load/tests/traces-local-v2-cutover/README.md @@ -0,0 +1,109 @@ +# Traces cutover — local simulation tooling + +Ad-hoc CLI scripts to stand up a representative dataset and live traffic on a **local** Opik, so the traces +buffered-cutover runbook (`apps/opik-backend/data-migrations/traces-local-v2-cutover`) can be rehearsed end to end and +iterated on quickly. + +| Script | What it does | Talks to | +|---|---|---| +| `seed_history.py` | Inserts N traces per week across several weeks, with back-dated `created_at` and matching-week UUIDv7 ids; optional far-future (`--bad-ids`) rows | ClickHouse (HTTP) | +| `live_traffic.py` | Emits new traces at a target TPS (with a share of updates) — "writes during the cutover window" | SDK / ingestion API | +| `delete_traffic.py` | Deletes existing traces at a target TPS — "deletes during the cutover window", the deletion-bridge exercise | SDK / REST API | + +**Why the seeder writes to ClickHouse directly:** the ingestion API stamps `created_at` server-side and treats it as +read-only, so it cannot produce back-dated rows — but the backfill slices the source by `created_at`. Direct inserts are +the only way to get the multi-week history the weekly backfill loop needs. The two traffic scripts use the normal APIs. + +## Setup + +```bash +# 1. Start Opik locally with host port mapping (exposes ClickHouse on localhost:8123 / :9000) AND deletion capture on, +# so deletes during the cutover window are recorded in the bridge (the whole point of the exercise): +ANALYTICS_DB_DATA_MODEL_TRACE_DELETION_EVENTS_CAPTURE_ENABLED=true ./opik.sh --port-mapping + +# 2. Install the SDK and these scripts' deps. +pip install -e sdks/python +pip install -r tests_load/tests/traces-local-v2-cutover/requirements.txt + +# 3. Point the SDK at the local install. +export OPIK_URL_OVERRIDE=http://localhost:5173/api/ +export OPIK_WORKSPACE=default + +# 4. Make clickhouse-client (used by the runbook driver scripts) resolve to the container's client, forwarding the +# CLICKHOUSE_* connection env the scripts set. The compose ClickHouse is user/password/db all "opik". +alias clickhouse-client='docker exec -i -e CLICKHOUSE_HOST -e CLICKHOUSE_USER -e CLICKHOUSE_PASSWORD opik-opik-clickhouse-1 clickhouse-client' +``` + +ClickHouse connection defaults (user/password/db all `opik`, host `localhost:8123`) match `--port-mapping`; override via +`OPIK_CH_HOST` / `OPIK_CH_PORT` / `OPIK_CH_USER` / `OPIK_CH_PASSWORD` / `OPIK_CH_DATABASE` if yours differ. + +## End-to-end rehearsal + +Every migration step runs through a driver script in the runbook's `scripts/` — no SQL is run by hand. + +```bash +# 1. Seed a few weeks of history (tune volumes for quick iteration). --bad-ids adds far-future-id rows. +python tests_load/tests/traces-local-v2-cutover/seed_history.py --weeks 6 --per-week 800 --bad-ids 40 + +RUNBOOK=apps/opik-backend/data-migrations/traces-local-v2-cutover +export CLICKHOUSE_HOST=localhost CLICKHOUSE_USER=opik CLICKHOUSE_PASSWORD=opik + +# 2. (Optional) Estimate the backfill ETA for a given config. +$RUNBOOK/scripts/estimate.sh --database opik --max-rows-per-insert 400 --pause-seconds 1 + +# 3. Generate concurrent write + delete traffic for the duration of the cutover (two more terminals). The overlap +# naturally produces delete-then-resurrect ids (a delete followed by an update to the same id) — the replay's +# resurrection guard keeps those live on the destination. +# NOTE: --bad-ids rows are ordinary deletable traces, so on a small dataset the delete traffic may remove them all +# before the cutover (they are then bridged + replayed like any delete — 0 leak — a valid path, but the far-future +# partition won't appear on the successor). To exercise the far-future-partition path specifically, seed with +# --bad-ids and run the backfill WITHOUT the delete traffic. +python tests_load/tests/traces-local-v2-cutover/live_traffic.py --tps 5 --duration 150 --update-ratio 0.4 +python tests_load/tests/traces-local-v2-cutover/delete_traffic.py --tps 3 --duration 150 # deletes existing (already-backfilled) traces + +# 4. Backfill (small --max-rows-per-insert exercises the adaptive sub-window splitting on modest data). Record the +# backfill_start it prints. +$RUNBOOK/scripts/backfill.sh --database opik --max-rows-per-insert 400 --pause-seconds 1 + +# 5. Delta + deletion replay, anchored at that backfill_start. +$RUNBOOK/scripts/delta_replay.sh --database opik --backfill-start '' + +# 6. QA the copy BEFORE the swap: normalized fidelity compare of source vs destination. +$RUNBOOK/scripts/verify.sh --database opik # add --drill-down to list differing keys on a mismatch +# Locally there is no async-insert buffer, so in-flight writes may still be settling: once traffic has stopped, +# re-run delta_replay.sh then verify.sh until it reports "PASSED: all N windows match" (convergence). In production +# the buffer holds writes during the cutover window instead. + +# 7. EXCHANGE (the data cutover; leaves traces a MergeTree so the backend's deletes keep working). It also renames the +# displaced old data to traces_pre_cutover_backup. --skip-wrap defers the sharding-ready Distributed wrap, which +# requires the delete DAO to target traces_local first. +$RUNBOOK/scripts/exchange_and_wrap.sh --database opik --skip-wrap +$RUNBOOK/scripts/verify.sh --database opik --old-table traces_pre_cutover_backup --new-table traces # post-swap fidelity +``` + +**Resetting between iterations depends on how far the last run got.** If you have **not** completed the `EXCHANGE` +(iterating on backfill/delta/verify), truncate **all three** tables and re-seed: `TRUNCATE TABLE traces`, +`TRUNCATE TABLE traces_local_v2`, `TRUNCATE TABLE deletion_events_local`. Also delete the persisted anchor +(`rm -f traces_cutover_backfill_start`) so the next `backfill.sh` captures a fresh `backfill_start` instead of reusing +the prior run's. Truncate the bridge and re-seed the source too, not just the shadow — a prior run leaves stale delete +events (and rows deleted-then-recreated in the previous window) behind, and a new run whose `backfill_start` is *after* +those events will neither copy nor replay them, so `verify.sh` reports a spurious mismatch. A real cutover has no such +residue: `backfill_start` is captured once, before any migration-window activity, so every relevant delete is covered +by the replay. + +**If you have already completed the `EXCHANGE`, truncate + re-seed is not enough** — the swap made `traces` the +non-nullable successor, and `seed_history.py` writes `NULL` `end_time`/`ttft` for some rows, which the successor rejects. +Restore the original schema first: **start from a fresh `opik.sh` volume** (re-runs the migrations), which is the clean +reset after any completed cutover. + +**Comparing the two tables:** compare **logical** rows (`SELECT uniqExact(workspace_id, project_id, id)`, or `count() +… FINAL`), not raw `count()`. A freshly-backfilled `traces_local_v2` holds un-merged `ReplacingMergeTree` versions +(a backfilled row plus its delta re-copy), so its raw count runs ahead of the long-merged `traces` even when the logical +content is identical — that is exactly what `verify.sh` compares (deduped, mask-honored, per week). The parked backup +also legitimately diverges from the live table: it is a frozen copy (`traces_pre_cutover_backup` after a successful +cutover, `traces_local_v2` after a rollback) while the live `traces` keeps changing. + +## Committing + +These are CLI tools (not pytest suites), so they add no CI cost and sit alongside the other `tests_load/tests` scripts. +Drop the directory before the PR if you'd rather keep it local. diff --git a/tests_load/tests/traces-local-v2-cutover/_common.py b/tests_load/tests/traces-local-v2-cutover/_common.py new file mode 100644 index 00000000000..eec373c6c72 --- /dev/null +++ b/tests_load/tests/traces-local-v2-cutover/_common.py @@ -0,0 +1,76 @@ +"""Shared helpers for the local traces-cutover simulation scripts. + +These scripts stand up a representative dataset and live traffic on a *local* Opik so the traces buffered-cutover runbook +(apps/opik-backend/data-migrations/traces-local-v2-cutover) can be rehearsed end to end. They are ad-hoc CLI tools, not +pytest suites. + +ClickHouse is reached directly (over HTTP) because the seeder must backdate `created_at`, which the ingestion API treats +as read-only — the SDK cannot produce multi-week history. Start Opik with `./opik.sh --port-mapping` so ClickHouse is on +localhost:8123. The SDK-based traffic scripts use the normal APIs and need only `OPIK_URL_OVERRIDE`. +""" + +import logging +import os +import time +from datetime import datetime, timezone + +import clickhouse_connect +import opik +from opik import id_helpers + +logging.basicConfig(level=logging.INFO, format="%(levelname)s [%(asctime)s]: %(message)s") +LOGGER = logging.getLogger("cutover") + +DEFAULT_PROJECT = "cutover-load-test" + + +def make_ch_client(): + """ClickHouse client for the local docker-compose analytics DB (defaults match `opik.sh --port-mapping`).""" + return clickhouse_connect.get_client( + host=os.environ.get("OPIK_CH_HOST", "localhost"), + port=int(os.environ.get("OPIK_CH_PORT", "8123")), + username=os.environ.get("OPIK_CH_USER", "opik"), + password=os.environ.get("OPIK_CH_PASSWORD", "opik"), + database=os.environ.get("OPIK_CH_DATABASE", "opik"), + ) + + +def make_opik_client() -> opik.Opik: + """SDK client. Reads OPIK_URL_OVERRIDE etc. from the environment, as the SDK normally does.""" + return opik.Opik() + + +def mint_uuid7(at: datetime) -> str: + """A UUIDv7 whose embedded timestamp is `at` — the backend derives `id_at` (the destination partition) from it.""" + return id_helpers.generate_id(at) + + +def utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def discover_workspace_and_project( + opik_client: opik.Opik, ch, project_name: str, timeout_s: int = 60 +) -> tuple[str, str]: + """Return the ClickHouse `(workspace_id, project_id)` for `project_name`. + + Logs one anchor trace through the SDK (which creates the project if needed), then reads that row back from + ClickHouse — so the seeder writes history into the same project the SDK traffic scripts target, without hardcoding + the workspace id. + """ + anchor_id = mint_uuid7(utcnow()) + opik_client.trace(id=anchor_id, name="cutover-anchor", project_name=project_name, input={"anchor": True}).end() + opik_client.flush() + + deadline = time.time() + timeout_s + while time.time() < deadline: + rows = ch.query( + "SELECT workspace_id, toString(project_id) FROM traces WHERE id = {id:String} LIMIT 1", + parameters={"id": anchor_id}, + ).result_rows + if rows: + workspace_id, project_id = rows[0] + LOGGER.info("Resolved project '%s': workspace_id=%s project_id=%s", project_name, workspace_id, project_id) + return workspace_id, project_id + time.sleep(0.5) + raise TimeoutError(f"anchor trace for project '{project_name}' did not appear in ClickHouse within {timeout_s}s") diff --git a/tests_load/tests/traces-local-v2-cutover/delete_traffic.py b/tests_load/tests/traces-local-v2-cutover/delete_traffic.py new file mode 100644 index 00000000000..532e178dfd1 --- /dev/null +++ b/tests_load/tests/traces-local-v2-cutover/delete_traffic.py @@ -0,0 +1,78 @@ +"""Delete existing traces at a steady rate through the normal SDK — the "deletes during the cutover window" reproducer. + +This is what makes the cutover interesting: with deletion capture enabled +(ANALYTICS_DB_DATA_MODEL_TRACE_DELETION_EVENTS_CAPTURE_ENABLED=true on the backend), each delete is recorded in the +deletion-events bridge and must be replayed onto the destination — otherwise it leaks across the swap. + +It pulls a pool of existing trace ids via search and deletes them at the target rate, refilling as it drains. Run it +during or after the backfill so the traces it deletes have already been copied — that is the leak the bridge prevents. + +Prerequisites: `OPIK_URL_OVERRIDE` pointing at the local install. Run `python delete_traffic.py --help` for options. +""" + +import signal +import time + +import click + +from _common import LOGGER, DEFAULT_PROJECT, make_opik_client + +_stop = False + + +def _handle_sigint(_signum, _frame): + global _stop + _stop = True + LOGGER.info("stopping after the current batch...") + + +def _fetch_ids(client, project, want, seen): + try: + traces = client.search_traces(project_name=project, max_results=want, truncate=True) + except Exception as exc: # search is best-effort; keep the loop alive + LOGGER.warning("search_traces failed: %s", exc) + return [] + return [t.id for t in traces if t.id not in seen] + + +@click.command() +@click.option("--project", default=DEFAULT_PROJECT, help="Project name to delete from.") +@click.option("--tps", default=2.0, help="Target deletes per second.") +@click.option("--duration", default=120, help="How long to run, in seconds (0 = until Ctrl-C).") +@click.option("--batch", default=1, help="Trace ids per delete call.") +def main(project, tps, duration, batch): + signal.signal(signal.SIGINT, _handle_sigint) + client = make_opik_client() + interval = batch / tps if tps > 0 else 0.0 + + seen: set[str] = set() + pool: list[str] = [] + deleted = 0 + started = time.time() + LOGGER.info("delete traffic: project='%s' tps=%.2f batch=%d duration=%ss (Ctrl-C to stop)", + project, tps, batch, duration or "∞") + + while not _stop and (duration == 0 or time.time() - started < duration): + tick = time.time() + if len(pool) < batch: + pool.extend(_fetch_ids(client, project, want=500, seen=seen)) + if not pool: + LOGGER.info("no more traces to delete; stopping") + break + ids = [pool.pop(0) for _ in range(min(batch, len(pool)))] + seen.update(ids) + client.rest_client.traces.delete_traces(ids=ids) + deleted += len(ids) + if deleted % 50 == 0: + LOGGER.info("deleted %d traces", deleted) + sleep = interval - (time.time() - tick) + if sleep > 0: + time.sleep(sleep) + + elapsed = time.time() - started + LOGGER.info("done: deleted=%d in %.1fs (%.2f deletes/s effective)", + deleted, elapsed, deleted / elapsed if elapsed else 0) + + +if __name__ == "__main__": + main() diff --git a/tests_load/tests/traces-local-v2-cutover/live_traffic.py b/tests_load/tests/traces-local-v2-cutover/live_traffic.py new file mode 100644 index 00000000000..ab40e296591 --- /dev/null +++ b/tests_load/tests/traces-local-v2-cutover/live_traffic.py @@ -0,0 +1,82 @@ +"""Emit new traces at a steady rate through the normal SDK — the "live writes during the cutover window" reproducer. + +Run it alongside the cutover so the delta-insert has fresh rows to catch. These use the ingestion API, so their +`created_at` is the current week; a share of them are logged as updates to an already-seen trace (a second `end()` with +new content) to exercise the version-bump path the delta relies on. + +Prerequisites: `OPIK_URL_OVERRIDE` pointing at the local install. Run `python live_traffic.py --help` for options. +""" + +import random +import signal +import string +import time + +import click + +from _common import LOGGER, DEFAULT_PROJECT, make_opik_client, utcnow + +_stop = False + + +def _handle_sigint(_signum, _frame): + global _stop + _stop = True + LOGGER.info("stopping after the current trace...") + + +def _text(n: int) -> str: + return "".join(random.choices(string.ascii_letters + " ", k=n)) + + +@click.command() +@click.option("--project", default=DEFAULT_PROJECT, help="Project name to write into.") +@click.option("--tps", default=5.0, help="Target traces per second.") +@click.option("--duration", default=120, help="How long to run, in seconds (0 = until Ctrl-C).") +@click.option("--update-ratio", default=0.2, help="Fraction of ticks that update a prior trace instead of creating one.") +def main(project, tps, duration, update_ratio): + signal.signal(signal.SIGINT, _handle_sigint) + client = make_opik_client() + interval = 1.0 / tps if tps > 0 else 0.0 + + created = 0 + updated = 0 + recent_ids: list[str] = [] + started = time.time() + LOGGER.info("live traffic: project='%s' tps=%.2f duration=%ss (Ctrl-C to stop)", project, tps, duration or "∞") + + while not _stop and (duration == 0 or time.time() - started < duration): + tick = time.time() + if recent_ids and random.random() < update_ratio: + # Update an existing trace: a new version with a fresh server-side last_updated_at. + trace_id = random.choice(recent_ids) + client.trace(id=trace_id, project_name=project, output={"update": _text(120)}) + updated += 1 + else: + trace = client.trace( + name="live-trace", + project_name=project, + start_time=utcnow(), + input={"prompt": _text(160)}, + output={"completion": _text(160)}, + ) + trace.end() + recent_ids.append(trace.id) + if len(recent_ids) > 500: + recent_ids.pop(0) + created += 1 + + if (created + updated) % 50 == 0: + client.flush() + sleep = interval - (time.time() - tick) + if sleep > 0: + time.sleep(sleep) + + client.flush() + elapsed = time.time() - started + LOGGER.info("done: created=%d updated=%d in %.1fs (%.2f traces/s effective)", + created, updated, elapsed, (created + updated) / elapsed if elapsed else 0) + + +if __name__ == "__main__": + main() diff --git a/tests_load/tests/traces-local-v2-cutover/requirements.txt b/tests_load/tests/traces-local-v2-cutover/requirements.txt new file mode 100644 index 00000000000..982b6fe2b36 --- /dev/null +++ b/tests_load/tests/traces-local-v2-cutover/requirements.txt @@ -0,0 +1,3 @@ +opik +click +clickhouse-connect diff --git a/tests_load/tests/traces-local-v2-cutover/seed_history.py b/tests_load/tests/traces-local-v2-cutover/seed_history.py new file mode 100644 index 00000000000..bc6b9f28b34 --- /dev/null +++ b/tests_load/tests/traces-local-v2-cutover/seed_history.py @@ -0,0 +1,174 @@ +"""Seed historical `traces` rows spread across several weeks, so the cutover backfill has multi-week data to iterate. + +Writes straight to ClickHouse: the ingestion API stamps `created_at` server-side (read-only), so it cannot produce +back-dated rows, and the backfill slices the source by `created_at`. Each row gets a `created_at` in its week and a +UUIDv7 `id` minted at the same instant, so `id_at` (the destination weekly partition) matches `created_at` — the shape +real accumulated history has. `--bad-ids` optionally adds rows whose `id` is minted in the far future (year ~2201, the +litellm-bug shape) while `created_at` stays real, to exercise the far-future-partition path. (The successor's `id_at` is +a 32-bit `DateTime` capped at 2106, so a 2201 id wraps to ~2065 — that is where those rows actually partition.) + +Every migrated column is populated with VARIED, realistic values so the fidelity compare (verify.sh) actually exercises +each column — an empty column would match on both sides even if the copy dropped it. Timestamps are written at true +NANOSECOND precision (the source columns are DateTime64(9), like real `now64(9)` rows), so the ns->us truncation the +successor performs is exercised, not skipped. A share of rows leave `end_time` / `ttft` NULL to exercise the +NULL->sentinel (epoch / NaN) normalization. + +Prerequisites: `./opik.sh --port-mapping` (ClickHouse on localhost:8123) and `OPIK_URL_OVERRIDE` pointing at the same +install. Run `python seed_history.py --help` for options. +""" + +import json +import random +import string +from datetime import datetime, timedelta, timezone + +import click + +from _common import LOGGER, DEFAULT_PROJECT, discover_workspace_and_project, make_ch_client, make_opik_client, mint_uuid7, utcnow + +# Every base column the backfill copies (MATERIALIZED columns like id_at are recomputed by CH and excluded). Order +# matches the tuple built in _row(). +COLUMNS = [ + "id", + "workspace_id", + "project_id", + "name", + "start_time", + "end_time", + "input", + "output", + "metadata", + "tags", + "created_at", + "last_updated_at", + "created_by", + "last_updated_by", + "error_info", + "thread_id", + "visibility_mode", + "truncation_threshold", + "input_slim", + "output_slim", + "ttft", + "source", + "environment", +] + +# A far-future instant matching the litellm UUIDv7 bug (ids whose embedded timestamp lands around the year 2201). +BAD_ID_INSTANT = utcnow().replace(year=2201) + +_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) +_TAG_POOL = ["prod", "llm", "rag", "eval", "v1", "v2", "canary", "batch", "stream", "agent"] +_SOURCES = ["sdk", "experiment", "playground", "optimization", "evaluator"] +_ENVIRONMENTS = ["production", "staging", "dev", ""] +_USERS = ["alice", "bob", "carol", "service-account", "ci-runner"] + + +def _text(lo: int, hi: int) -> str: + return "".join(random.choices(string.ascii_letters + string.digits + " ", k=random.randint(lo, hi))) + + +def _payload(kind: str) -> str: + return json.dumps({kind: _text(80, 240)}) + + +def _ns(dt: datetime) -> int: + """DateTime64(9) tick value (ns since epoch) with a random sub-microsecond remainder, so ns->us truncation runs.""" + whole_us = int((dt - _EPOCH).total_seconds() * 1_000_000) # microseconds (integer, no float-precision loss at 2^53) + return whole_us * 1_000 + random.randint(1, 999) + + +def _us(dt: datetime) -> int: + """DateTime64(6) tick value (us since epoch).""" + return int((dt - _EPOCH).total_seconds() * 1_000_000) + + +def _row(created_at_dt: datetime, id_instant: datetime, workspace_id: str, project_id: str) -> tuple: + trace_id = mint_uuid7(id_instant) + created_ns = _ns(created_at_dt) + # 30% leave end_time NULL (the "not ended" case -> epoch sentinel on the successor); else a real duration. + end_ns = None if random.random() < 0.3 else created_ns + random.randint(5_000_000, 3_000_000_000) + # 40% leave ttft NULL (-> NaN sentinel); else a plausible time-to-first-token in seconds. + ttft = None if random.random() < 0.4 else round(random.uniform(0.005, 5.0), 6) + payload_in, payload_out = _payload("prompt"), _payload("completion") + return ( + trace_id, + workspace_id, + project_id, + "seed-trace", + created_ns, # start_time ~ created_at + end_ns, + payload_in, + payload_out, + json.dumps({"model": random.choice(["gpt-4", "claude", "llama"]), + "temperature": round(random.random(), 3), "max_tokens": random.randint(16, 4000)}), + random.sample(_TAG_POOL, random.randint(0, 4)), + created_ns, # created_at — the backfill slice column, at ns precision + _us(created_at_dt), # last_updated_at (us) ~= created_at, so the delta never re-copies these historical rows + random.choice(_USERS), + random.choice(_USERS), + "" if random.random() < 0.85 else json.dumps( + {"exception_type": "ValueError", "message": _text(10, 60), "traceback": _text(20, 80)}), + "" if random.random() < 0.5 else "".join(random.choices("0123456789abcdef", k=16)), + "hidden" if random.random() < 0.1 else "default", + random.choice([10001, 20001]), + payload_in[:200], + payload_out[:200], + ttft, + random.choice(_SOURCES), + random.choice(_ENVIRONMENTS), + ) + + +@click.command() +@click.option("--project", default=DEFAULT_PROJECT, help="Project name to seed into.") +@click.option("--weeks", default=8, help="Number of consecutive weeks of history, ending at the current week.") +@click.option("--per-week", default=2000, help="Traces per week.") +@click.option("--bad-ids", default=0, help="Extra rows with a far-future (year ~2201) UUIDv7 id but a real created_at.") +@click.option("--batch", default=5000, help="Rows per ClickHouse INSERT.") +@click.option("--workspace-id", default=None, help="Override workspace_id (default: auto-discovered via the SDK).") +@click.option("--project-id", default=None, help="Override project_id (default: auto-discovered via the SDK).") +def main(project, weeks, per_week, bad_ids, batch, workspace_id, project_id): + ch = make_ch_client() + + if workspace_id is None or project_id is None: + workspace_id, project_id = discover_workspace_and_project(make_opik_client(), ch, project) + + now = utcnow() + # Rows land uniformly within each week, oldest week first; the newest week ends at "now". + rows: list[tuple] = [] + per_week_counts: dict[str, int] = {} + for week in range(weeks): + week_end = now - timedelta(weeks=week) + week_start = week_end - timedelta(weeks=1) + label = week_start.date().isoformat() + for _ in range(per_week): + span = (week_end - week_start).total_seconds() + created_at = week_start + timedelta(seconds=random.uniform(0, span)) + rows.append(_row(created_at, created_at, workspace_id, project_id)) + per_week_counts[label] = per_week + + for _ in range(bad_ids): + created_at = now - timedelta(weeks=random.uniform(0, max(weeks - 1, 1))) + rows.append(_row(created_at, BAD_ID_INSTANT, workspace_id, project_id)) + + random.shuffle(rows) # interleave weeks so inserts look like real ingestion, not one week at a time + LOGGER.info("Inserting %d traces (%d weeks x %d + %d bad-id) into project_id=%s", len(rows), weeks, per_week, + bad_ids, project_id) + for start in range(0, len(rows), batch): + chunk = rows[start:start + batch] + ch.insert("traces", chunk, column_names=COLUMNS) + LOGGER.info(" inserted %d/%d", min(start + batch, len(rows)), len(rows)) + + total = ch.query( + "SELECT count() FROM traces WHERE project_id = {p:String}", parameters={"p": project_id} + ).result_rows[0][0] + LOGGER.info("Done. project '%s' now has %s live traces in ClickHouse.", project, total) + LOGGER.info("Per-week seeded (created_at week -> count): %s", + {k: per_week_counts[k] for k in sorted(per_week_counts)}) + if bad_ids: + LOGGER.info("Plus %d far-future-id rows (id_at ~2201) to exercise the bad-id partition path.", bad_ids) + + +if __name__ == "__main__": + main() From 4ad452927df9d6bdd67176ff75c7fbda4d963679 Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Wed, 22 Jul 2026 16:53:35 +0200 Subject: [PATCH 02/28] =?UTF-8?q?test(cutover):=20address=20Baz=20review?= =?UTF-8?q?=20=E2=80=94=20safe=20wrap=20default,=20input=20validation,=20s?= =?UTF-8?q?im=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the Distributed wrap opt-in in exchange_and_wrap.sh (default EXCHANGE-only, --with-wrap to combine; --skip-wrap kept as an explicit alias) so a bare run can't apply the wrap and break trace deletes before the DAO retarget. Validate --database as a ClickHouse identifier across all drivers and the timestamp args (--backfill-start/--cutover-start) and verify's table args, so operator inputs interpolated into the reference SQL fail cleanly instead of altering the query. Load-sim fixes: end() the live-traffic update so it flushes as completed traffic; fill only the missing one of --workspace-id/--project-id; exclude in-flight pool ids from delete-traffic refills; correct the seeding-order comment. Runbook + README updated for the new wrap default and the replay-wall-time wording. --- .../traces-local-v2-cutover/README.md | 15 +++---- .../scripts/backfill.sh | 5 +++ .../scripts/delta_replay.sh | 3 ++ .../scripts/estimate.sh | 2 + .../scripts/exchange_and_wrap.sh | 41 ++++++++++++------- .../scripts/finalize.sh | 2 + .../scripts/rollback.sh | 3 ++ .../traces-local-v2-cutover/scripts/verify.sh | 4 ++ .../traces-local-v2-cutover/delete_traffic.py | 7 ++-- .../traces-local-v2-cutover/live_traffic.py | 5 ++- .../traces-local-v2-cutover/seed_history.py | 8 +++- 11 files changed, 67 insertions(+), 28 deletions(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md index fd6e9104e09..7ebeb1fdda6 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md @@ -122,7 +122,7 @@ new table before the EXCHANGE. The replay matches the **full key**, not `id` alo (reference SQL [`000002_delta_and_deletion_replay.sql`](scripts/db-app-analytics/000002_delta_and_deletion_replay.sql)) — delta-insert (anchored at `backfill_start`), then **deletion replay**. The replay runs with `lightweight_deletes_sync = 2`, so it returns only once the delete mutation has applied on **every** replica. - clickhouse-client prints each statement's wall time; record the replay's. + clickhouse-client prints each statement's wall time; record the replay's wall time. ```bash CLICKHOUSE_HOST= CLICKHOUSE_PASSWORD= ./scripts/delta_replay.sh --database opik --backfill-start '' ``` @@ -134,10 +134,11 @@ new table before the EXCHANGE. The replay matches the **full key**, not `id` alo on `traces_local_v2`, across all replicas via `clusterAllReplicas`) so no replica swaps in an incomplete table (`--force` overrides); then records and prints `cutover_start`, runs `EXCHANGE TABLES ... ON CLUSTER` and renames the displaced old data to - `traces_pre_cutover_backup` (see "Naming and the parked backup"), then (unless `--skip-wrap`) the `RENAME` + - `Distributed` wrap. Restore the buffer ceiling and verify. + `traces_pre_cutover_backup` (see "Naming and the parked backup"). It **stops there by default** (EXCHANGE only, + leaving `traces` a `MergeTree` where deletes still work); the `RENAME` + `Distributed` wrap runs only with + `--with-wrap`. Restore the buffer ceiling and verify. ```bash - CLICKHOUSE_HOST= CLICKHOUSE_PASSWORD= ./scripts/exchange_and_wrap.sh --database opik --skip-wrap + CLICKHOUSE_HOST= CLICKHOUSE_PASSWORD= ./scripts/exchange_and_wrap.sh --database opik ``` > **HARD PREREQUISITE for the wrap (step 4, part 2): the delete/mutation DAO must target `traces_local` first.** A @@ -148,9 +149,9 @@ new table before the EXCHANGE. The replay matches the **full key**, not `id` alo > So the moment the wrap is applied, **both** the product's delete-by-id (`TraceDAO.DELETE_BY_ID`) **and** the retention > sweep (`DELETE_FOR_RETENTION`) start returning 500 against `traces`. This is prep work that must ship **before** the > wrap: point those DAO paths at `traces_local` (reads and inserts can stay on the Distributed `traces`). The `EXCHANGE` -> alone is the data cutover and leaves `traces` a `MergeTree` where deletes still work — so use `exchange_and_wrap.sh -> --skip-wrap` and defer the wrap until the sharding-aware DAO ships. The wrap is the sharding-readiness layer, not the -> cutover. +> alone is the data cutover and leaves `traces` a `MergeTree` where deletes still work — which is why the wrap is +> **opt-in** (`--with-wrap`) and the default stops after the EXCHANGE. Defer the wrap until the sharding-aware DAO +> ships. The wrap is the sharding-readiness layer, not the cutover. > > **Applying the deferred wrap later:** once the sharding-aware DAO has shipped, run > `exchange_and_wrap.sh --database opik --wrap-only` — it runs the settle gate and applies **only** the `RENAME` + diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh index 6d5cda94770..f5271557fbe 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh @@ -89,6 +89,11 @@ while [[ $# -gt 0 ]]; do done [[ -n "$DATABASE" ]] || { echo "ERROR: --database is required" >&2; exit 2; } +# --database is interpolated into the reference SQL; require a plain ClickHouse identifier so it cannot alter the query. +[[ "$DATABASE" =~ ^[A-Za-z0-9_]+$ ]] || { echo "ERROR: --database must be a ClickHouse identifier (letters, digits, underscore)." >&2; exit 2; } +# --state-file is an operator-owned path read with cat and written with > (both quoted); reject a blank or multi-line +# value so the single-line anchor round-trips cleanly. +[[ -n "$STATE_FILE" && "$STATE_FILE" != *$'\n'* ]] || { echo "ERROR: --state-file must be a non-empty single-line path." >&2; exit 2; } [[ -f "$BACKFILL_SQL" ]] || { echo "ERROR: cannot find backfill SQL at $BACKFILL_SQL" >&2; exit 2; } # Every query runs against the analytics database; --query keeps output scriptable (TSV, no formatting). diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/delta_replay.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/delta_replay.sh index cf4421a9a46..33d7d371bf0 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/delta_replay.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/delta_replay.sh @@ -31,7 +31,10 @@ while [[ $# -gt 0 ]]; do done [[ -n "$DATABASE" ]] || { echo "ERROR: --database is required" >&2; exit 2; } +# --database and --backfill-start are interpolated into the reference SQL; validate their shapes so neither can alter it. +[[ "$DATABASE" =~ ^[A-Za-z0-9_]+$ ]] || { echo "ERROR: --database must be a ClickHouse identifier (letters, digits, underscore)." >&2; exit 2; } [[ -n "$BACKFILL_START" ]] || { echo "ERROR: --backfill-start is required (printed by backfill.sh)" >&2; exit 2; } +[[ "$BACKFILL_START" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}\ [0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?$ ]] || { echo "ERROR: --backfill-start must be 'YYYY-MM-DD HH:MM:SS[.ffffff]'." >&2; exit 2; } [[ -f "$SQL_FILE" ]] || { echo "ERROR: cannot find $SQL_FILE" >&2; exit 2; } echo "Reminder: raise databaseAnalytics.asyncInsertBusyTimeoutMaxMs before this step (backend config, not SQL) and" diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/estimate.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/estimate.sh index 36fbaf92531..6e81d426a13 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/estimate.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/estimate.sh @@ -49,6 +49,8 @@ while [[ $# -gt 0 ]]; do done [[ -n "$DATABASE" ]] || { echo "ERROR: --database is required" >&2; exit 2; } +# --database is interpolated into the probe/size SQL; require a plain ClickHouse identifier so it cannot alter the query. +[[ "$DATABASE" =~ ^[A-Za-z0-9_]+$ ]] || { echo "ERROR: --database must be a ClickHouse identifier (letters, digits, underscore)." >&2; exit 2; } ch() { clickhouse-client --database "$DATABASE" --query "$1" diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh index 3576bc17bab..69bb58db4fa 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh @@ -2,9 +2,14 @@ # # Driver for step 3 of the buffered traces cutover: EXCHANGE + Distributed wrap (runbook: ../README.md). # -# Captures and prints cutover_start (needed by rollback.sh if you roll back after this), runs the `exchange` block of -# db-app-analytics/000003_exchange_and_wrap.sql, then the `wrap` block unless --skip-wrap. Run it right after the delta + -# replay + verify, while the async-insert buffer is still holding writes. +# Captures and prints cutover_start (needed by rollback.sh if you roll back after this), then runs the `exchange` block +# of db-app-analytics/000003_exchange_and_wrap.sql. By default it stops there (EXCHANGE only) — the Distributed `wrap` +# block runs only with --with-wrap. Run it right after the delta + replay + verify, while the async-insert buffer is +# still holding writes. +# +# The wrap is OPT-IN on purpose: a lightweight DELETE against a Distributed table is unsupported, so wrapping `traces` +# breaks the product's trace-delete / retention paths until those DAOs target `traces_local`. The safe default is to +# leave `traces` a MergeTree (deletes keep working) and apply the wrap later, once the DAOs are sharding-aware. # # Guarded like rollback.sh: it asserts the live `traces` topology matches the requested action before touching anything, # so a re-run cannot silently swap the tables back, and a partial EXCHANGE (swap done, post-swap RENAME not) is detected @@ -14,13 +19,13 @@ # # Options: # --database NAME analytics database (e.g. opik). Required. -# --skip-wrap stop after the EXCHANGE (the data cutover); do not create the Distributed wrapper. Use this until -# the delete/read DAOs are sharding-aware — a lightweight DELETE against a Distributed table is -# unsupported. -# --wrap-only run ONLY the Distributed wrap on the already-EXCHANGEd `traces`, skipping the EXCHANGE (and the -# cutover_start capture — the data cutover already happened). This is the deferred second half of a -# prior `--skip-wrap` run: apply it once the delete/read DAOs are sharding-aware. Mutually exclusive -# with --skip-wrap. The replication-settle gate still runs first. +# (default) run ONLY the EXCHANGE (the data cutover), then stop — leaves `traces` a MergeTree where deletes +# still work. The Distributed wrap is deferred (see above). +# --with-wrap also apply the Distributed wrap in the same run (EXCHANGE + wrap). Use only once the delete/read +# DAOs are sharding-aware. Mutually exclusive with --skip-wrap / --wrap-only. +# --skip-wrap explicit alias for the default (EXCHANGE only); accepted for clarity and back-compat. +# --wrap-only run ONLY the Distributed wrap on the already-EXCHANGEd `traces` (no EXCHANGE, no new cutover_start) +# — the deferred second half of a prior EXCHANGE-only run. Mutually exclusive with the above. # --force skip the replication-settle gate. By default the swap aborts while any replica still # has replication-queue backlog or an unfinished mutation on traces / traces_local_v2, since a # behind replica would swap in an incomplete table. Use only if settlement is confirmed out of band. @@ -32,6 +37,7 @@ SQL_FILE="$SCRIPT_DIR/db-app-analytics/000003_exchange_and_wrap.sql" DATABASE="" SKIP_WRAP=0 +WITH_WRAP=0 WRAP_ONLY=0 FORCE=0 @@ -39,6 +45,7 @@ while [[ $# -gt 0 ]]; do case "$1" in --database) DATABASE="$2"; shift 2 ;; --skip-wrap) SKIP_WRAP=1; shift ;; + --with-wrap) WITH_WRAP=1; shift ;; --wrap-only) WRAP_ONLY=1; shift ;; --force) FORCE=1; shift ;; *) echo "Unknown argument: $1" >&2; exit 2 ;; @@ -46,8 +53,13 @@ while [[ $# -gt 0 ]]; do done [[ -n "$DATABASE" ]] || { echo "ERROR: --database is required" >&2; exit 2; } +# --database is interpolated into the reference SQL; require a plain ClickHouse identifier so it cannot alter the query. +[[ "$DATABASE" =~ ^[A-Za-z0-9_]+$ ]] || { echo "ERROR: --database must be a ClickHouse identifier (letters, digits, underscore)." >&2; exit 2; } [[ -f "$SQL_FILE" ]] || { echo "ERROR: cannot find $SQL_FILE" >&2; exit 2; } -[[ "$SKIP_WRAP" == "1" && "$WRAP_ONLY" == "1" ]] && { echo "ERROR: --skip-wrap and --wrap-only are mutually exclusive" >&2; exit 2; } +# At most one wrap mode. Default (none set) is EXCHANGE only. +if (( SKIP_WRAP + WITH_WRAP + WRAP_ONLY > 1 )); then + echo "ERROR: --skip-wrap, --with-wrap and --wrap-only are mutually exclusive" >&2; exit 2 +fi ch() { clickhouse-client --database "$DATABASE" --query "$1" @@ -174,11 +186,12 @@ echo "RECORD cutover_start=$CUTOVER_START (pass to rollback.sh --cutover-start run_block exchange echo "EXCHANGE done: 'traces' is now the partitioned data; the old data is parked as 'traces_pre_cutover_backup'." -if [[ "$SKIP_WRAP" == "1" ]]; then - echo "Skipping the Distributed wrap (--skip-wrap). The data cutover is complete." -else +if [[ "$WITH_WRAP" == "1" ]]; then run_block wrap echo "Distributed wrap done: 'traces' fronts 'traces_local' via sipHash64(project_id)." +else + echo "Distributed wrap deferred (default). Deletes still work on the MergeTree 'traces'. Apply the wrap later with" + echo "--wrap-only once the delete/read DAOs target traces_local." fi echo "Restore databaseAnalytics.asyncInsertBusyTimeoutMaxMs to default, verify, and keep traces_pre_cutover_backup for the soak." diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/finalize.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/finalize.sh index b261dbe6a9f..5f50278d7af 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/finalize.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/finalize.sh @@ -35,6 +35,8 @@ while [[ $# -gt 0 ]]; do done [[ -n "$DATABASE" ]] || { echo "ERROR: --database is required" >&2; exit 2; } +# --database is interpolated into the drop/exists SQL; require a plain ClickHouse identifier so it cannot alter the query. +[[ "$DATABASE" =~ ^[A-Za-z0-9_]+$ ]] || { echo "ERROR: --database must be a ClickHouse identifier (letters, digits, underscore)." >&2; exit 2; } ch() { clickhouse-client --database "$DATABASE" --query "$1" diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh index 2daf5d5f47d..550e55331fa 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh @@ -37,6 +37,9 @@ while [[ $# -gt 0 ]]; do done [[ -n "$DATABASE" ]] || { echo "ERROR: --database is required" >&2; exit 2; } +# --database and --cutover-start are interpolated into the reference SQL; validate their shapes so neither can alter it. +[[ "$DATABASE" =~ ^[A-Za-z0-9_]+$ ]] || { echo "ERROR: --database must be a ClickHouse identifier (letters, digits, underscore)." >&2; exit 2; } +[[ -z "$CUTOVER_START" || "$CUTOVER_START" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}\ [0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?$ ]] || { echo "ERROR: --cutover-start must be 'YYYY-MM-DD HH:MM:SS[.ffffff]'." >&2; exit 2; } case "$STAGE" in A|B|C) ;; *) echo "ERROR: --stage must be A, B or C" >&2; exit 2 ;; diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh index 90681d5af3b..9e34dae7b50 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh @@ -57,6 +57,10 @@ while [[ $# -gt 0 ]]; do done [[ -n "$DATABASE" ]] || { echo "ERROR: --database is required" >&2; exit 2; } +# --database / --old-table / --new-table are interpolated into the reference SQL; require plain ClickHouse identifiers. +for _ident in "$DATABASE" "$OLD_TABLE" "$NEW_TABLE"; do + [[ "$_ident" =~ ^[A-Za-z0-9_]+$ ]] || { echo "ERROR: --database/--old-table/--new-table must be ClickHouse identifiers (letters, digits, underscore): '$_ident'" >&2; exit 2; } +done [[ -f "$VERIFY_SQL" ]] || { echo "ERROR: cannot find verify SQL at $VERIFY_SQL" >&2; exit 2; } ch() { diff --git a/tests_load/tests/traces-local-v2-cutover/delete_traffic.py b/tests_load/tests/traces-local-v2-cutover/delete_traffic.py index 532e178dfd1..67ac1c79b99 100644 --- a/tests_load/tests/traces-local-v2-cutover/delete_traffic.py +++ b/tests_load/tests/traces-local-v2-cutover/delete_traffic.py @@ -26,13 +26,13 @@ def _handle_sigint(_signum, _frame): LOGGER.info("stopping after the current batch...") -def _fetch_ids(client, project, want, seen): +def _fetch_ids(client, project, want, exclude): try: traces = client.search_traces(project_name=project, max_results=want, truncate=True) except Exception as exc: # search is best-effort; keep the loop alive LOGGER.warning("search_traces failed: %s", exc) return [] - return [t.id for t in traces if t.id not in seen] + return [t.id for t in traces if t.id not in exclude] @click.command() @@ -55,7 +55,8 @@ def main(project, tps, duration, batch): while not _stop and (duration == 0 or time.time() - started < duration): tick = time.time() if len(pool) < batch: - pool.extend(_fetch_ids(client, project, want=500, seen=seen)) + # Exclude both already-deleted ids and those still queued in `pool`, so a refill can't requeue an in-flight id. + pool.extend(_fetch_ids(client, project, want=500, exclude=seen | set(pool))) if not pool: LOGGER.info("no more traces to delete; stopping") break diff --git a/tests_load/tests/traces-local-v2-cutover/live_traffic.py b/tests_load/tests/traces-local-v2-cutover/live_traffic.py index ab40e296591..b7d93ec2b31 100644 --- a/tests_load/tests/traces-local-v2-cutover/live_traffic.py +++ b/tests_load/tests/traces-local-v2-cutover/live_traffic.py @@ -48,9 +48,10 @@ def main(project, tps, duration, update_ratio): while not _stop and (duration == 0 or time.time() - started < duration): tick = time.time() if recent_ids and random.random() < update_ratio: - # Update an existing trace: a new version with a fresh server-side last_updated_at. + # Update an existing trace: a new version with a fresh server-side last_updated_at. end() finalizes the + # update so it flushes as completed traffic, not just a create-attempt. trace_id = random.choice(recent_ids) - client.trace(id=trace_id, project_name=project, output={"update": _text(120)}) + client.trace(id=trace_id, project_name=project, output={"update": _text(120)}).end() updated += 1 else: trace = client.trace( diff --git a/tests_load/tests/traces-local-v2-cutover/seed_history.py b/tests_load/tests/traces-local-v2-cutover/seed_history.py index bc6b9f28b34..c7e589827dd 100644 --- a/tests_load/tests/traces-local-v2-cutover/seed_history.py +++ b/tests_load/tests/traces-local-v2-cutover/seed_history.py @@ -132,10 +132,14 @@ def main(project, weeks, per_week, bad_ids, batch, workspace_id, project_id): ch = make_ch_client() if workspace_id is None or project_id is None: - workspace_id, project_id = discover_workspace_and_project(make_opik_client(), ch, project) + # Fill only the value(s) not supplied, so a single --workspace-id or --project-id override is honored. + discovered_workspace_id, discovered_project_id = discover_workspace_and_project(make_opik_client(), ch, project) + workspace_id = workspace_id or discovered_workspace_id + project_id = project_id or discovered_project_id now = utcnow() - # Rows land uniformly within each week, oldest week first; the newest week ends at "now". + # Each week's rows land uniformly within the week. week 0 is the current week (ending at "now"); the loop walks + # backward to older weeks. Generation order doesn't matter — rows are shuffled before insert (below). rows: list[tuple] = [] per_week_counts: dict[str, int] = {} for week in range(weeks): From b3cf335db4b50e4b3f78b1829a04090fe2d4ba80 Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Wed, 22 Jul 2026 17:12:00 +0200 Subject: [PATCH 03/28] test(cutover): validate the resumed backfill_start anchor content On resume, backfill.sh read the operator-owned --state-file and reused its content as backfill_start without checking it. A corrupted or wrong file would feed a garbage anchor forward to step 2. Validate the content is a well-formed timestamp and fail fast otherwise. This addresses the spirit of the --state-file finding by constraining the anchor's value rather than allow-listing its path (the path is operator-owned; allow-listing would break legitimate absolute paths and crosses no privilege boundary the operator lacks). --- .../traces-local-v2-cutover/scripts/backfill.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh index f5271557fbe..ec411862ec2 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh @@ -256,6 +256,9 @@ preflight_capacity if [[ "$DRY_RUN" != "1" ]]; then if [[ -s "$STATE_FILE" ]]; then BACKFILL_START="$(cat "$STATE_FILE")" + # Validate the resumed content: the state file is operator-owned, so a corrupted or wrong file would otherwise + # feed a garbage anchor forward to step 2. Fail fast unless it is a well-formed timestamp. + [[ "$BACKFILL_START" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}\ [0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?$ ]] || { echo "ERROR: $STATE_FILE does not contain a valid backfill_start timestamp ('YYYY-MM-DD HH:MM:SS[.ffffff]')." >&2; exit 1; } log "REUSING backfill_start=$BACKFILL_START from $STATE_FILE (resume: original anchor kept)" else BACKFILL_START="$(ch "SELECT toString(now64(6))")" From a8d910a2064dc5478f7e7d7a1980fcacf7fb99b2 Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Wed, 22 Jul 2026 17:41:02 +0200 Subject: [PATCH 04/28] =?UTF-8?q?test(cutover):=20address=20second=20Baz?= =?UTF-8?q?=20review=20=E2=80=94=20wrap-only=20split-state=20guard=20+=20f?= =?UTF-8?q?ixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - exchange_and_wrap.sh --wrap-only now refuses when traces_local_v2 still exists (EXCHANGE done but post-swap RENAME not): wrapping then would orphan the old data under the wrong name. Prints the rename to finish first. - seed_history: build BAD_ID_INSTANT from a fixed date so it can't raise ValueError at import on a Feb-29 run (2201 is not a leap year). - delete_traffic: a transient search_traces failure now backs off and retries instead of being mistaken for "no more traces" and ending the run. - README: document the mutually-exclusive wrap flags and EXCHANGE-only default; reword the awkward "EXCHANGEd" to "already-swapped". --- .../traces-local-v2-cutover/README.md | 6 +++++- .../scripts/exchange_and_wrap.sh | 13 +++++++++++-- .../tests/traces-local-v2-cutover/delete_traffic.py | 13 +++++++++---- .../tests/traces-local-v2-cutover/seed_history.py | 5 +++-- 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md index 7ebeb1fdda6..208a39ce4c8 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md @@ -155,8 +155,12 @@ new table before the EXCHANGE. The replay matches the **full key**, not `id` alo > > **Applying the deferred wrap later:** once the sharding-aware DAO has shipped, run > `exchange_and_wrap.sh --database opik --wrap-only` — it runs the settle gate and applies **only** the `RENAME` + -> `Distributed` wrap on the already-EXCHANGEd `traces` (no second EXCHANGE, no new `cutover_start`). To roll the wrap +> `Distributed` wrap on the already-swapped `traces` (no second EXCHANGE, no new `cutover_start`). To roll the wrap > back, use `rollback.sh --stage C`. +> +> **Wrap flags** (`exchange_and_wrap.sh`, mutually exclusive; default is EXCHANGE-only): omit them (or pass +> `--skip-wrap`, an explicit alias) to run the EXCHANGE and stop; `--with-wrap` to also apply the wrap in the same run; +> `--wrap-only` to apply just the deferred wrap later. **Dedup note.** After the delta, a row can have two physical versions on `traces_local_v2` (the backfilled one and the delta one). This is normal — `ReplacingMergeTree` collapses them on merge / under `FINAL` / `LIMIT 1 BY id`, highest diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh index 69bb58db4fa..4482dd0abad 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh @@ -24,7 +24,7 @@ # --with-wrap also apply the Distributed wrap in the same run (EXCHANGE + wrap). Use only once the delete/read # DAOs are sharding-aware. Mutually exclusive with --skip-wrap / --wrap-only. # --skip-wrap explicit alias for the default (EXCHANGE only); accepted for clarity and back-compat. -# --wrap-only run ONLY the Distributed wrap on the already-EXCHANGEd `traces` (no EXCHANGE, no new cutover_start) +# --wrap-only run ONLY the Distributed wrap on the already-swapped `traces` (no EXCHANGE, no new cutover_start) # — the deferred second half of a prior EXCHANGE-only run. Mutually exclusive with the above. # --force skip the replication-settle gate. By default the swap aborts while any replica still # has replication-queue backlog or an unfinished mutation on traces / traces_local_v2, since a @@ -103,7 +103,10 @@ assert_pre_exchange_topology() { [[ -n "$(traces_engine traces_local_v2)" ]] || { echo "ERROR: successor 'traces_local_v2' not found; run the backfill + delta first." >&2; exit 1; } } -# --wrap-only precondition: traces must be the post-EXCHANGE successor MergeTree (not the original, not already wrapped). +# --wrap-only precondition: traces must be the post-EXCHANGE successor MergeTree (not the original, not already wrapped), +# AND the post-swap RENAME must have completed. In the split state (EXCHANGE done, RENAME not) `traces` already holds +# the successor schema but `traces_local_v2` still holds the old data — wrapping then would orphan the old data under +# the wrong name (finalize.sh would misread it as the disposable successor). So refuse until the rename is finished. assert_pre_wrap_topology() { local engine end_time engine="$(traces_engine traces)" @@ -117,6 +120,12 @@ assert_pre_wrap_topology() { echo "ERROR: --wrap-only expects the post-EXCHANGE state (traces = successor schema), but traces has Nullable end_time (the EXCHANGE has not run). Run without --wrap-only first." >&2 exit 1 } + if [[ -n "$(traces_engine traces_local_v2)" ]]; then + echo "ERROR: --wrap-only: 'traces_local_v2' still exists — the post-EXCHANGE RENAME did not complete, so wrapping" >&2 + echo " now would orphan the old data under the wrong name. Finish the rename first, then re-run --wrap-only:" >&2 + echo " clickhouse-client --database $DATABASE --query \"RENAME TABLE $DATABASE.traces_local_v2 TO $DATABASE.traces_pre_cutover_backup ON CLUSTER '{cluster}'\"" >&2 + exit 1 + fi } # Pre-EXCHANGE gate: the swap is metadata-only and near-instant, but each replica reads its own local parts afterwards. diff --git a/tests_load/tests/traces-local-v2-cutover/delete_traffic.py b/tests_load/tests/traces-local-v2-cutover/delete_traffic.py index 67ac1c79b99..87a14393014 100644 --- a/tests_load/tests/traces-local-v2-cutover/delete_traffic.py +++ b/tests_load/tests/traces-local-v2-cutover/delete_traffic.py @@ -29,9 +29,9 @@ def _handle_sigint(_signum, _frame): def _fetch_ids(client, project, want, exclude): try: traces = client.search_traces(project_name=project, max_results=want, truncate=True) - except Exception as exc: # search is best-effort; keep the loop alive - LOGGER.warning("search_traces failed: %s", exc) - return [] + except Exception as exc: # transient search failure: signal the caller to retry, not to treat the pool as drained + LOGGER.warning("search_traces failed (will retry): %s", exc) + return None return [t.id for t in traces if t.id not in exclude] @@ -56,7 +56,12 @@ def main(project, tps, duration, batch): tick = time.time() if len(pool) < batch: # Exclude both already-deleted ids and those still queued in `pool`, so a refill can't requeue an in-flight id. - pool.extend(_fetch_ids(client, project, want=500, exclude=seen | set(pool))) + fetched = _fetch_ids(client, project, want=500, exclude=seen | set(pool)) + if fetched is None: + # transient search failure — back off and retry rather than mistaking it for "no more traces". + time.sleep(interval if interval > 0 else 0.5) + continue + pool.extend(fetched) if not pool: LOGGER.info("no more traces to delete; stopping") break diff --git a/tests_load/tests/traces-local-v2-cutover/seed_history.py b/tests_load/tests/traces-local-v2-cutover/seed_history.py index c7e589827dd..7da4bcf30d0 100644 --- a/tests_load/tests/traces-local-v2-cutover/seed_history.py +++ b/tests_load/tests/traces-local-v2-cutover/seed_history.py @@ -54,8 +54,9 @@ "environment", ] -# A far-future instant matching the litellm UUIDv7 bug (ids whose embedded timestamp lands around the year 2201). -BAD_ID_INSTANT = utcnow().replace(year=2201) +# A far-future instant matching the litellm UUIDv7 bug (ids whose embedded timestamp lands around the year 2201). Built +# from a fixed date (not now().replace(year=2201)) so it never hits Feb 29 -> ValueError at import on a leap-day run. +BAD_ID_INSTANT = datetime(2201, 6, 1, tzinfo=timezone.utc) _EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) _TAG_POOL = ["prod", "llm", "rag", "eval", "v1", "v2", "canary", "batch", "stream", "agent"] From 64f627e1fe9ed5dfda19001ab1d567960dbbe357 Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Wed, 22 Jul 2026 19:10:33 +0200 Subject: [PATCH 05/28] =?UTF-8?q?test(cutover):=20address=20third=20Baz=20?= =?UTF-8?q?review=20=E2=80=94=20numeric=20arg=20validation=20+=20comment?= =?UTF-8?q?=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Validate numeric args that flow into the reference SQL / window arithmetic across verify.sh, estimate.sh, delta_replay.sh and backfill.sh (--sample-mod, --max-rows-per-insert, --probe-rows, --max-insert-block-size, week/stride/ pause/factor args) so they fail cleanly instead of altering the query. - Fix the stage-C rollback comment: the reverse-replay runs but matches zero ids here (no deletes bridged in the window), rather than being skipped. - Correct the durationMismatches Javadoc to state the actual 1.5 us tolerance and why (both endpoints truncate ns->us); the threshold is unchanged. --- .../traces-local-v2-cutover/scripts/backfill.sh | 8 ++++++++ .../traces-local-v2-cutover/scripts/delta_replay.sh | 1 + .../traces-local-v2-cutover/scripts/estimate.sh | 6 ++++++ .../traces-local-v2-cutover/scripts/verify.sh | 5 +++++ .../opik/infrastructure/TracesLocalV2CutoverTest.java | 7 +++++-- 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh index ec411862ec2..ceea384f04c 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh @@ -94,6 +94,14 @@ done # --state-file is an operator-owned path read with cat and written with > (both quoted); reject a blank or multi-line # value so the single-line anchor round-trips cleanly. [[ -n "$STATE_FILE" && "$STATE_FILE" != *$'\n'* ]] || { echo "ERROR: --state-file must be a non-empty single-line path." >&2; exit 2; } +# Numeric args flow into the reference SQL / window arithmetic; require sane numeric shapes so none can alter the query. +[[ "$MAX_ROWS" =~ ^[1-9][0-9]*$ ]] || { echo "ERROR: --max-rows-per-insert must be a positive integer." >&2; exit 2; } +[[ "$MAX_INSERT_BLOCK_SIZE" =~ ^[1-9][0-9]*$ ]] || { echo "ERROR: --max-insert-block-size must be a positive integer." >&2; exit 2; } +[[ "$FROM_WEEK" =~ ^[0-9]+$ ]] || { echo "ERROR: --from-week must be a non-negative integer." >&2; exit 2; } +[[ -z "$TO_WEEK" || "$TO_WEEK" =~ ^[0-9]+$ ]] || { echo "ERROR: --to-week must be a non-negative integer." >&2; exit 2; } +[[ "$PAUSE_SECONDS" =~ ^[0-9]+$ ]] || { echo "ERROR: --pause-seconds must be a non-negative integer." >&2; exit 2; } +[[ "$DIVERGENCE" =~ ^[0-9]+(\.[0-9]+)?$ ]] || { echo "ERROR: --divergence must be a number." >&2; exit 2; } +[[ "$MIN_FREE_FACTOR" =~ ^[0-9]+(\.[0-9]+)?$ ]] || { echo "ERROR: --min-free-factor must be a number." >&2; exit 2; } [[ -f "$BACKFILL_SQL" ]] || { echo "ERROR: cannot find backfill SQL at $BACKFILL_SQL" >&2; exit 2; } # Every query runs against the analytics database; --query keeps output scriptable (TSV, no formatting). diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/delta_replay.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/delta_replay.sh index 33d7d371bf0..8b6c1c139e1 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/delta_replay.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/delta_replay.sh @@ -35,6 +35,7 @@ done [[ "$DATABASE" =~ ^[A-Za-z0-9_]+$ ]] || { echo "ERROR: --database must be a ClickHouse identifier (letters, digits, underscore)." >&2; exit 2; } [[ -n "$BACKFILL_START" ]] || { echo "ERROR: --backfill-start is required (printed by backfill.sh)" >&2; exit 2; } [[ "$BACKFILL_START" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}\ [0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?$ ]] || { echo "ERROR: --backfill-start must be 'YYYY-MM-DD HH:MM:SS[.ffffff]'." >&2; exit 2; } +[[ "$MAX_INSERT_BLOCK_SIZE" =~ ^[1-9][0-9]*$ ]] || { echo "ERROR: --max-insert-block-size must be a positive integer." >&2; exit 2; } [[ -f "$SQL_FILE" ]] || { echo "ERROR: cannot find $SQL_FILE" >&2; exit 2; } echo "Reminder: raise databaseAnalytics.asyncInsertBusyTimeoutMaxMs before this step (backend config, not SQL) and" diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/estimate.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/estimate.sh index 6e81d426a13..88ed166319c 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/estimate.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/estimate.sh @@ -51,6 +51,12 @@ done [[ -n "$DATABASE" ]] || { echo "ERROR: --database is required" >&2; exit 2; } # --database is interpolated into the probe/size SQL; require a plain ClickHouse identifier so it cannot alter the query. [[ "$DATABASE" =~ ^[A-Za-z0-9_]+$ ]] || { echo "ERROR: --database must be a ClickHouse identifier (letters, digits, underscore)." >&2; exit 2; } +# Numeric args flow into the probe/estimate SQL and awk; require sane numeric shapes so none can alter the query. +[[ "$MAX_ROWS" =~ ^[1-9][0-9]*$ ]] || { echo "ERROR: --max-rows-per-insert must be a positive integer." >&2; exit 2; } +[[ "$PROBE_ROWS" =~ ^[1-9][0-9]*$ ]] || { echo "ERROR: --probe-rows must be a positive integer." >&2; exit 2; } +[[ "$PAUSE_SECONDS" =~ ^[0-9]+$ ]] || { echo "ERROR: --pause-seconds must be a non-negative integer." >&2; exit 2; } +[[ "$WRITE_COST_FACTOR" =~ ^[0-9]+(\.[0-9]+)?$ ]] || { echo "ERROR: --write-cost-factor must be a number." >&2; exit 2; } +[[ -z "$ROWS_PER_SEC" || "$ROWS_PER_SEC" =~ ^[0-9]+(\.[0-9]+)?$ ]] || { echo "ERROR: --rows-per-sec must be a number." >&2; exit 2; } ch() { clickhouse-client --database "$DATABASE" --query "$1" diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh index 9e34dae7b50..540796ba105 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh @@ -61,6 +61,11 @@ done for _ident in "$DATABASE" "$OLD_TABLE" "$NEW_TABLE"; do [[ "$_ident" =~ ^[A-Za-z0-9_]+$ ]] || { echo "ERROR: --database/--old-table/--new-table must be ClickHouse identifiers (letters, digits, underscore): '$_ident'" >&2; exit 2; } done +# Numeric args are interpolated into the reference SQL / week arithmetic; require integer shapes so none can alter it. +[[ "$SAMPLE_MOD" =~ ^[1-9][0-9]*$ ]] || { echo "ERROR: --sample-mod must be a positive integer." >&2; exit 2; } +[[ "$FROM_WEEK" =~ ^[0-9]+$ ]] || { echo "ERROR: --from-week must be a non-negative integer." >&2; exit 2; } +[[ "$WEEKS_STRIDE" =~ ^[1-9][0-9]*$ ]] || { echo "ERROR: --weeks-stride must be a positive integer." >&2; exit 2; } +[[ -z "$TO_WEEK" || "$TO_WEEK" =~ ^[0-9]+$ ]] || { echo "ERROR: --to-week must be a non-negative integer." >&2; exit 2; } [[ -f "$VERIFY_SQL" ]] || { echo "ERROR: cannot find verify SQL at $VERIFY_SQL" >&2; exit 2; } ch() { diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java index 0eb7f3fbdc7..cf4f5f737a8 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java @@ -568,7 +568,8 @@ void rollbackTopologySignalsDistinguishEveryStage() { .isEqualTo("Distributed"); // Restore the canonical baseline (traces = original, traces_local_v2 = successor) so @BeforeEach's reset — which - // assumes a regular `traces` — works for the next test. No deletes were bridged, so the reverse-replay is a no-op. + // assumes a regular `traces` — works for the next test. The stage-C reverse-replay still runs here; this test + // bridged no deletes in the (cutoverStart, ∞) window, so it matches zero ids and deletes nothing. rollbackAfterWrap(cutoverStart); } @@ -1242,7 +1243,9 @@ private Fingerprint derivedFingerprint(String table, String workspaceId) { * Count of fidelity-cohort rows whose {@code duration} disagrees between source and destination beyond the intended * ns-to-us truncation. The source computes duration from nanosecond timestamps and is {@code NULL} when unset; the * successor computes it from the microsecond copy and is {@code NaN} when unset. So a faithful row is unset on both - * (source NULL, dest NaN) or set on both within one microsecond (0.001 ms); anything else is a real divergence. + * (source NULL, dest NaN) or set on both within 1.5 microseconds (0.0015 ms); anything else is a real divergence. + * The bound is 1.5 us, not 1 us: truncating both the start and end timestamps ns-to-us can each shift the computed + * duration by up to ~1 us, so 0.0015 ms is a deliberate small margin over that (tightening it risks a flaky test). */ private long durationMismatches(String workspaceId) { var sql = """ From 36f075fee9f3508eb00d338181b77fff12b2b671 Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Wed, 22 Jul 2026 20:11:18 +0200 Subject: [PATCH 06/28] test(cutover): self-heal leaked topology in resetTables @BeforeEach reset assumed the prior test left `traces` a regular table. A test that failed mid-cutover (after a swap/wrap, before its rollback) would leave `traces` a Distributed wrapper (or the successor) over dropped/renamed tables, cascading into every later test. Reverse whichever of the two completed non-canonical topologies (post-wrap, post-EXCHANGE) is present before truncating, so one mid-cutover failure no longer corrupts the rest of the run. --- .../infrastructure/TracesLocalV2CutoverTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java index cf4f5f737a8..218315b6c4e 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java @@ -216,6 +216,22 @@ class TracesLocalV2CutoverTest { */ @BeforeEach void resetTables() { + // Self-heal a topology leaked by a test that failed mid-cutover (after a swap/wrap, before its rollback): a green + // run always leaves the canonical layout, but a failure in between would otherwise leave `traces` a Distributed + // wrapper (or the successor) over dropped/renamed tables and cascade into every later test. The tests only ever + // hold two completed non-canonical topologies across a boundary; reverse whichever is present, then truncate. + if (isDistributed("traces")) { // post-wrap: traces = Distributed over traces_local; original parked as backup + execute("DROP TABLE traces ON CLUSTER '{cluster}' SYNC", _ -> { + }); + execute("RENAME TABLE traces_pre_cutover_backup TO traces, traces_local TO traces_local_v2 ON CLUSTER '{cluster}'", + _ -> { + }); + } else if (tableExists("traces_pre_cutover_backup")) { // post-EXCHANGE: traces = successor; original parked as backup + execute("EXCHANGE TABLES traces AND traces_pre_cutover_backup ON CLUSTER '{cluster}'", _ -> { + }); + execute("RENAME TABLE traces_pre_cutover_backup TO traces_local_v2 ON CLUSTER '{cluster}'", _ -> { + }); + } execute("DROP TABLE IF EXISTS traces_local ON CLUSTER '{cluster}' SYNC", _ -> { }); execute("DROP TABLE IF EXISTS traces_pre_cutover_backup ON CLUSTER '{cluster}' SYNC", _ -> { From 9793a9b7cb09ff5046e119e7d05ac3080c60241e Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Wed, 22 Jul 2026 20:37:00 +0200 Subject: [PATCH 07/28] test(cutover): make resetTables normalize from any leaked topology MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prior self-heal reversed only the two completed topologies and ran DDL without confirming every referenced table existed, so a partial swap/wrap (only the first of a two-statement helper) could make the reset itself throw and cascade. Rewrite it to normalize to canonical from every reachable state — canonical, partial/complete EXCHANGE, partial/complete wrap — using the Nullable-end_time schema check to tell the original table from the successor, and guarding every DDL on the tables it touches so no leaked state can throw. Verified: all 8 tests in the class pass. --- .../TracesLocalV2CutoverTest.java | 51 +++++++++++++------ 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java index 218315b6c4e..34303fc2504 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java @@ -208,30 +208,49 @@ class TracesLocalV2CutoverTest { } /** - * Each rollback stage ends in the canonical state (traces = original schema, traces_local_v2 = successor), which is - * also the pre-EXCHANGE baseline — so truncating the three tables (and dropping any stray wrap/rename artifacts) is a - * clean reset that keeps the tests independent of each other and of run order. The stray artifacts are the - * post-wrap {@code traces_local} shard and the post-EXCHANGE {@code traces_pre_cutover_backup} — a test that fails - * mid-cutover (after the swap/wrap, before its rollback) would otherwise leak one of them into the next test. + * Restore the canonical baseline (traces = original schema, traces_local_v2 = successor schema, both empty; no stray + * wrap/rename artifacts) before every test, independent of what the previous test left behind. A green run always + * ends canonical, but a test that fails mid-cutover can leak any intermediate topology, so rather than assume a clean + * hand-off this normalizes whatever is present back to canonical. The cutover only ever produces these shapes: the + * completed EXCHANGE (traces = successor, original parked as traces_pre_cutover_backup) and wrap (traces = + * Distributed over traces_local), plus the partial states where only the first of a two-statement swap/wrap ran. + * Every DDL below is guarded on the tables it touches, so no leaked state can make the reset itself throw and + * cascade into later tests. {@code end_time} being Nullable is the original schema, non-Nullable the successor. */ @BeforeEach void resetTables() { - // Self-heal a topology leaked by a test that failed mid-cutover (after a swap/wrap, before its rollback): a green - // run always leaves the canonical layout, but a failure in between would otherwise leave `traces` a Distributed - // wrapper (or the successor) over dropped/renamed tables and cascade into every later test. The tests only ever - // hold two completed non-canonical topologies across a boundary; reverse whichever is present, then truncate. - if (isDistributed("traces")) { // post-wrap: traces = Distributed over traces_local; original parked as backup + // 1. Wrap: `traces` is a Distributed wrapper holding no data of its own — drop it, leaving the successor under + // traces_local and the original under traces_pre_cutover_backup (the same shape as a partial wrap). + if (isDistributed("traces")) { execute("DROP TABLE traces ON CLUSTER '{cluster}' SYNC", _ -> { }); - execute("RENAME TABLE traces_pre_cutover_backup TO traces, traces_local TO traces_local_v2 ON CLUSTER '{cluster}'", - _ -> { - }); - } else if (tableExists("traces_pre_cutover_backup")) { // post-EXCHANGE: traces = successor; original parked as backup - execute("EXCHANGE TABLES traces AND traces_pre_cutover_backup ON CLUSTER '{cluster}'", _ -> { + } + // 2. Wrap (completed or partial): successor parked as traces_local, original as traces_pre_cutover_backup, with + // `traces` absent. Restore both names. + if (!tableExists("traces_local_v2") && tableExists("traces_local")) { + execute("RENAME TABLE traces_local TO traces_local_v2 ON CLUSTER '{cluster}'", _ -> { }); - execute("RENAME TABLE traces_pre_cutover_backup TO traces_local_v2 ON CLUSTER '{cluster}'", _ -> { + } + if (!tableExists("traces") && tableExists("traces_pre_cutover_backup")) { + execute("RENAME TABLE traces_pre_cutover_backup TO traces ON CLUSTER '{cluster}'", _ -> { }); } + // 3. EXCHANGE (completed or partial): `traces` exists but holds the SUCCESSOR schema. Un-swap it with the parked + // original — under traces_pre_cutover_backup once the EXCHANGE completed, or still under traces_local_v2 if + // only the EXCHANGE ran and its follow-up RENAME did not. + if (tableExists("traces") && !columnType("traces", "end_time").startsWith("Nullable")) { + if (tableExists("traces_pre_cutover_backup")) { + execute("EXCHANGE TABLES traces AND traces_pre_cutover_backup ON CLUSTER '{cluster}'", _ -> { + }); + execute("RENAME TABLE traces_pre_cutover_backup TO traces_local_v2 ON CLUSTER '{cluster}'", _ -> { + }); + } else if (tableExists("traces_local_v2")) { + execute("EXCHANGE TABLES traces AND traces_local_v2 ON CLUSTER '{cluster}'", _ -> { + }); + } + } + // 4. Canonical now; truncate the two tables and clear any residual artifacts (IF EXISTS so a genuinely + // unrecoverable partial state still cannot throw here). execute("DROP TABLE IF EXISTS traces_local ON CLUSTER '{cluster}' SYNC", _ -> { }); execute("DROP TABLE IF EXISTS traces_pre_cutover_backup ON CLUSTER '{cluster}' SYNC", _ -> { From 049b66815c9f8c5d21c717f8442259026eea6c8c Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Fri, 24 Jul 2026 15:18:33 +0200 Subject: [PATCH 08/28] =?UTF-8?q?test(cutover):=20address=20Thiago=20revie?= =?UTF-8?q?w=20=E2=80=94=20drop-size=20guard,=20UTC=20bounds,=20log=5Fcomm?= =?UTF-8?q?ent,=20doc=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Substantive: - Add SETTINGS max_table_size_to_drop = 0 to the stage-A TRUNCATE and the finalize DROP so rollback/finalize don't throw on prod-scale (>50 GB) tables. - Force 'UTC' on the toDateTime64 window bounds (000001 INSERT + backfill.sh reconciliation) so the window invariant is enforced, not assumed to match the UTC-forced settled/split checks. - Log both counts in the reconciliation "OK" line; reserve "equal" for exact match. - Add log_comment to the heavy backfill/delta INSERTs and the deletion-replay DELETE for query_log/mutations attribution during the cutover. Docs/comments: - Fix the false "id skip index" claim on source traces (000088 indexes only created_at/last_updated_at); reconcile the README. - Note the deferred --wrap-only path briefly makes traces unavailable and needs the buffer re-raised (driver reminder + README). - Reword the Go/No-Go replay item to gate on the final-delta->EXCHANGE gap. - Restate prereq 1 without a never-running retention cycle. - Correct the 000001 "read in destination order" comment (no such guarantee). - Document the workspace-scoped replay residual (retired by OPIK-7483; caught by 000005), stage-B non-atomic recovery, and stage-C brief traces absence. Co-Authored-By: Claude Opus 4.8 --- .../traces-local-v2-cutover/README.md | 25 +++++++++++++------ .../scripts/backfill.sh | 16 ++++++------ .../000001_backfill_traces_local_v2.sql | 13 ++++++---- .../000002_delta_and_deletion_replay.sql | 17 ++++++++++--- ...000004_rollback_stage_a_discard_shadow.sql | 6 ++++- .../000004_rollback_stage_b_exchange_back.sql | 7 ++++++ ...0004_rollback_stage_c_promote_original.sql | 5 ++++ .../scripts/exchange_and_wrap.sh | 8 ++++++ .../scripts/finalize.sh | 4 ++- 9 files changed, 76 insertions(+), 25 deletions(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md index 208a39ce4c8..3578c964e59 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md @@ -52,8 +52,10 @@ new table before the EXCHANGE. The replay matches the **full key**, not `id` alo ## Prerequisites (do not start without these) -1. **24h UUIDv7 ingestion validation** live for **≥ one full retention cycle**. Without it a stray future-dated UUID - creates an out-of-range weekly partition on the destination. +1. **24h UUIDv7 ingestion validation** live long enough that no un-validated future-dated ids land in newly ingested + weeks. This is not tied to a retention cycle (retention never runs — prereq 8). Pre-validation bad-id rows already in + the table are *not* blocked by this: they are copied by the `created_at` slice and surfaced by the far-future audit + query below — this prereq only ensures no *new* out-of-range partitions are created mid-cutover. 2. **`traces_local_v2` exists and is empty** (migration 000101). 3. **Successor storage/TTL parity.** `traces_local_v2` must resolve the **same `storage_policy` and TTL-to-cold rules** as `traces` (tiering is configured per environment, not in the base DDL). If `traces` tiers hot→cold but the @@ -158,6 +160,11 @@ new table before the EXCHANGE. The replay matches the **full key**, not `id` alo > `Distributed` wrap on the already-swapped `traces` (no second EXCHANGE, no new `cutover_start`). To roll the wrap > back, use `rollback.sh --stage C`. > +> The wrap is a non-atomic `RENAME`→`CREATE`, so `traces` is **briefly absent** (INSERT/SELECT fail with "Table doesn't +> exist" during that window; `ON CLUSTER` widens it per node). The same-run path is covered by the still-raised EXCHANGE +> buffer; for the deferred `--wrap-only` path, **re-raise `asyncInsertBusyTimeoutMaxMs` (or quiesce ingestion / take a +> maintenance window) first** so the wrap runs under the same buffered conditions. +> > **Wrap flags** (`exchange_and_wrap.sh`, mutually exclusive; default is EXCHANGE-only): omit them (or pass > `--skip-wrap`, an explicit alias) to run the EXCHANGE and stop; `--with-wrap` to also apply the wrap in the same run; > `--wrap-only` to apply just the deferred wrap later. @@ -259,9 +266,12 @@ and it does **not** decide where a row lands on the destination: that is always the original), so a row never migrates between weekly slices mid-backfill — none is copied twice or skipped. `last_updated_at` would *not* be safe here (it moves on every upsert, and is client-settable). -**On adding an `id` index (and destination write locality).** An `id` skip index on the source is *not* needed — the -delta uses the `created_at`/`last_updated_at` skip indexes, and the replay uses the primary key (full-key match). It -would not rescue `id`-slicing either (the ~2201 span is a *data* problem, not an index one). Destination write locality +**On adding an `id` index (and destination write locality).** An `id` skip index on the source is *not* needed. The +delta uses the `created_at`/`last_updated_at` skip indexes, and the replay's outer DELETE matches the full primary key. +Its resurrection-guard subquery does read the source `traces` by bare `id` (which has **no** skip index — 000088 indexes +only `created_at`/`last_updated_at`; the `id` minmax/bloom indexes exist only on `traces_local_v2` per 000101), but the +`id IN (deleted-ids since anchor)` set is tiny (retention off → user-scale deletes), so it is a bounded id-filtered read, +not a full-table scan. An index still would not rescue `id`-slicing (the ~2201 span is a *data* problem, not an index one). Destination write locality is naturally good with `created_at` slicing (`id_at ≈ created_at` once validation holds); slicing by *workspace* would instead scatter each insert across every weekly partition that workspace spans → a small-part explosion on a 4 TB table. @@ -574,8 +584,9 @@ cheap (stage A); the bridge stays enabled so nothing is lost on a retry. production **topology**, not just data shape — same replica count and tiered-storage policy — since the multi-replica settle gates, storage/TTL parity, and buffer-flush timing are otherwise untested until production. - [ ] **Deletion test green** — `TracesLocalV2CutoverTest` passes; **0 deletion leaks** confirmed on staging. -- [ ] **Replay duration fits inside the buffer window with margin** — measured replay wall time < the widened - `asyncInsertBusyTimeoutMaxMs`. +- [ ] **Final-delta→EXCHANGE gap fits inside the buffer hold with margin** — the binding invariant is the gap between + the final delta and the EXCHANGE completing (≈ replay wall time + EXCHANGE), staying within the buffer hold and + accounting for size-triggered flushes — **not** "replay < buffer window" alone (see "The final cutover window"). - [ ] **Far-future partitions quantified** — run the bad-`id` audit query above; remediated or explicitly accepted. - [ ] **`EXCHANGE TABLES ... ON CLUSTER` works end-to-end** — or the fallback `RENAME` sequence is documented for the variant that needs it. diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh index ceea384f04c..a08b0577369 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh @@ -146,8 +146,8 @@ preflight_capacity() { count_src() { ch "SELECT count() FROM $SRC_TABLE - WHERE created_at >= toDateTime64('$1', 9) - AND created_at < toDateTime64('$2', 9)" + WHERE created_at >= toDateTime64('$1', 9, 'UTC') + AND created_at < toDateTime64('$2', 9, 'UTC')" } # Distinct LOGICAL rows in [lo, hi), by the ReplacingMergeTree dedup key. Reconciliation must be dedup-aware: raw @@ -156,15 +156,15 @@ count_src() { count_src_uniq() { ch "SELECT uniqExact(workspace_id, project_id, id) FROM $SRC_TABLE - WHERE created_at >= toDateTime64('$1', 9) - AND created_at < toDateTime64('$2', 9)" + WHERE created_at >= toDateTime64('$1', 9, 'UTC') + AND created_at < toDateTime64('$2', 9, 'UTC')" } count_dst_uniq() { ch "SELECT uniqExact(workspace_id, project_id, id) FROM $DST_TABLE - WHERE created_at >= toDateTime64('$1', 6) - AND created_at < toDateTime64('$2', 6)" + WHERE created_at >= toDateTime64('$1', 6, 'UTC') + AND created_at < toDateTime64('$2', 6, 'UTC')" } # Render the reference INSERT for one window by substituting placeholders (pure bash, no envsubst dependency). @@ -219,8 +219,10 @@ insert_window() { log "$label ($lo .. $hi): live window (src_uniq=$src dst_uniq=$dst) — the delta-insert will reconcile concurrent writes" elif [[ "$dst" -gt "$src" ]]; then log "$label ($lo .. $hi): src_uniq=$src dst_uniq=$dst — concurrent source deletes; the deletion replay will reconcile" - else + elif [[ "$dst" == "$src" ]]; then log "$label ($lo .. $hi): OK (src_uniq=dst_uniq=$src)" + else + log "$label ($lo .. $hi): OK within tolerance (src_uniq=$src dst_uniq=$dst)" fi if [[ "$PAUSE_SECONDS" != "0" ]]; then diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000001_backfill_traces_local_v2.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000001_backfill_traces_local_v2.sql index aad961d0b0c..b983421ce02 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000001_backfill_traces_local_v2.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000001_backfill_traces_local_v2.sql @@ -12,8 +12,10 @@ -- * end_time and ttft are the two denullified columns: coalesce them to their sentinels (epoch / NaN). -- * is_deleted is omitted so the new column defaults to 0. -- * apply_deleted_mask stays at its default 1, so rows already lightweight-deleted on the source are skipped. --- * No explicit ORDER BY: source and destination share ORDER BY (workspace_id, project_id, id), so rows are read in --- destination order and parts form cleanly (see README "Why slice by created_at"). +-- * No explicit ORDER BY: omitted deliberately to avoid a full per-window sort (memory). A parallel SELECT gives no +-- output-order guarantee, so inserted blocks may span/interleave partitions; the destination ReplacingMergeTree +-- dedups regardless of insert order and background merges compact the parts. This is NOT a claim that rows arrive +-- in sort-key order — do not rely on it (see README "Why slice by created_at"). -- * SETTINGS max_insert_block_size bounds the rows per part-forming block; peak insert memory is a small multiple of -- the smaller of that and min_insert_block_size_bytes (256 MB default), which dominates for wide trace rows. @@ -67,9 +69,10 @@ SELECT source, environment FROM ${ANALYTICS_DB_DATABASE_NAME}.traces -WHERE created_at >= toDateTime64('${WINDOW_LO}', 9) - AND created_at < toDateTime64('${WINDOW_HI}', 9) -SETTINGS max_insert_block_size = ${MAX_INSERT_BLOCK_SIZE}; +WHERE created_at >= toDateTime64('${WINDOW_LO}', 9, 'UTC') + AND created_at < toDateTime64('${WINDOW_HI}', 9, 'UTC') +SETTINGS max_insert_block_size = ${MAX_INSERT_BLOCK_SIZE}, + log_comment = 'traces_local_v2_backfill:${WINDOW_LO}:${WINDOW_HI}'; -- Per-window reconciliation is automated by backfill.sh (uniqExact of the dedup key, aborting on > 0.01% divergence); -- fidelity QA across the whole copy is 000005 via verify.sh. Rollback before the EXCHANGE: rollback.sh --stage A. diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000002_delta_and_deletion_replay.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000002_delta_and_deletion_replay.sql index dedb4bad996..f7ae2b73875 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000002_delta_and_deletion_replay.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000002_delta_and_deletion_replay.sql @@ -83,7 +83,8 @@ SELECT FROM ${ANALYTICS_DB_DATABASE_NAME}.traces WHERE created_at >= toDateTime64('${BACKFILL_START}', 6) OR last_updated_at >= toDateTime64('${BACKFILL_START}', 6) -SETTINGS max_insert_block_size = ${MAX_INSERT_BLOCK_SIZE}; +SETTINGS max_insert_block_size = ${MAX_INSERT_BLOCK_SIZE}, + log_comment = 'traces_local_v2_cutover:delta_insert'; -- Step 4: Deletion replay — remove from the destination every row that was deleted on the source since backfill_start -- AND is still deleted there. Two branches, mirroring the product's two delete paths (TraceService.delete): a delete @@ -97,8 +98,15 @@ SETTINGS max_insert_block_size = ${MAX_INSERT_BLOCK_SIZE}; -- during the window (client-supplied ids; the delete is a mask, a newer insert wins under FINAL). Such an id is bridged -- as deleted but is LIVE again on the source, and the backfill/delta already copied its live version. Deleting it by key -- would drop a row that is live on the source — silent data loss. So each branch deletes only ids that are NOT currently --- live on the source (mask-honored). The `id IN (deleted_ids since anchor)` bound keeps the source lookup pruned to the --- window's deletes (id skip index), not a full scan. +-- live on the source (mask-honored). The `id IN (deleted_ids since anchor)` bound keeps the deleted-id set tiny +-- (retention is off, so these are user-scale deletes); `traces` has no id skip index (000088 indexes only +-- created_at/last_updated_at — id minmax/bloom indexes exist only on traces_local_v2), so this source lookup is a +-- bounded id-filtered read of that tiny set, not a value-indexed prune of the ~4 TB table. +-- KNOWN RESIDUAL (workspace-scoped arm only): its guard keys on (workspace_id, id), so an id live in ONE project shields +-- the deletion of that id's now-deleted copies in OTHER projects (ids are not globally unique). Requires cross-project id +-- reuse + a workspace-scoped delete + a resurrection in the window — rare. The 000005 FINAL fingerprint flags it as an +-- extra destination row (ok=0) rather than passing silently. OPIK-7483 removes the workspace-scoped delete path at the +-- source (deletes always carry project_id), retiring this arm and the residual. -- allow_nondeterministic_mutations: a lightweight DELETE with cross-table subqueries is flagged nondeterministic, but -- deletion_events_local and traces are replicated and identical on every node and the window predicate is fixed, so the -- subqueries resolve to the same set on every replica. Idempotent (never masks a live-on-source id, so re-runs converge). @@ -158,7 +166,8 @@ OR ( ) ) SETTINGS allow_nondeterministic_mutations = 1, - lightweight_deletes_sync = 2; + lightweight_deletes_sync = 2, + log_comment = 'traces_local_v2_cutover:deletion_replay'; -- Step 5: Measure the replay. Compare its wall time against the buffer window (must fit with margin — acceptance -- criterion). Re-run steps 3-4 if new rows/deletes accumulated during the replay itself; convergence is fast because diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_a_discard_shadow.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_a_discard_shadow.sql index e603f190e0b..f11db87f16a 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_a_discard_shadow.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_a_discard_shadow.sql @@ -10,4 +10,8 @@ -- running it, so it cannot fire post-EXCHANGE — by which point the old original has been renamed to -- `traces_pre_cutover_backup` and `traces_local_v2` no longer exists, so there is nothing here that could touch the -- backup. -TRUNCATE TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 ON CLUSTER '{cluster}'; +-- +-- max_table_size_to_drop = 0 disables the drop-size guard for this statement: TRUNCATE is subject to +-- max_table_size_to_drop (default 50 GB) just like DROP, and the shadow is well over that on any instance large enough +-- to need this runbook — without the override the rollback throws exactly when it is needed. +TRUNCATE TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 ON CLUSTER '{cluster}' SETTINGS max_table_size_to_drop = 0; diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_b_exchange_back.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_b_exchange_back.sql index 591e4b3fa0c..a0a584decfe 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_b_exchange_back.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_b_exchange_back.sql @@ -6,6 +6,13 @@ -- parked), identical to pre-EXCHANGE. Non-destructive. rollback.sh runs the reverse-replay -- (000004_rollback_reverse_replay.sql) right after this so deletes since cutover_start do not resurrect. rollback.sh -- asserts the post-EXCHANGE, pre-wrap topology (traces = successor schema, not Distributed) before running it. +-- +-- These two statements are NOT jointly atomic (unlike stage C's single multi-target RENAME). The EXCHANGE already makes +-- live `traces` the original again; if the following RENAME fails (e.g. a transient per-replica ON CLUSTER DDL error), +-- the abandoned successor is left under `traces_pre_cutover_backup` instead of the canonical `traces_local_v2`, and +-- rollback.sh cannot re-drive it (stage B/A both abort on that split state). RECOVERY: live data is already correct; +-- only the relabel remains — run it manually: +-- RENAME TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces_pre_cutover_backup TO ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 ON CLUSTER '{cluster}'; EXCHANGE TABLES ${ANALYTICS_DB_DATABASE_NAME}.traces AND ${ANALYTICS_DB_DATABASE_NAME}.traces_pre_cutover_backup ON CLUSTER '{cluster}'; RENAME TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces_pre_cutover_backup TO ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 ON CLUSTER '{cluster}'; diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql index e58a7e2e1ec..e624092cc89 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql @@ -8,6 +8,11 @@ -- -- rollback.sh runs the reverse-replay (000004_rollback_reverse_replay.sql) right after this so deletes since -- cutover_start do not resurrect, and asserts the post-wrap topology (traces = Distributed) before running it. +-- +-- Note: between the DROP and the RENAME the name `traces` does not resolve on any node, so a read/insert hitting +-- `traces` in that brief window fails with "Table doesn't exist" (the RENAME itself is atomic, but the preceding DROP is +-- not gapless like stage B's EXCHANGE). Acceptable within the rollback maintenance window — the wrapper holds no data and +-- the wrap has already disrupted the product's delete paths, so `traces` is not fully serving anyway. DROP TABLE IF EXISTS ${ANALYTICS_DB_DATABASE_NAME}.traces ON CLUSTER '{cluster}' SYNC; RENAME TABLE diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh index 4482dd0abad..c81ee49673a 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh @@ -184,6 +184,14 @@ if [[ "$WRAP_ONLY" == "1" ]]; then # Deferred second half: the EXCHANGE already happened in a prior --skip-wrap run, so `traces` is the live # partitioned data. Do not re-EXCHANGE (that would swap the parked original back in) and do not capture a new # cutover_start (the data cutover is already done). Just apply the Distributed wrap. + # + # The wrap is two non-atomic statements (RENAME traces -> traces_local, then CREATE Distributed traces); between them + # `traces` does not exist, so concurrent INSERT/SELECT fails with "Table traces doesn't exist" (ON CLUSTER widens the + # window per-node). The same-run path is covered by the still-raised EXCHANGE buffer, but --wrap-only runs later + # against live, unbuffered ingestion. PRECONDITION: re-raise databaseAnalytics.asyncInsertBusyTimeoutMaxMs (or quiesce + # ingestion / assert a maintenance window) so the wrap runs under the same buffered conditions as the EXCHANGE. + echo "PRECONDITION: the wrap briefly makes 'traces' unavailable (non-atomic RENAME->CREATE). Re-raise the async-insert" + echo "buffer (asyncInsertBusyTimeoutMaxMs) or quiesce ingestion / assert a maintenance window before continuing." run_block wrap echo "Distributed wrap done: 'traces' fronts 'traces_local' via sipHash64(project_id). (EXCHANGE was a prior step.)" exit 0 diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/finalize.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/finalize.sh index 5f50278d7af..18870b9f105 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/finalize.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/finalize.sh @@ -82,5 +82,7 @@ if [[ "$CONFIRM" != "1" ]]; then exit 0 fi -ch "DROP TABLE IF EXISTS $BACKUP ON CLUSTER '{cluster}' SYNC" +# max_table_size_to_drop = 0 disables the drop-size guard (default 50 GB): the parked backup is the full old original +# (multi-TB after a successful cutover), so without the override the DROP throws "size exceeds the limit". +ch "DROP TABLE IF EXISTS $BACKUP ON CLUSTER '{cluster}' SYNC SETTINGS max_table_size_to_drop = 0" echo "Dropped $DATABASE.$BACKUP. The cutover is finalized." From 2959f33699062bc24eebc37bbd7e684b773a5275 Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Fri, 24 Jul 2026 16:04:06 +0200 Subject: [PATCH 09/28] test(cutover): make stage-C rollback gapless; enforce --wrap-only maintenance gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage C previously DROP-then-RENAMEd, leaving `traces` briefly absent. Promote the original back with an atomic EXCHANGE instead (traces is never absent on a node), then drop the now data-less wrapper and park the shard — same end state, gaplessly. Updated the gate test's rollbackAfterWrap() to match; 8/8 green. exchange_and_wrap.sh: the deferred --wrap-only path is a non-atomic RENAME->CREATE that briefly makes `traces` unavailable against live, unbuffered ingestion. Turn the documented precondition into an enforced gate — --wrap-only now requires --confirm-maintenance (checked up front, before touching ClickHouse) asserting the buffer is re-raised / ingestion quiesced / a maintenance window is in effect. True per-node gaplessness for the wrap itself (pre-created Distributed + atomic multi-RENAME rotate) is deferred to prod-clone validation (OPIK-7404/7405). Co-Authored-By: Claude Opus 4.8 --- ...0004_rollback_stage_c_promote_original.sql | 27 +++++++++---------- .../scripts/exchange_and_wrap.sh | 17 ++++++++++-- .../TracesLocalV2CutoverTest.java | 19 ++++++------- 3 files changed, 36 insertions(+), 27 deletions(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql index e624092cc89..c61b44217dc 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql @@ -1,21 +1,20 @@ -- runbook traces-local-v2-cutover — ROLLBACK stage C: promote the original back (driven by ../rollback.sh --stage C) -- -- Use when the wrap ran. Post-wrap topology: `traces` is a Distributed wrapper over `traces_local` (successor data); --- `traces_pre_cutover_backup` parks the original. Drop the wrapper (it stores no data — a routing definition only), --- then in one atomic RENAME promote the original back to `traces` and park the successor data back under --- `traces_local_v2` — ending in the canonical state (traces = original live, traces_local_v2 = successor parked), with --- no leftover names. +-- `traces_pre_cutover_backup` parks the original. Promote the original back to `traces` GAPLESSLY: EXCHANGE swaps the +-- names atomically, so `traces` becomes the original MergeTree with no window where the name is absent (unlike a +-- DROP-then-RENAME). The wrapper (now parked under the backup name) holds no data, so it is dropped next, and the +-- successor shard is parked back under `traces_local_v2` — ending in the canonical state (traces = original live, +-- traces_local_v2 = successor parked), with no leftover names. -- -- rollback.sh runs the reverse-replay (000004_rollback_reverse_replay.sql) right after this so deletes since -- cutover_start do not resurrect, and asserts the post-wrap topology (traces = Distributed) before running it. --- --- Note: between the DROP and the RENAME the name `traces` does not resolve on any node, so a read/insert hitting --- `traces` in that brief window fails with "Table doesn't exist" (the RENAME itself is atomic, but the preceding DROP is --- not gapless like stage B's EXCHANGE). Acceptable within the rollback maintenance window — the wrapper holds no data and --- the wrap has already disrupted the product's delete paths, so `traces` is not fully serving anyway. -DROP TABLE IF EXISTS ${ANALYTICS_DB_DATABASE_NAME}.traces ON CLUSTER '{cluster}' SYNC; -RENAME TABLE - ${ANALYTICS_DB_DATABASE_NAME}.traces_pre_cutover_backup TO ${ANALYTICS_DB_DATABASE_NAME}.traces, - ${ANALYTICS_DB_DATABASE_NAME}.traces_local TO ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 - ON CLUSTER '{cluster}'; +-- 1. Gapless promote: `traces` immediately becomes the original MergeTree; the Distributed wrapper moves to the backup name. +EXCHANGE TABLES ${ANALYTICS_DB_DATABASE_NAME}.traces AND ${ANALYTICS_DB_DATABASE_NAME}.traces_pre_cutover_backup ON CLUSTER '{cluster}'; + +-- 2. Drop the now data-less Distributed wrapper (routing definition only — no size guard needed). +DROP TABLE IF EXISTS ${ANALYTICS_DB_DATABASE_NAME}.traces_pre_cutover_backup ON CLUSTER '{cluster}' SYNC; + +-- 3. Park the successor shard back under its canonical name. +RENAME TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces_local TO ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 ON CLUSTER '{cluster}'; diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh index c81ee49673a..9dbb9043a27 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh @@ -29,6 +29,10 @@ # --force skip the replication-settle gate. By default the swap aborts while any replica still # has replication-queue backlog or an unfinished mutation on traces / traces_local_v2, since a # behind replica would swap in an incomplete table. Use only if settlement is confirmed out of band. +# --confirm-maintenance REQUIRED with --wrap-only. The wrap is a non-atomic RENAME->CREATE that briefly makes `traces` +# unavailable; unlike the same-run --with-wrap path (still buffered from the EXCHANGE), --wrap-only +# runs later against live, unbuffered ingestion. This flag asserts the async-insert buffer is +# re-raised (or ingestion quiesced / a maintenance window is in effect); the wrap refuses without it. set -euo pipefail @@ -40,6 +44,7 @@ SKIP_WRAP=0 WITH_WRAP=0 WRAP_ONLY=0 FORCE=0 +CONFIRM_MAINTENANCE=0 while [[ $# -gt 0 ]]; do case "$1" in @@ -48,6 +53,7 @@ while [[ $# -gt 0 ]]; do --with-wrap) WITH_WRAP=1; shift ;; --wrap-only) WRAP_ONLY=1; shift ;; --force) FORCE=1; shift ;; + --confirm-maintenance) CONFIRM_MAINTENANCE=1; shift ;; *) echo "Unknown argument: $1" >&2; exit 2 ;; esac done @@ -60,6 +66,14 @@ done if (( SKIP_WRAP + WITH_WRAP + WRAP_ONLY > 1 )); then echo "ERROR: --skip-wrap, --with-wrap and --wrap-only are mutually exclusive" >&2; exit 2 fi +# The deferred wrap briefly makes `traces` unavailable (non-atomic RENAME->CREATE) and runs against live, unbuffered +# ingestion — unlike the same-run --with-wrap path, which is still buffered from the EXCHANGE. Refuse (fail fast, before +# touching ClickHouse) unless the operator asserts the buffer is re-raised / ingestion quiesced / a maintenance window. +if [[ "$WRAP_ONLY" == "1" && "$CONFIRM_MAINTENANCE" != "1" ]]; then + echo "ERROR: --wrap-only requires --confirm-maintenance. Re-raise asyncInsertBusyTimeoutMaxMs (or quiesce ingestion /" >&2 + echo " take a maintenance window) first — the wrap briefly makes 'traces' unavailable — then re-run with it." >&2 + exit 2 +fi ch() { clickhouse-client --database "$DATABASE" --query "$1" @@ -190,8 +204,7 @@ if [[ "$WRAP_ONLY" == "1" ]]; then # window per-node). The same-run path is covered by the still-raised EXCHANGE buffer, but --wrap-only runs later # against live, unbuffered ingestion. PRECONDITION: re-raise databaseAnalytics.asyncInsertBusyTimeoutMaxMs (or quiesce # ingestion / assert a maintenance window) so the wrap runs under the same buffered conditions as the EXCHANGE. - echo "PRECONDITION: the wrap briefly makes 'traces' unavailable (non-atomic RENAME->CREATE). Re-raise the async-insert" - echo "buffer (asyncInsertBusyTimeoutMaxMs) or quiesce ingestion / assert a maintenance window before continuing." + # --confirm-maintenance was already enforced up front (the wrap briefly makes `traces` unavailable). run_block wrap echo "Distributed wrap done: 'traces' fronts 'traces_local' via sipHash64(project_id). (EXCHANGE was a prior step.)" exit 0 diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java index 34303fc2504..eccb6af4ece 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java @@ -890,20 +890,17 @@ private void rollbackExchangeBack(String cutoverStart) { } /** - * Rollback stage C (000004_rollback_stage_c + reverse_replay): drop the Distributed wrapper (no data), then in one - * atomic RENAME promote the parked original (from {@code traces_pre_cutover_backup}) back to {@code traces} and park - * the successor data back under {@code traces_local_v2}, ending in the canonical state with no leftover names. Then - * reverse-replay. + * Rollback stage C (000004_rollback_stage_c + reverse_replay): promote the parked original back to {@code traces} + * GAPLESSLY — EXCHANGE swaps it in atomically (no window where {@code traces} is absent), then the now-data-less + * Distributed wrapper (parked under the backup name) is dropped and the successor shard is parked back under + * {@code traces_local_v2}, ending in the canonical state with no leftover names. Then reverse-replay. */ private void rollbackAfterWrap(String cutoverStart) { - execute("DROP TABLE traces ON CLUSTER '{cluster}' SYNC", _ -> { + execute("EXCHANGE TABLES traces AND traces_pre_cutover_backup ON CLUSTER '{cluster}'", _ -> { }); - execute(""" - RENAME TABLE - traces_pre_cutover_backup TO traces, - traces_local TO traces_local_v2 - ON CLUSTER '{cluster}' - """, _ -> { + execute("DROP TABLE IF EXISTS traces_pre_cutover_backup ON CLUSTER '{cluster}' SYNC", _ -> { + }); + execute("RENAME TABLE traces_local TO traces_local_v2 ON CLUSTER '{cluster}'", _ -> { }); reverseReplay(cutoverStart); } From 33851e6a4b7ab815f8d1aa37283e76ddea9d5417 Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Fri, 24 Jul 2026 16:29:37 +0200 Subject: [PATCH 10/28] test(cutover): make stage-B rollback atomic; clarify buffer prereq is operator-verified MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage B was EXCHANGE + RENAME (two non-atomic statements) — a failed RENAME could strand the successor under traces_pre_cutover_backup. Replace with a single atomic multi-target RENAME rotate (traces -> traces_local_v2, traces_pre_cutover_backup -> traces, the name freed by the first clause), so there is no orphan window. The freed-name-reuse rename-rotate is validated against ClickHouse 26.3 by the gate test (8/8 green); updated rollbackExchangeBack() to match. README prereq 6: state explicitly that the async-insert ceiling is operator-verified (confirmed on the prod-clone/staging load test + Go/No-Go), not script-enforced — it's a backend per-query setting the migration's own clickhouse-client session cannot see or gate on. Co-Authored-By: Claude Opus 4.8 --- .../traces-local-v2-cutover/README.md | 10 ++++--- .../000004_rollback_stage_b_exchange_back.sql | 27 +++++++++---------- .../TracesLocalV2CutoverTest.java | 18 ++++++++----- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md index 3578c964e59..157f4528547 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md @@ -70,10 +70,12 @@ new table before the EXCHANGE. The replay matches the **full key**, not `id` alo 6. **Cutover buffer knob ready** — `databaseAnalytics.asyncInsertBusyTimeoutMaxMs` (env `ANALYTICS_DB_ASYNC_INSERT_BUSY_TIMEOUT_MAX_MS`), unset by default so the buffer inherits the `async_insert_busy_timeout_max_ms=250` carried by `queryParameters`. Raise it to ~10000 for the cutover, then unset it - again. On a staging load test, confirm the adaptive-timeout widening actually takes effect before production. **Also - confirm client/SDK insert timeouts exceed the widened buffer** (~10s) — with `wait_for_async_insert=1` a raised - ceiling blocks each insert until it flushes, so a shorter client timeout would surface as ingestion errors during the - window. + again. **This is operator-verified, not script-enforced:** the ceiling is a backend per-query setting applied on the + backend's own ClickHouse client, so it is not visible to the migration scripts' direct `clickhouse-client` session — + they cannot gate on it. Confirm it took effect on the prod-clone/staging load test (this is the Go/No-Go "Async-insert + ceiling confirmed" item), before the write-facing stages run in production. **Also confirm client/SDK insert timeouts + exceed the widened buffer** (~10s) — with `wait_for_async_insert=1` a raised ceiling blocks each insert until it + flushes, so a shorter client timeout would surface as ingestion errors during the window. 7. **Schema-state flag wired, with a rollout plan** — `databaseAnalyticsDataModel.traceColumnsNonNullable` (env `ANALYTICS_DB_DATA_MODEL_TRACE_COLUMNS_NON_NULLABLE`, default `false`). The successor's `end_time`/`ttft` are **non-nullable sentinel** columns, so the app must bind epoch/NaN instead of `null` once they are live — a `null` diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_b_exchange_back.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_b_exchange_back.sql index a0a584decfe..eb82fb5175e 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_b_exchange_back.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_b_exchange_back.sql @@ -1,18 +1,15 @@ -- runbook traces-local-v2-cutover — ROLLBACK stage B: swap the tables back (driven by ../rollback.sh --stage B) -- -- Use when the EXCHANGE ran but the wrap did NOT. `traces` holds the successor data and `traces_pre_cutover_backup` --- parks the original. Swap them back (so `traces` is the original live again), then rename the now-parked successor --- back to `traces_local_v2` — restoring the canonical state (traces = original live, traces_local_v2 = successor --- parked), identical to pre-EXCHANGE. Non-destructive. rollback.sh runs the reverse-replay --- (000004_rollback_reverse_replay.sql) right after this so deletes since cutover_start do not resurrect. rollback.sh --- asserts the post-EXCHANGE, pre-wrap topology (traces = successor schema, not Distributed) before running it. --- --- These two statements are NOT jointly atomic (unlike stage C's single multi-target RENAME). The EXCHANGE already makes --- live `traces` the original again; if the following RENAME fails (e.g. a transient per-replica ON CLUSTER DDL error), --- the abandoned successor is left under `traces_pre_cutover_backup` instead of the canonical `traces_local_v2`, and --- rollback.sh cannot re-drive it (stage B/A both abort on that split state). RECOVERY: live data is already correct; --- only the relabel remains — run it manually: --- RENAME TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces_pre_cutover_backup TO ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 ON CLUSTER '{cluster}'; -EXCHANGE TABLES ${ANALYTICS_DB_DATABASE_NAME}.traces AND ${ANALYTICS_DB_DATABASE_NAME}.traces_pre_cutover_backup ON CLUSTER '{cluster}'; - -RENAME TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces_pre_cutover_backup TO ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 ON CLUSTER '{cluster}'; +-- parks the original. A SINGLE atomic multi-target RENAME rotates both names back: the successor (`traces`) returns to +-- `traces_local_v2`, and the original (`traces_pre_cutover_backup`) returns to `traces` (the name freed by the first +-- clause) — restoring the canonical state (traces = original live, traces_local_v2 = successor parked), identical to +-- pre-EXCHANGE. Gapless and with no orphan risk: because it is one atomic statement, there is no window where a partial +-- failure could strand the successor under the backup name (the flaw of a separate EXCHANGE + RENAME). Non-destructive. +-- rollback.sh runs the reverse-replay (000004_rollback_reverse_replay.sql) right after this so deletes since +-- cutover_start do not resurrect. rollback.sh asserts the post-EXCHANGE, pre-wrap topology (traces = successor schema, +-- not Distributed) before running it. +RENAME TABLE + ${ANALYTICS_DB_DATABASE_NAME}.traces TO ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2, + ${ANALYTICS_DB_DATABASE_NAME}.traces_pre_cutover_backup TO ${ANALYTICS_DB_DATABASE_NAME}.traces + ON CLUSTER '{cluster}'; diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java index eccb6af4ece..310070a3799 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java @@ -876,15 +876,19 @@ private void rollbackDiscardShadow() { } /** - * Rollback stage B (000004_rollback_stage_b + reverse_replay): swap the original (parked as - * {@code traces_pre_cutover_backup}) back to live {@code traces}, rename the now-parked successor back to - * {@code traces_local_v2}, then reverse-replay so a delete on the successor since {@code cutoverStart} does not - * resurrect on the restored original. + * Rollback stage B (000004_rollback_stage_b + reverse_replay): a single atomic multi-target RENAME rotates both + * names back — the successor ({@code traces}) returns to {@code traces_local_v2} and the original + * ({@code traces_pre_cutover_backup}) returns to {@code traces} (the name freed by the first clause) — so there is no + * window where a partial failure strands the successor under the backup name. Then reverse-replay so a delete on the + * successor since {@code cutoverStart} does not resurrect on the restored original. */ private void rollbackExchangeBack(String cutoverStart) { - execute("EXCHANGE TABLES traces AND traces_pre_cutover_backup ON CLUSTER '{cluster}'", _ -> { - }); - execute("RENAME TABLE traces_pre_cutover_backup TO traces_local_v2 ON CLUSTER '{cluster}'", _ -> { + execute(""" + RENAME TABLE + traces TO traces_local_v2, + traces_pre_cutover_backup TO traces + ON CLUSTER '{cluster}' + """, _ -> { }); reverseReplay(cutoverStart); } From fdf8b7b6ca3d1f3f217d0394103cf45bcec16eba Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Fri, 24 Jul 2026 16:56:24 +0200 Subject: [PATCH 11/28] test(cutover): make the Distributed wrap gapless per node The wrap was RENAME traces->traces_local then CREATE Distributed traces, leaving `traces` absent in between. Replace with the validated rename-rotate: pre-build the Distributed wrapper under a temp name (traces_dist; its traces_local target need not exist yet), then one atomic multi-target RENAME rotates the data to traces_local and the wrapper into traces (the name freed by the first clause). So `traces` transitions MergeTree->Distributed with no absent window on a node. A cross-node ON CLUSTER propagation skew still exists, so --wrap-only keeps requiring --confirm-maintenance (comments/README updated to reflect per-node gapless + the remaining cross-node window). resetTables drops a leaked traces_dist (a test dying between the CREATE and the RENAME). Gate test 8/8 green on CH 26.3. Co-Authored-By: Claude Opus 4.8 --- .../traces-local-v2-cutover/README.md | 16 ++++++++------- .../000003_exchange_and_wrap.sql | 20 +++++++++++++++---- .../scripts/exchange_and_wrap.sh | 20 ++++++++++--------- .../TracesLocalV2CutoverTest.java | 19 ++++++++++++++---- 4 files changed, 51 insertions(+), 24 deletions(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md index 157f4528547..ffe42dc116c 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md @@ -158,14 +158,16 @@ new table before the EXCHANGE. The replay matches the **full key**, not `id` alo > ships. The wrap is the sharding-readiness layer, not the cutover. > > **Applying the deferred wrap later:** once the sharding-aware DAO has shipped, run -> `exchange_and_wrap.sh --database opik --wrap-only` — it runs the settle gate and applies **only** the `RENAME` + -> `Distributed` wrap on the already-swapped `traces` (no second EXCHANGE, no new `cutover_start`). To roll the wrap -> back, use `rollback.sh --stage C`. +> `exchange_and_wrap.sh --database opik --wrap-only --confirm-maintenance` — it runs the settle gate and applies **only** +> the wrap on the already-swapped `traces` (no second EXCHANGE, no new `cutover_start`). To roll the wrap back, use +> `rollback.sh --stage C`. > -> The wrap is a non-atomic `RENAME`→`CREATE`, so `traces` is **briefly absent** (INSERT/SELECT fail with "Table doesn't -> exist" during that window; `ON CLUSTER` widens it per node). The same-run path is covered by the still-raised EXCHANGE -> buffer; for the deferred `--wrap-only` path, **re-raise `asyncInsertBusyTimeoutMaxMs` (or quiesce ingestion / take a -> maintenance window) first** so the wrap runs under the same buffered conditions. +> The wrap is **gapless per node**: it pre-builds the `Distributed` wrapper under a temp name, then one atomic +> multi-target `RENAME` rotates the data to `traces_local` and the wrapper into `traces`, so `traces` is never absent on +> a node. A brief **cross-node** `ON CLUSTER` propagation skew still exists (as for any `ON CLUSTER` DDL), during which a +> Distributed query could route to a not-yet-created `traces_local` on a lagging node — so the deferred `--wrap-only` +> path still **requires `--confirm-maintenance`** (re-raise `asyncInsertBusyTimeoutMaxMs` / quiesce ingestion / take a +> maintenance window). The same-run `--with-wrap` path is already covered by the still-raised EXCHANGE buffer. > > **Wrap flags** (`exchange_and_wrap.sh`, mutually exclusive; default is EXCHANGE-only): omit them (or pass > `--skip-wrap`, an explicit alias) to run the EXCHANGE and stop; `--with-wrap` to also apply the wrap in the same run; diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000003_exchange_and_wrap.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000003_exchange_and_wrap.sql index e89fc94673c..33b4d0882bb 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000003_exchange_and_wrap.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000003_exchange_and_wrap.sql @@ -20,7 +20,7 @@ RENAME TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 TO ${ANALYTICS_DB_DAT -- >>> END exchange -- >>> BEGIN wrap --- Sharding-ready wrap: rename the partitioned table to *_local and front it with a Distributed table keyed on +-- Sharding-ready wrap: move the partitioned table under *_local and front it with a Distributed table keyed on -- sipHash64(project_id). Transparent on a single shard; switching on sharding later is config-only. The {cluster} macro -- (not the literal 'cluster') keeps the DDL portable; it is resolved server-side. -- HARD PREREQUISITE: a Distributed table supports SELECT and INSERT but NOT mutations — a lightweight DELETE returns @@ -28,10 +28,22 @@ RENAME TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 TO ${ANALYTICS_DB_DAT -- (code 48). So the product's delete-by-id AND retention deletes both break the moment this wrap is applied. Do NOT run -- the wrap until those DAO paths target `traces_local` (see README "The Distributed wrap"). The EXCHANGE above is the -- data cutover and leaves `traces` a MergeTree where deletes still work; the wrap is a separate, gated step. -RENAME TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces TO ${ANALYTICS_DB_DATABASE_NAME}.traces_local ON CLUSTER '{cluster}'; - -CREATE TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces ON CLUSTER '{cluster}' AS ${ANALYTICS_DB_DATABASE_NAME}.traces_local +-- +-- GAPLESS per node: build the Distributed wrapper under a temp name FIRST (its 'traces_local' target need not exist +-- yet — Distributed resolves it lazily), then a SINGLE atomic multi-target RENAME rotates the data to `traces_local` +-- and the wrapper into `traces` (the name freed by the first clause). So `traces` transitions MergeTree->Distributed +-- with no window where the name is absent — unlike a RENAME-then-CREATE, which leaves `traces` missing in between. +-- (A cross-node ON CLUSTER propagation skew still exists, as for any ON CLUSTER DDL; the driver's --confirm-maintenance +-- gate covers it.) Partial-failure recovery: if the RENAME fails after the CREATE, `traces` is untouched (still the +-- successor MergeTree, live) and only the temp wrapper lingers — drop it and retry: +-- DROP TABLE IF EXISTS ${ANALYTICS_DB_DATABASE_NAME}.traces_dist ON CLUSTER '{cluster}' SYNC; +CREATE TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces_dist ON CLUSTER '{cluster}' AS ${ANALYTICS_DB_DATABASE_NAME}.traces ENGINE = Distributed('{cluster}', '${ANALYTICS_DB_DATABASE_NAME}', 'traces_local', sipHash64(project_id)); + +RENAME TABLE + ${ANALYTICS_DB_DATABASE_NAME}.traces TO ${ANALYTICS_DB_DATABASE_NAME}.traces_local, + ${ANALYTICS_DB_DATABASE_NAME}.traces_dist TO ${ANALYTICS_DB_DATABASE_NAME}.traces + ON CLUSTER '{cluster}'; -- >>> END wrap -- After the wrap: restore the buffer ceiling (unset asyncInsertBusyTimeoutMaxMs), verify (README "Verifying the diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh index 9dbb9043a27..bee9d1b97af 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh @@ -29,10 +29,11 @@ # --force skip the replication-settle gate. By default the swap aborts while any replica still # has replication-queue backlog or an unfinished mutation on traces / traces_local_v2, since a # behind replica would swap in an incomplete table. Use only if settlement is confirmed out of band. -# --confirm-maintenance REQUIRED with --wrap-only. The wrap is a non-atomic RENAME->CREATE that briefly makes `traces` -# unavailable; unlike the same-run --with-wrap path (still buffered from the EXCHANGE), --wrap-only -# runs later against live, unbuffered ingestion. This flag asserts the async-insert buffer is -# re-raised (or ingestion quiesced / a maintenance window is in effect); the wrap refuses without it. +# --confirm-maintenance REQUIRED with --wrap-only. The wrap is gapless per node (atomic rotate), but a brief cross-node +# ON CLUSTER propagation skew remains, during which a Distributed query can hit a not-yet-created +# `traces_local` on a lagging node. Unlike the same-run --with-wrap path (still buffered from the +# EXCHANGE), --wrap-only runs later against live, unbuffered ingestion. This flag asserts the +# async-insert buffer is re-raised (or ingestion quiesced / a maintenance window is in effect). set -euo pipefail @@ -66,12 +67,13 @@ done if (( SKIP_WRAP + WITH_WRAP + WRAP_ONLY > 1 )); then echo "ERROR: --skip-wrap, --with-wrap and --wrap-only are mutually exclusive" >&2; exit 2 fi -# The deferred wrap briefly makes `traces` unavailable (non-atomic RENAME->CREATE) and runs against live, unbuffered -# ingestion — unlike the same-run --with-wrap path, which is still buffered from the EXCHANGE. Refuse (fail fast, before -# touching ClickHouse) unless the operator asserts the buffer is re-raised / ingestion quiesced / a maintenance window. +# The deferred wrap is gapless per node but has a brief cross-node ON CLUSTER propagation skew, and --wrap-only runs +# against live, unbuffered ingestion — unlike the same-run --with-wrap path, still buffered from the EXCHANGE. Refuse +# (fail fast, before touching ClickHouse) unless the operator asserts the buffer is re-raised / ingestion quiesced / a +# maintenance window is in effect. if [[ "$WRAP_ONLY" == "1" && "$CONFIRM_MAINTENANCE" != "1" ]]; then echo "ERROR: --wrap-only requires --confirm-maintenance. Re-raise asyncInsertBusyTimeoutMaxMs (or quiesce ingestion /" >&2 - echo " take a maintenance window) first — the wrap briefly makes 'traces' unavailable — then re-run with it." >&2 + echo " take a maintenance window) first — the wrap has a brief cross-node window — then re-run with it." >&2 exit 2 fi @@ -204,7 +206,7 @@ if [[ "$WRAP_ONLY" == "1" ]]; then # window per-node). The same-run path is covered by the still-raised EXCHANGE buffer, but --wrap-only runs later # against live, unbuffered ingestion. PRECONDITION: re-raise databaseAnalytics.asyncInsertBusyTimeoutMaxMs (or quiesce # ingestion / assert a maintenance window) so the wrap runs under the same buffered conditions as the EXCHANGE. - # --confirm-maintenance was already enforced up front (the wrap briefly makes `traces` unavailable). + # --confirm-maintenance was already enforced up front (gapless per node, but a brief cross-node ON CLUSTER window). run_block wrap echo "Distributed wrap done: 'traces' fronts 'traces_local' via sipHash64(project_id). (EXCHANGE was a prior step.)" exit 0 diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java index 310070a3799..d1ec523050a 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java @@ -250,7 +250,10 @@ void resetTables() { } } // 4. Canonical now; truncate the two tables and clear any residual artifacts (IF EXISTS so a genuinely - // unrecoverable partial state still cannot throw here). + // unrecoverable partial state still cannot throw here). traces_dist is the temp wrapper the gapless wrap + // builds before its atomic rename — a leak only if a test died between the CREATE and the RENAME. + execute("DROP TABLE IF EXISTS traces_dist ON CLUSTER '{cluster}' SYNC", _ -> { + }); execute("DROP TABLE IF EXISTS traces_local ON CLUSTER '{cluster}' SYNC", _ -> { }); execute("DROP TABLE IF EXISTS traces_pre_cutover_backup ON CLUSTER '{cluster}' SYNC", _ -> { @@ -860,13 +863,21 @@ private void exchangeTables() { }); } + // Gapless wrap (000003 wrap block): build the Distributed wrapper under a temp name first (its 'traces_local' target + // need not exist yet), then one atomic multi-target RENAME rotates the data to traces_local and the wrapper into + // traces (the name freed by the first clause), so traces is never absent on a node. private void wrapInDistributed() { - execute("RENAME TABLE traces TO traces_local ON CLUSTER '{cluster}'", _ -> { - }); - execute(("CREATE TABLE traces ON CLUSTER '{cluster}' AS traces_local " + execute(("CREATE TABLE traces_dist ON CLUSTER '{cluster}' AS traces " + "ENGINE = Distributed('{cluster}', '" + DATABASE_NAME + "', 'traces_local', sipHash64(project_id))"), _ -> { }); + execute(""" + RENAME TABLE + traces TO traces_local, + traces_dist TO traces + ON CLUSTER '{cluster}' + """, _ -> { + }); } /** Rollback stage A (000004_rollback_stage_a): discard the disposable shadow; the live `traces` is untouched. */ From ee97d5f14d3676c03e3af69f3d914da8ff35d34a Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Fri, 24 Jul 2026 17:04:47 +0200 Subject: [PATCH 12/28] test(cutover): stage-C rollback drops the ex-wrapper by an unambiguous temp name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage C did EXCHANGE traces<->backup then DROP traces_pre_cutover_backup — a DROP of a name that a data-bearing table had just occupied, which reads as unsafe even though ON CLUSTER DDL ordering makes it correct. Replace with a single atomic 3-way RENAME rotate (traces -> traces_dist_old, traces_pre_cutover_backup -> traces, traces_local -> traces_local_v2), then DROP traces_dist_old — a fresh name only the data-less Distributed wrapper ever held, so the DROP structurally cannot hit the original data regardless of replica timing. Still gapless. resetTables also clears a leaked traces_dist_old; gate test 8/8 green on CH 26.3. Also append --confirm-maintenance to the deferred-wrap guidance echo so the printed command is copy-paste complete. Co-Authored-By: Claude Opus 4.8 --- ...0004_rollback_stage_c_promote_original.sql | 29 +++++++++++-------- .../scripts/exchange_and_wrap.sh | 2 +- .../TracesLocalV2CutoverTest.java | 27 +++++++++++------ 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql index c61b44217dc..f47f8f8e492 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql @@ -1,20 +1,25 @@ -- runbook traces-local-v2-cutover — ROLLBACK stage C: promote the original back (driven by ../rollback.sh --stage C) -- -- Use when the wrap ran. Post-wrap topology: `traces` is a Distributed wrapper over `traces_local` (successor data); --- `traces_pre_cutover_backup` parks the original. Promote the original back to `traces` GAPLESSLY: EXCHANGE swaps the --- names atomically, so `traces` becomes the original MergeTree with no window where the name is absent (unlike a --- DROP-then-RENAME). The wrapper (now parked under the backup name) holds no data, so it is dropped next, and the --- successor shard is parked back under `traces_local_v2` — ending in the canonical state (traces = original live, --- traces_local_v2 = successor parked), with no leftover names. +-- `traces_pre_cutover_backup` parks the original. Promote the original back to `traces` GAPLESSLY with a single atomic +-- multi-target RENAME that rotates all three names at once: the data-less Distributed wrapper (`traces`) moves to an +-- explicit temp name, the original (`traces_pre_cutover_backup`) becomes live `traces` (the name freed by the first +-- clause), and the successor shard (`traces_local`) parks back under `traces_local_v2` — ending in the canonical state +-- (traces = original live, traces_local_v2 = successor parked). `traces` is never absent on a node. +-- +-- Then drop the ex-wrapper. It is dropped under `traces_dist_old` — a fresh name that ONLY the data-less wrapper ever +-- occupied — so the DROP cannot hit the original data regardless of per-replica DDL timing (the concern with dropping a +-- name that a data-bearing table previously used). -- -- rollback.sh runs the reverse-replay (000004_rollback_reverse_replay.sql) right after this so deletes since -- cutover_start do not resurrect, and asserts the post-wrap topology (traces = Distributed) before running it. --- 1. Gapless promote: `traces` immediately becomes the original MergeTree; the Distributed wrapper moves to the backup name. -EXCHANGE TABLES ${ANALYTICS_DB_DATABASE_NAME}.traces AND ${ANALYTICS_DB_DATABASE_NAME}.traces_pre_cutover_backup ON CLUSTER '{cluster}'; - --- 2. Drop the now data-less Distributed wrapper (routing definition only — no size guard needed). -DROP TABLE IF EXISTS ${ANALYTICS_DB_DATABASE_NAME}.traces_pre_cutover_backup ON CLUSTER '{cluster}' SYNC; +-- 1. Gapless promote: rotate all three names atomically. +RENAME TABLE + ${ANALYTICS_DB_DATABASE_NAME}.traces TO ${ANALYTICS_DB_DATABASE_NAME}.traces_dist_old, + ${ANALYTICS_DB_DATABASE_NAME}.traces_pre_cutover_backup TO ${ANALYTICS_DB_DATABASE_NAME}.traces, + ${ANALYTICS_DB_DATABASE_NAME}.traces_local TO ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 + ON CLUSTER '{cluster}'; --- 3. Park the successor shard back under its canonical name. -RENAME TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces_local TO ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 ON CLUSTER '{cluster}'; +-- 2. Drop the ex-wrapper by its unambiguous temp name (data-less Distributed routing definition — no size guard needed). +DROP TABLE IF EXISTS ${ANALYTICS_DB_DATABASE_NAME}.traces_dist_old ON CLUSTER '{cluster}' SYNC; diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh index bee9d1b97af..02ff3c8f220 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh @@ -223,7 +223,7 @@ if [[ "$WITH_WRAP" == "1" ]]; then echo "Distributed wrap done: 'traces' fronts 'traces_local' via sipHash64(project_id)." else echo "Distributed wrap deferred (default). Deletes still work on the MergeTree 'traces'. Apply the wrap later with" - echo "--wrap-only once the delete/read DAOs target traces_local." + echo "'--wrap-only --confirm-maintenance' once the delete/read DAOs target traces_local." fi echo "Restore databaseAnalytics.asyncInsertBusyTimeoutMaxMs to default, verify, and keep traces_pre_cutover_backup for the soak." diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java index d1ec523050a..86782244b11 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java @@ -250,10 +250,13 @@ void resetTables() { } } // 4. Canonical now; truncate the two tables and clear any residual artifacts (IF EXISTS so a genuinely - // unrecoverable partial state still cannot throw here). traces_dist is the temp wrapper the gapless wrap - // builds before its atomic rename — a leak only if a test died between the CREATE and the RENAME. + // unrecoverable partial state still cannot throw here). traces_dist / traces_dist_old are the temp wrapper + // names the gapless wrap and stage-C rollback use around their atomic renames — leaks only if a test died + // between a CREATE/RENAME and the following statement. execute("DROP TABLE IF EXISTS traces_dist ON CLUSTER '{cluster}' SYNC", _ -> { }); + execute("DROP TABLE IF EXISTS traces_dist_old ON CLUSTER '{cluster}' SYNC", _ -> { + }); execute("DROP TABLE IF EXISTS traces_local ON CLUSTER '{cluster}' SYNC", _ -> { }); execute("DROP TABLE IF EXISTS traces_pre_cutover_backup ON CLUSTER '{cluster}' SYNC", _ -> { @@ -906,16 +909,22 @@ private void rollbackExchangeBack(String cutoverStart) { /** * Rollback stage C (000004_rollback_stage_c + reverse_replay): promote the parked original back to {@code traces} - * GAPLESSLY — EXCHANGE swaps it in atomically (no window where {@code traces} is absent), then the now-data-less - * Distributed wrapper (parked under the backup name) is dropped and the successor shard is parked back under - * {@code traces_local_v2}, ending in the canonical state with no leftover names. Then reverse-replay. + * GAPLESSLY with a single atomic multi-target RENAME that rotates all three names — the data-less wrapper + * ({@code traces}) to an explicit temp name, the original ({@code traces_pre_cutover_backup}) to live {@code traces} + * (the name freed by the first clause), and the successor shard to {@code traces_local_v2}. Then the ex-wrapper is + * dropped under its temp name {@code traces_dist_old} — a name only the data-less wrapper ever held, so the DROP + * cannot hit the original data regardless of replica timing. Then reverse-replay. */ private void rollbackAfterWrap(String cutoverStart) { - execute("EXCHANGE TABLES traces AND traces_pre_cutover_backup ON CLUSTER '{cluster}'", _ -> { - }); - execute("DROP TABLE IF EXISTS traces_pre_cutover_backup ON CLUSTER '{cluster}' SYNC", _ -> { + execute(""" + RENAME TABLE + traces TO traces_dist_old, + traces_pre_cutover_backup TO traces, + traces_local TO traces_local_v2 + ON CLUSTER '{cluster}' + """, _ -> { }); - execute("RENAME TABLE traces_local TO traces_local_v2 ON CLUSTER '{cluster}'", _ -> { + execute("DROP TABLE IF EXISTS traces_dist_old ON CLUSTER '{cluster}' SYNC", _ -> { }); reverseReplay(cutoverStart); } From c7b4eed8b03322b55fc160a7166a19b67a4b053d Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Fri, 24 Jul 2026 17:36:10 +0200 Subject: [PATCH 13/28] test(cutover): enforce wrap prerequisites, tag DDL, correct buffer-coverage claim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - exchange_and_wrap.sh: require --confirm-daos-retargeted whenever the wrap is applied (--with-wrap or --wrap-only) — the OPIK-7455 hard prerequisite, else DELETE FROM the Distributed traces returns 500. Fail fast, before ClickHouse. - --wrap-only now also asserts traces_pre_cutover_backup exists, so it can't wrap into a state where stage C rollback has nothing to restore (one-way cutover). - Tag the exchange/wrap DDL (000003) and all rollback stages (000004 A/B/C + reverse-replay) with log_comment via a leading SET (reliable for DDL), for cutover attribution in system.query_log. - README: correct the "already covered" claim — the same-run --with-wrap path shares the brief cross-node ON CLUSTER window; the buffer reduces but does not eliminate INSERT exposure, and SELECTs are unbuffered. Driver-only guards + reference-SQL tags (not exercised by the gate test's embedded SQL); to be exercised in the post-merge local rehearsal. Co-Authored-By: Claude Opus 4.8 --- .../traces-local-v2-cutover/README.md | 5 ++++- .../000003_exchange_and_wrap.sql | 4 ++++ .../000004_rollback_reverse_replay.sql | 3 ++- ...000004_rollback_stage_a_discard_shadow.sql | 1 + .../000004_rollback_stage_b_exchange_back.sql | 1 + ...0004_rollback_stage_c_promote_original.sql | 1 + .../scripts/exchange_and_wrap.sh | 19 +++++++++++++++++++ 7 files changed, 32 insertions(+), 2 deletions(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md index ffe42dc116c..90a323a8fdc 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md @@ -167,7 +167,10 @@ new table before the EXCHANGE. The replay matches the **full key**, not `id` alo > a node. A brief **cross-node** `ON CLUSTER` propagation skew still exists (as for any `ON CLUSTER` DDL), during which a > Distributed query could route to a not-yet-created `traces_local` on a lagging node — so the deferred `--wrap-only` > path still **requires `--confirm-maintenance`** (re-raise `asyncInsertBusyTimeoutMaxMs` / quiesce ingestion / take a -> maintenance window). The same-run `--with-wrap` path is already covered by the still-raised EXCHANGE buffer. +> maintenance window). The same-run `--with-wrap` path **shares that cross-node window** — the still-raised EXCHANGE +> buffer parks INSERTs (reducing, not eliminating, the exposure to a size-triggered flush routed at a not-yet-created +> `traces_local`), and SELECTs are not buffered — so the brief skew is an accepted cost of the cutover window either way, +> not something the buffer fully covers. > > **Wrap flags** (`exchange_and_wrap.sh`, mutually exclusive; default is EXCHANGE-only): omit them (or pass > `--skip-wrap`, an explicit alias) to run the EXCHANGE and stop; `--with-wrap` to also apply the wrap in the same run; diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000003_exchange_and_wrap.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000003_exchange_and_wrap.sql index 33b4d0882bb..4d220ef72f3 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000003_exchange_and_wrap.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000003_exchange_and_wrap.sql @@ -14,6 +14,9 @@ -- not the "v2" successor (rationale: README "Naming and the parked backup"). Requires an Atomic database (default). If -- the Liquibase ClickHouse extension cannot execute EXCHANGE ON CLUSTER in the downtime-based path, use the fallback -- RENAME sequence in the README instead. +-- log_comment tags these DDL statements in system.query_log for cutover attribution (DDL takes it via a leading SET, +-- not a trailing SETTINGS clause). +SET log_comment = 'traces_local_v2_cutover:exchange'; EXCHANGE TABLES ${ANALYTICS_DB_DATABASE_NAME}.traces AND ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 ON CLUSTER '{cluster}'; RENAME TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 TO ${ANALYTICS_DB_DATABASE_NAME}.traces_pre_cutover_backup ON CLUSTER '{cluster}'; @@ -37,6 +40,7 @@ RENAME TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 TO ${ANALYTICS_DB_DAT -- gate covers it.) Partial-failure recovery: if the RENAME fails after the CREATE, `traces` is untouched (still the -- successor MergeTree, live) and only the temp wrapper lingers — drop it and retry: -- DROP TABLE IF EXISTS ${ANALYTICS_DB_DATABASE_NAME}.traces_dist ON CLUSTER '{cluster}' SYNC; +SET log_comment = 'traces_local_v2_cutover:wrap'; CREATE TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces_dist ON CLUSTER '{cluster}' AS ${ANALYTICS_DB_DATABASE_NAME}.traces ENGINE = Distributed('{cluster}', '${ANALYTICS_DB_DATABASE_NAME}', 'traces_local', sipHash64(project_id)); diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_reverse_replay.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_reverse_replay.sql index 67a6ca52bb8..6573f710d1d 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_reverse_replay.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_reverse_replay.sql @@ -34,4 +34,5 @@ OR (workspace_id, id) IN ( -- lightweight_deletes_sync = 2: wait for the mutation on every replica so the restored `traces` is consistent -- cluster-wide before the rollback is declared done (see 000002 for the rationale). SETTINGS allow_nondeterministic_mutations = 1, - lightweight_deletes_sync = 2; + lightweight_deletes_sync = 2, + log_comment = 'traces_local_v2_rollback:reverse_replay'; diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_a_discard_shadow.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_a_discard_shadow.sql index f11db87f16a..075e2a29019 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_a_discard_shadow.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_a_discard_shadow.sql @@ -14,4 +14,5 @@ -- max_table_size_to_drop = 0 disables the drop-size guard for this statement: TRUNCATE is subject to -- max_table_size_to_drop (default 50 GB) just like DROP, and the shadow is well over that on any instance large enough -- to need this runbook — without the override the rollback throws exactly when it is needed. +SET log_comment = 'traces_local_v2_rollback:stage_a'; TRUNCATE TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 ON CLUSTER '{cluster}' SETTINGS max_table_size_to_drop = 0; diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_b_exchange_back.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_b_exchange_back.sql index eb82fb5175e..9b5eef81315 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_b_exchange_back.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_b_exchange_back.sql @@ -9,6 +9,7 @@ -- rollback.sh runs the reverse-replay (000004_rollback_reverse_replay.sql) right after this so deletes since -- cutover_start do not resurrect. rollback.sh asserts the post-EXCHANGE, pre-wrap topology (traces = successor schema, -- not Distributed) before running it. +SET log_comment = 'traces_local_v2_rollback:stage_b'; RENAME TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces TO ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2, ${ANALYTICS_DB_DATABASE_NAME}.traces_pre_cutover_backup TO ${ANALYTICS_DB_DATABASE_NAME}.traces diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql index f47f8f8e492..83312b0b82d 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql @@ -15,6 +15,7 @@ -- cutover_start do not resurrect, and asserts the post-wrap topology (traces = Distributed) before running it. -- 1. Gapless promote: rotate all three names atomically. +SET log_comment = 'traces_local_v2_rollback:stage_c'; RENAME TABLE ${ANALYTICS_DB_DATABASE_NAME}.traces TO ${ANALYTICS_DB_DATABASE_NAME}.traces_dist_old, ${ANALYTICS_DB_DATABASE_NAME}.traces_pre_cutover_backup TO ${ANALYTICS_DB_DATABASE_NAME}.traces, diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh index 02ff3c8f220..e7c6ba36731 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh @@ -34,6 +34,10 @@ # `traces_local` on a lagging node. Unlike the same-run --with-wrap path (still buffered from the # EXCHANGE), --wrap-only runs later against live, unbuffered ingestion. This flag asserts the # async-insert buffer is re-raised (or ingestion quiesced / a maintenance window is in effect). +# --confirm-daos-retargeted REQUIRED whenever the wrap is applied (--with-wrap or --wrap-only). Asserts the trace +# delete/mutation DAOs already target `traces_local` (OPIK-7455) — a Distributed `traces` rejects +# mutations, so without the retarget delete-by-id and retention deletes return 500 the moment the +# wrap lands. The script cannot inspect backend code, so the operator must assert it. set -euo pipefail @@ -46,6 +50,7 @@ WITH_WRAP=0 WRAP_ONLY=0 FORCE=0 CONFIRM_MAINTENANCE=0 +CONFIRM_DAOS_RETARGETED=0 while [[ $# -gt 0 ]]; do case "$1" in @@ -55,6 +60,7 @@ while [[ $# -gt 0 ]]; do --wrap-only) WRAP_ONLY=1; shift ;; --force) FORCE=1; shift ;; --confirm-maintenance) CONFIRM_MAINTENANCE=1; shift ;; + --confirm-daos-retargeted) CONFIRM_DAOS_RETARGETED=1; shift ;; *) echo "Unknown argument: $1" >&2; exit 2 ;; esac done @@ -76,6 +82,14 @@ if [[ "$WRAP_ONLY" == "1" && "$CONFIRM_MAINTENANCE" != "1" ]]; then echo " take a maintenance window) first — the wrap has a brief cross-node window — then re-run with it." >&2 exit 2 fi +# HARD PREREQUISITE (OPIK-7455): a Distributed table rejects mutations, so once the wrap is applied the product's +# DELETE_BY_ID and retention deletes return 500 against `traces` unless those DAO paths already target `traces_local`. +# The script can't inspect backend code, so any wrap-applying mode must assert it. Fail fast, before touching ClickHouse. +if [[ ( "$WITH_WRAP" == "1" || "$WRAP_ONLY" == "1" ) && "$CONFIRM_DAOS_RETARGETED" != "1" ]]; then + echo "ERROR: applying the wrap requires --confirm-daos-retargeted. The trace delete/mutation DAOs must target" >&2 + echo " 'traces_local' (OPIK-7455) before 'traces' becomes Distributed, or deletes/retention break at runtime." >&2 + exit 2 +fi ch() { clickhouse-client --database "$DATABASE" --query "$1" @@ -142,6 +156,11 @@ assert_pre_wrap_topology() { echo " clickhouse-client --database $DATABASE --query \"RENAME TABLE $DATABASE.traces_local_v2 TO $DATABASE.traces_pre_cutover_backup ON CLUSTER '{cluster}'\"" >&2 exit 1 fi + if [[ -z "$(traces_engine traces_pre_cutover_backup)" ]]; then + echo "ERROR: --wrap-only: 'traces_pre_cutover_backup' (the parked original) does not exist — stage C rollback would" >&2 + echo " have nothing to restore, making the wrap one-way. Refusing. (Did finalize.sh already drop the backup?)" >&2 + exit 1 + fi } # Pre-EXCHANGE gate: the swap is metadata-only and near-instant, but each replica reads its own local parts afterwards. From 2447cc02ef15b2578f83b1cedc7fc46ec9e7fa2f Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Fri, 24 Jul 2026 18:36:13 +0200 Subject: [PATCH 14/28] test(cutover): enforce the async-insert buffer precondition on the EXCHANGE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The buffer (asyncInsertBusyTimeoutMaxMs) is the zero-loss mechanism for the forward cutover — it holds writes across the final-delta->EXCHANGE gap so they land on the new table; at the default, a write in that window can commit to the old table and be lost after the swap. It was only a documented/echoed reminder. Add a fail-fast --confirm-buffer-raised gate, required on every EXCHANGE path (default and --with-wrap; not --wrap-only, which does no EXCHANGE), matching the --confirm-maintenance / --confirm-daos-retargeted pattern. It's an operator assertion, not verification (the buffer is a backend per-query setting the migration's clickhouse-client session can't read), so the Go/No-Go load-test item remains the actual "took effect" check. README + prereq #6 + Go/No-Go and the EXCHANGE example updated to match. Co-Authored-By: Claude Opus 4.8 --- .../traces-local-v2-cutover/README.md | 17 +++++++++++------ .../scripts/exchange_and_wrap.sh | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md index 90a323a8fdc..76c979da4f5 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md @@ -70,10 +70,12 @@ new table before the EXCHANGE. The replay matches the **full key**, not `id` alo 6. **Cutover buffer knob ready** — `databaseAnalytics.asyncInsertBusyTimeoutMaxMs` (env `ANALYTICS_DB_ASYNC_INSERT_BUSY_TIMEOUT_MAX_MS`), unset by default so the buffer inherits the `async_insert_busy_timeout_max_ms=250` carried by `queryParameters`. Raise it to ~10000 for the cutover, then unset it - again. **This is operator-verified, not script-enforced:** the ceiling is a backend per-query setting applied on the - backend's own ClickHouse client, so it is not visible to the migration scripts' direct `clickhouse-client` session — - they cannot gate on it. Confirm it took effect on the prod-clone/staging load test (this is the Go/No-Go "Async-insert - ceiling confirmed" item), before the write-facing stages run in production. **Also confirm client/SDK insert timeouts + again. The ceiling is a backend per-query setting applied on the backend's own ClickHouse client, so the migration + scripts' direct `clickhouse-client` session **cannot read or verify it**. It is therefore **operator-asserted**: + `exchange_and_wrap.sh` refuses the EXCHANGE without `--confirm-buffer-raised` (a fail-fast acknowledgment gate — it + forces the operator to confirm the step, though it cannot prove the value took effect). Confirm it actually took + effect on the prod-clone/staging load test (the Go/No-Go "Async-insert ceiling confirmed" item) before production. + **Also confirm client/SDK insert timeouts exceed the widened buffer** (~10s) — with `wait_for_async_insert=1` a raised ceiling blocks each insert until it flushes, so a shorter client timeout would surface as ingestion errors during the window. 7. **Schema-state flag wired, with a rollout plan** — `databaseAnalyticsDataModel.traceColumnsNonNullable` (env @@ -142,8 +144,10 @@ new table before the EXCHANGE. The replay matches the **full key**, not `id` alo leaving `traces` a `MergeTree` where deletes still work); the `RENAME` + `Distributed` wrap runs only with `--with-wrap`. Restore the buffer ceiling and verify. ```bash - CLICKHOUSE_HOST= CLICKHOUSE_PASSWORD= ./scripts/exchange_and_wrap.sh --database opik + CLICKHOUSE_HOST= CLICKHOUSE_PASSWORD= ./scripts/exchange_and_wrap.sh --database opik --confirm-buffer-raised ``` + `--confirm-buffer-raised` is required on every EXCHANGE path: it asserts the async-insert buffer is raised so writes in + the final window survive the swap (add `--with-wrap --confirm-daos-retargeted` only once the DAOs target `traces_local`). > **HARD PREREQUISITE for the wrap (step 4, part 2): the delete/mutation DAO must target `traces_local` first.** A > `Distributed` table supports `SELECT` and `INSERT` but **not** mutations. Verified on ClickHouse 26.3: @@ -598,7 +602,8 @@ cheap (stage A); the bridge stays enabled so nothing is lost on a retry. - [ ] **`EXCHANGE TABLES ... ON CLUSTER` works end-to-end** — or the fallback `RENAME` sequence is documented for the variant that needs it. - [ ] **Async-insert ceiling confirmed** — raising `asyncInsertBusyTimeoutMaxMs` demonstrably widens the adaptive buffer - under load, not just the cap. + under load, not just the cap. `exchange_and_wrap.sh` enforces the acknowledgment via `--confirm-buffer-raised`, but + that is an assertion only — this checklist item is the actual "it took effect under load" verification. - [ ] **Data Retention confirmed disabled** for the cutover window (`RETENTION_ENABLED=false`). - [ ] **Reconciliation clean** — per-window source/dest counts within 0.01% across the whole backfill. - [ ] **Replication settled before the EXCHANGE** — `replication_queue` empty and the deletion-replay mutation diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh index e7c6ba36731..7a1040ea31c 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh @@ -38,6 +38,11 @@ # delete/mutation DAOs already target `traces_local` (OPIK-7455) — a Distributed `traces` rejects # mutations, so without the retarget delete-by-id and retention deletes return 500 the moment the # wrap lands. The script cannot inspect backend code, so the operator must assert it. +# --confirm-buffer-raised REQUIRED for every EXCHANGE path (the default and --with-wrap; not --wrap-only). Asserts the +# async-insert buffer (asyncInsertBusyTimeoutMaxMs) is raised on every backend instance — it holds +# writes across the swap so they land on the new table; at the default, a write in the final window +# can commit to the old table and be lost after the EXCHANGE. It's a backend setting the script can't +# read, so the operator must assert it. set -euo pipefail @@ -51,6 +56,7 @@ WRAP_ONLY=0 FORCE=0 CONFIRM_MAINTENANCE=0 CONFIRM_DAOS_RETARGETED=0 +CONFIRM_BUFFER_RAISED=0 while [[ $# -gt 0 ]]; do case "$1" in @@ -61,6 +67,7 @@ while [[ $# -gt 0 ]]; do --force) FORCE=1; shift ;; --confirm-maintenance) CONFIRM_MAINTENANCE=1; shift ;; --confirm-daos-retargeted) CONFIRM_DAOS_RETARGETED=1; shift ;; + --confirm-buffer-raised) CONFIRM_BUFFER_RAISED=1; shift ;; *) echo "Unknown argument: $1" >&2; exit 2 ;; esac done @@ -90,6 +97,17 @@ if [[ ( "$WITH_WRAP" == "1" || "$WRAP_ONLY" == "1" ) && "$CONFIRM_DAOS_RETARGETE echo " 'traces_local' (OPIK-7455) before 'traces' becomes Distributed, or deletes/retention break at runtime." >&2 exit 2 fi +# The EXCHANGE is the zero-loss step: writes in the final-delta -> EXCHANGE gap must be held by the raised async-insert +# buffer (asyncInsertBusyTimeoutMaxMs) so they flush onto the new table after the swap; if the buffer is at its default, +# such a write can commit to the old table just before the swap and be silently lost. The buffer is a backend per-query +# setting the script can't read, so require the operator to assert it. Applies to every EXCHANGE path (not --wrap-only, +# which does no EXCHANGE). Fail fast, before touching ClickHouse. +if [[ "$WRAP_ONLY" != "1" && "$CONFIRM_BUFFER_RAISED" != "1" ]]; then + echo "ERROR: the EXCHANGE requires --confirm-buffer-raised. Raise databaseAnalytics.asyncInsertBusyTimeoutMaxMs on" >&2 + echo " every backend instance first — it holds writes across the swap; at the default, writes in the final" >&2 + echo " window can commit to the old table and be lost after the EXCHANGE — then re-run with the flag." >&2 + exit 2 +fi ch() { clickhouse-client --database "$DATABASE" --query "$1" From dcf62084a590baf1c895eb458940beb57605ab7d Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Fri, 24 Jul 2026 19:23:32 +0200 Subject: [PATCH 15/28] =?UTF-8?q?test(cutover):=20address=20Baz=20review?= =?UTF-8?q?=20=E2=80=94=20arg=20guards,=20query=20tags,=20verify=20skip-in?= =?UTF-8?q?dex,=20rollback=20restart,=20replay=20coverage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Drivers: guard value-taking flags with ${2:?...} so a bare flag prints a clear "requires a value" error instead of a set -u unbound/shift crash; tag every ch() read (and estimate probe, exchange cutover_start, verify compare/drill-down) with --log_comment for system.query_log attribution. - 000005 verify: add use_skip_indexes_if_final=1 to the compare + drill-down FINAL queries so the created_at skip index prunes the scan on the 4 TB table. - README rollback: reverting traceColumnsNonNullable requires a rolling restart (config is a startup snapshot via toInstance); until then the app mixes sentinel and null semantics against the Nullable column. - Test: add workspaceScopedReplaySparesLiveCrossProjectRow — the empty-project replay branch must spare a live cross-project reused id (guard coverage); rename weekInstant's misleading minuteOffset param to secondOffset. 9/9 green. Co-Authored-By: Claude Opus 4.8 --- .../traces-local-v2-cutover/README.md | 10 ++-- .../scripts/backfill.sh | 20 ++++---- .../000005_verify_migration.sql | 5 +- .../scripts/delta_replay.sh | 6 +-- .../scripts/estimate.sh | 16 +++--- .../scripts/exchange_and_wrap.sh | 6 +-- .../scripts/finalize.sh | 4 +- .../scripts/rollback.sh | 8 +-- .../traces-local-v2-cutover/scripts/verify.sh | 20 ++++---- .../TracesLocalV2CutoverTest.java | 50 ++++++++++++++++++- 10 files changed, 98 insertions(+), 47 deletions(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md index 76c979da4f5..cda93600945 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md @@ -432,9 +432,13 @@ Pick the stage by how far the cutover got (`cutover_start` is the value `exchang and parks the successor under `traces_local_v2`, then the reverse replay. (Guarded: aborts unless `traces` is `Distributed`.) -After a stage B or C rollback, `traces` is the Nullable original again — **revert `traceColumnsNonNullable` to `false`** -(config, operator-owned) so the app resumes binding `null` for absent values and filters use Nullable logic. Leaving it -`true` is write-safe but would make absent-value filters/sorts use sentinel logic against a Nullable column. +After a stage B or C rollback, `traces` is the Nullable original again — **revert `traceColumnsNonNullable` to `false` +AND roll-restart every backend instance**. The flag is read from a **startup snapshot** of `OpikConfiguration` (bound via +`toInstance`), so a config change does **not** take effect until each instance restarts — exactly like the forward +rollout before the EXCHANGE. Until the restart completes, the app keeps binding sentinels (epoch/NaN) and using +sentinel-based absent-value logic against the now-Nullable column, mixing sentinel and `null` representations: not a hard +write failure, but inconsistent absent-value reads/filters/sorts. The rollback is therefore not fully complete until the +rolling restart lands on the whole fleet. **Point of no return.** The `EXCHANGE` is reversible for as long as the parked backup exists (stage B/C). Dropping that backup with `finalize.sh` is the one irreversible step, so gate it on an explicit soak: diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh index a08b0577369..aa87f9b92b0 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh @@ -74,16 +74,16 @@ MIN_WINDOW_SECONDS=60 while [[ $# -gt 0 ]]; do case "$1" in - --database) DATABASE="$2"; shift 2 ;; + --database) DATABASE="${2:?"$1 requires a value"}"; shift 2 ;; --dry-run) DRY_RUN=1; shift ;; - --from-week) FROM_WEEK="$2"; shift 2 ;; - --to-week) TO_WEEK="$2"; shift 2 ;; - --max-rows-per-insert) MAX_ROWS="$2"; shift 2 ;; - --max-insert-block-size) MAX_INSERT_BLOCK_SIZE="$2"; shift 2 ;; - --divergence) DIVERGENCE="$2"; shift 2 ;; - --pause-seconds) PAUSE_SECONDS="$2"; shift 2 ;; - --min-free-factor) MIN_FREE_FACTOR="$2"; shift 2 ;; - --state-file) STATE_FILE="$2"; shift 2 ;; + --from-week) FROM_WEEK="${2:?"$1 requires a value"}"; shift 2 ;; + --to-week) TO_WEEK="${2:?"$1 requires a value"}"; shift 2 ;; + --max-rows-per-insert) MAX_ROWS="${2:?"$1 requires a value"}"; shift 2 ;; + --max-insert-block-size) MAX_INSERT_BLOCK_SIZE="${2:?"$1 requires a value"}"; shift 2 ;; + --divergence) DIVERGENCE="${2:?"$1 requires a value"}"; shift 2 ;; + --pause-seconds) PAUSE_SECONDS="${2:?"$1 requires a value"}"; shift 2 ;; + --min-free-factor) MIN_FREE_FACTOR="${2:?"$1 requires a value"}"; shift 2 ;; + --state-file) STATE_FILE="${2:?"$1 requires a value"}"; shift 2 ;; *) echo "Unknown argument: $1" >&2; exit 2 ;; esac done @@ -106,7 +106,7 @@ done # Every query runs against the analytics database; --query keeps output scriptable (TSV, no formatting). ch() { - clickhouse-client --database "$DATABASE" --query "$1" + clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_cutover:backfill' --query "$1" } log() { diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000005_verify_migration.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000005_verify_migration.sql index cf147a3de19..53979bb608c 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000005_verify_migration.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000005_verify_migration.sql @@ -102,7 +102,8 @@ SELECT ifNull(src.h, 0) AS src_checksum, ifNull(dst.h, 0) AS dst_checksum, (src.c = dst.c AND ifNull(src.h, 0) = ifNull(dst.h, 0)) AS ok -FROM src, dst; +FROM src, dst +SETTINGS use_skip_indexes_if_final = 1; -- >>> END compare -- >>> BEGIN drill-down @@ -183,5 +184,5 @@ WHERE src_hash != dst_hash OR src_hash IS NULL OR dst_hash IS NULL LIMIT 100 -SETTINGS join_use_nulls = 1; +SETTINGS join_use_nulls = 1, use_skip_indexes_if_final = 1; -- >>> END drill-down diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/delta_replay.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/delta_replay.sh index 8b6c1c139e1..66f5262522c 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/delta_replay.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/delta_replay.sh @@ -23,9 +23,9 @@ MAX_INSERT_BLOCK_SIZE=1048576 while [[ $# -gt 0 ]]; do case "$1" in - --database) DATABASE="$2"; shift 2 ;; - --backfill-start) BACKFILL_START="$2"; shift 2 ;; - --max-insert-block-size) MAX_INSERT_BLOCK_SIZE="$2"; shift 2 ;; + --database) DATABASE="${2:?"$1 requires a value"}"; shift 2 ;; + --backfill-start) BACKFILL_START="${2:?"$1 requires a value"}"; shift 2 ;; + --max-insert-block-size) MAX_INSERT_BLOCK_SIZE="${2:?"$1 requires a value"}"; shift 2 ;; *) echo "Unknown argument: $1" >&2; exit 2 ;; esac done diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/estimate.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/estimate.sh index 88ed166319c..bf92a434c59 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/estimate.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/estimate.sh @@ -38,12 +38,12 @@ ROWS_PER_SEC="" while [[ $# -gt 0 ]]; do case "$1" in - --database) DATABASE="$2"; shift 2 ;; - --max-rows-per-insert) MAX_ROWS="$2"; shift 2 ;; - --pause-seconds) PAUSE_SECONDS="$2"; shift 2 ;; - --probe-rows) PROBE_ROWS="$2"; shift 2 ;; - --write-cost-factor) WRITE_COST_FACTOR="$2"; shift 2 ;; - --rows-per-sec) ROWS_PER_SEC="$2"; shift 2 ;; + --database) DATABASE="${2:?"$1 requires a value"}"; shift 2 ;; + --max-rows-per-insert) MAX_ROWS="${2:?"$1 requires a value"}"; shift 2 ;; + --pause-seconds) PAUSE_SECONDS="${2:?"$1 requires a value"}"; shift 2 ;; + --probe-rows) PROBE_ROWS="${2:?"$1 requires a value"}"; shift 2 ;; + --write-cost-factor) WRITE_COST_FACTOR="${2:?"$1 requires a value"}"; shift 2 ;; + --rows-per-sec) ROWS_PER_SEC="${2:?"$1 requires a value"}"; shift 2 ;; *) echo "Unknown argument: $1" >&2; exit 2 ;; esac done @@ -59,7 +59,7 @@ done [[ -z "$ROWS_PER_SEC" || "$ROWS_PER_SEC" =~ ^[0-9]+(\.[0-9]+)?$ ]] || { echo "ERROR: --rows-per-sec must be a number." >&2; exit 2; } ch() { - clickhouse-client --database "$DATABASE" --query "$1" + clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_cutover:estimate' --query "$1" } # Physical rows to copy (count() honors the deleted-row mask, so masked rows are excluded — as the backfill excludes @@ -105,7 +105,7 @@ FACTOR_NOTE="" if [[ -z "$ROWS_PER_SEC" ]]; then PROBE_ACTUAL="$(awk -v a="$PROBE_ROWS" -v b="$TOTAL_ROWS" 'BEGIN { print (a < b) ? a : b }')" echo "Probing read throughput with a $PROBE_ACTUAL-row SELECT ... FORMAT Null (no table created)..." - ELAPSED="$(clickhouse-client --database "$DATABASE" --time --query \ + ELAPSED="$(clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_cutover:estimate' --time --query \ "SELECT * FROM traces LIMIT $PROBE_ROWS FORMAT Null" 2>&1 1>/dev/null)" READ_RPS="$(awk -v r="$PROBE_ACTUAL" -v t="$ELAPSED" 'BEGIN { print (t > 0) ? r / t : 0 }')" [[ "$(awk -v v="$READ_RPS" 'BEGIN { print (v > 0) ? 1 : 0 }')" == "1" ]] || { diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh index 7a1040ea31c..f68fab0028c 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh @@ -60,7 +60,7 @@ CONFIRM_BUFFER_RAISED=0 while [[ $# -gt 0 ]]; do case "$1" in - --database) DATABASE="$2"; shift 2 ;; + --database) DATABASE="${2:?"$1 requires a value"}"; shift 2 ;; --skip-wrap) SKIP_WRAP=1; shift ;; --with-wrap) WITH_WRAP=1; shift ;; --wrap-only) WRAP_ONLY=1; shift ;; @@ -110,7 +110,7 @@ if [[ "$WRAP_ONLY" != "1" && "$CONFIRM_BUFFER_RAISED" != "1" ]]; then fi ch() { - clickhouse-client --database "$DATABASE" --query "$1" + clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_cutover:exchange_and_wrap' --query "$1" } # Single scalar (empty string if the object does not exist). @@ -249,7 +249,7 @@ if [[ "$WRAP_ONLY" == "1" ]]; then exit 0 fi -CUTOVER_START="$(clickhouse-client --database "$DATABASE" --query "SELECT toString(now64(6))")" +CUTOVER_START="$(clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_cutover:exchange_and_wrap' --query "SELECT toString(now64(6))")" echo "RECORD cutover_start=$CUTOVER_START (pass to rollback.sh --cutover-start if you roll back after this point)" run_block exchange diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/finalize.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/finalize.sh index 18870b9f105..8ed0440517d 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/finalize.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/finalize.sh @@ -28,7 +28,7 @@ CONFIRM=0 while [[ $# -gt 0 ]]; do case "$1" in - --database) DATABASE="$2"; shift 2 ;; + --database) DATABASE="${2:?"$1 requires a value"}"; shift 2 ;; --confirm) CONFIRM=1; shift ;; *) echo "Unknown argument: $1" >&2; exit 2 ;; esac @@ -39,7 +39,7 @@ done [[ "$DATABASE" =~ ^[A-Za-z0-9_]+$ ]] || { echo "ERROR: --database must be a ClickHouse identifier (letters, digits, underscore)." >&2; exit 2; } ch() { - clickhouse-client --database "$DATABASE" --query "$1" + clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_cutover:finalize' --query "$1" } exists() { diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh index 550e55331fa..792f36571e2 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh @@ -29,9 +29,9 @@ CUTOVER_START="" while [[ $# -gt 0 ]]; do case "$1" in - --database) DATABASE="$2"; shift 2 ;; - --stage) STAGE="$2"; shift 2 ;; - --cutover-start) CUTOVER_START="$2"; shift 2 ;; + --database) DATABASE="${2:?"$1 requires a value"}"; shift 2 ;; + --stage) STAGE="${2:?"$1 requires a value"}"; shift 2 ;; + --cutover-start) CUTOVER_START="${2:?"$1 requires a value"}"; shift 2 ;; *) echo "Unknown argument: $1" >&2; exit 2 ;; esac done @@ -46,7 +46,7 @@ case "$STAGE" in esac ch() { - clickhouse-client --database "$DATABASE" --query "$1" + clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_rollback' --query "$1" } # Single scalar (or empty string if the object does not exist). Used by the topology guards below. diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh index 540796ba105..bac7d45b653 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh @@ -44,13 +44,13 @@ DRILL_DOWN=0 while [[ $# -gt 0 ]]; do case "$1" in - --database) DATABASE="$2"; shift 2 ;; - --old-table) OLD_TABLE="$2"; shift 2 ;; - --new-table) NEW_TABLE="$2"; shift 2 ;; - --sample-mod) SAMPLE_MOD="$2"; shift 2 ;; - --from-week) FROM_WEEK="$2"; shift 2 ;; - --to-week) TO_WEEK="$2"; shift 2 ;; - --weeks-stride) WEEKS_STRIDE="$2"; shift 2 ;; + --database) DATABASE="${2:?"$1 requires a value"}"; shift 2 ;; + --old-table) OLD_TABLE="${2:?"$1 requires a value"}"; shift 2 ;; + --new-table) NEW_TABLE="${2:?"$1 requires a value"}"; shift 2 ;; + --sample-mod) SAMPLE_MOD="${2:?"$1 requires a value"}"; shift 2 ;; + --from-week) FROM_WEEK="${2:?"$1 requires a value"}"; shift 2 ;; + --to-week) TO_WEEK="${2:?"$1 requires a value"}"; shift 2 ;; + --weeks-stride) WEEKS_STRIDE="${2:?"$1 requires a value"}"; shift 2 ;; --drill-down) DRILL_DOWN=1; shift ;; *) echo "Unknown argument: $1" >&2; exit 2 ;; esac @@ -69,7 +69,7 @@ done [[ -f "$VERIFY_SQL" ]] || { echo "ERROR: cannot find verify SQL at $VERIFY_SQL" >&2; exit 2; } ch() { - clickhouse-client --database "$DATABASE" --query "$1" + clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_cutover:verify' --query "$1" } log() { @@ -93,12 +93,12 @@ render_block() { # Verdict TSV row for one window: src_rows dst_rows src_checksum dst_checksum ok compare_window() { - clickhouse-client --database "$DATABASE" --multiquery --query "$(render_block compare "$1" "$2")" + clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_cutover:verify' --multiquery --query "$(render_block compare "$1" "$2")" } # Per-key differences for one window (only run on a mismatch, under --drill-down). drill_down_window() { - clickhouse-client --database "$DATABASE" --multiquery --query "$(render_block drill-down "$1" "$2")" + clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_cutover:verify' --multiquery --query "$(render_block drill-down "$1" "$2")" } ROWS="$(ch "SELECT count() FROM $OLD_TABLE")" diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java index 86782244b11..5baca4fad84 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java @@ -674,6 +674,52 @@ void deleteThenResurrectSurvivesTheReplay(boolean resolvedProject) { .isEqualTo(survivors.size()); } + /** + * The workspace-scoped ({@code project_id = ''}) replay branch must SPARE a live row that shares an {@code id} with + * another project. A delete-by-ids fallback that cannot resolve a project is bridged with an empty project and + * replayed on the {@code (workspace_id, id)} key; its resurrection guard keys only on {@code (workspace_id, id)}, so + * this proves it does not over-delete a cross-project live copy. Complements the single-project + * {@link #deleteThenResurrectSurvivesTheReplay} and the full-key reused-id case in + * {@link #bufferedCutoverPreservesEveryDeletionAcrossExchange}. + */ + @Test + void workspaceScopedReplaySparesLiveCrossProjectRow() { + var workspaceId = UUID.randomUUID().toString(); + var projectA = ID_GENERATOR.generateId(); + var projectB = ID_GENERATOR.generateId(); + var reusedInstant = weekInstant(0, 1); + var reusedId = ID_GENERATOR.generateId(reusedInstant); + var reused = List.of(CategorizedId.builder().id(reusedId).createdAt(reusedInstant).build()); + // Same id in two projects — ids are not globally unique. + seedTraces(reused, workspaceId, projectA); + seedTraces(reused, workspaceId, projectB); + + var backfillStart = nowMicros(); + for (int week = 0; week < SEED_WEEKS; week++) { + backfillWeek(week); + } + + // Workspace-scoped delete: bridged with an empty project_id (the delete-by-ids fallback). The source LWD is + // workspace-scoped, so it removes the id from BOTH projects; then resurrect it in projectB only (a newer version + // wins under FINAL), so it is live again on the source in B and caught by the delta's last_updated_at arm. + recordDeletionEvents(Set.of(reusedId.toString()), workspaceId, "", "user_request"); + lightweightDelete(Set.of(reusedId.toString()), workspaceId); + insertRows(reused, workspaceId, projectB, "resurrected", _ -> Instant.now()); + + deltaInsert(backfillStart); + replayDeletions(backfillStart); + replayDeletions(backfillStart); // idempotent + + assertThat(liveCountScoped("traces_local_v2", Set.of(reusedId.toString()), workspaceId, projectB)) + .as("workspace-scoped replay spares the live projectB copy that shares an id with projectA") + .isEqualTo(1L); + // Known residual (OPIK-7483; the 000005 fingerprint flags it as an extra destination row, ok=0): because the id + // is live in B, the (workspace_id, id) guard skips the whole id, so the stale projectA copy is not removed here. + assertThat(liveCountScoped("traces_local_v2", Set.of(reusedId.toString()), workspaceId, projectA)) + .as("documented residual: the stale other-project copy remains until OPIK-7483") + .isEqualTo(1L); + } + /** * Schema-drift guard. The cutover copies a fixed column list, and the fidelity fingerprint also lists fixed * columns — so a base column added to {@code traces} by a future migration would be silently left uncopied, with no @@ -1419,8 +1465,8 @@ private static String deltaName() { } /** A within-day offset so ids/created_at in the same week are distinct but stay inside their weekly partition. */ - private Instant weekInstant(int weekOffset, int minuteOffset) { - return ANCHOR_MONDAY.plusWeeks(weekOffset).atTime(1, 0).plusSeconds(minuteOffset).toInstant(ZoneOffset.UTC); + private Instant weekInstant(int weekOffset, int secondOffset) { + return ANCHOR_MONDAY.plusWeeks(weekOffset).atTime(1, 0).plusSeconds(secondOffset).toInstant(ZoneOffset.UTC); } @Builder(toBuilder = true) From a5c9b7afdef34f3ce9cb86e7ef019e0f7d2a3461 Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Mon, 27 Jul 2026 09:37:49 +0200 Subject: [PATCH 16/28] =?UTF-8?q?test(cutover):=20address=20JetoPistola=20?= =?UTF-8?q?review=20(clear=20wins)=20=E2=80=94=20align=20test=20SQL,=20fix?= =?UTF-8?q?=20flakes/footguns?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Gate test: replace parseDateTime64BestEffort with toDateTime64(..., N[, 'UTC']) matching the reference SQL exactly (also drops the OPIK-5694 anti-pattern that pollutes system.errors); derive the during-window delta instants from the CH server clock (via a later now64(6)) instead of the JVM host clock, so host<-> container skew can't push last_updated_at/created_at below backfillStart and flake the delta arms. - delete_traffic.py: tolerate a few consecutive empty refills (delete-mask visibility lag) before concluding the project is drained, so a blip doesn't end the run early. - requirements.txt: drop `opik` (README installs it editable; listing it here lets pip shadow the local unreleased SDK) and pin a clickhouse-connect floor. - verify.sh / estimate.sh: capture the clickhouse-client output before `read <<<` so set -e catches an infra failure instead of it being swallowed and mislabeled (verify: a false MISMATCH; estimate: a false "table is empty"). Gate test 9/9 green on CH 26.3. Co-Authored-By: Claude Opus 4.8 --- .../scripts/estimate.sh | 5 +++- .../traces-local-v2-cutover/scripts/verify.sh | 6 +++- .../TracesLocalV2CutoverTest.java | 30 ++++++++++--------- .../traces-local-v2-cutover/delete_traffic.py | 17 +++++++++-- .../traces-local-v2-cutover/requirements.txt | 7 +++-- 5 files changed, 45 insertions(+), 20 deletions(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/estimate.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/estimate.sh index bf92a434c59..f8e87d8868f 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/estimate.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/estimate.sh @@ -65,7 +65,9 @@ ch() { # Physical rows to copy (count() honors the deleted-row mask, so masked rows are excluded — as the backfill excludes # them) and the projected window count: each week splits until every sub-window is <= MAX_ROWS, so a week of `cnt` rows # yields ~ceil(cnt / MAX_ROWS) inserts. Both come from one grouped scan. -read -r TOTAL_ROWS EST_WINDOWS WEEKS <<< "$(ch " +# Capture first so set -e catches a clickhouse-client failure, rather than a swallowed here-string substitution leaving +# TOTAL_ROWS empty and mislabeling the run as "table is empty". +sizing="$(ch " SELECT sum(cnt), sum(if(cnt = 0, 0, toUInt64(ceil(cnt / $MAX_ROWS)))), @@ -79,6 +81,7 @@ read -r TOTAL_ROWS EST_WINDOWS WEEKS <<< "$(ch " ) FORMAT TSV ")" +read -r TOTAL_ROWS EST_WINDOWS WEEKS <<< "$sizing" if [[ -z "$TOTAL_ROWS" || "$TOTAL_ROWS" == "0" ]]; then echo "Source table 'traces' is empty — nothing to backfill." diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh index bac7d45b653..08fbfe34a5f 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh @@ -122,7 +122,11 @@ for (( week=FROM_WEEK; week<=TO_WEEK; week+=WEEKS_STRIDE )); do LO="$(ch "SELECT toString(addWeeks(toDate('$ANCHOR'), $week))") 00:00:00" HI="$(ch "SELECT toString(addWeeks(toDate('$ANCHOR'), $((week + 1))))") 00:00:00" - read -r src_rows dst_rows src_checksum dst_checksum ok <<< "$(compare_window "$LO" "$HI")" + # Capture first (not read <<< "$(...)"): a here-string command substitution is exempt from set -e, so a + # clickhouse-client failure here would be swallowed, leaving `ok` empty and the week mislabeled as a MISMATCH. A + # plain assignment IS caught by set -e, so an infra blip aborts with the real error instead of a false fidelity fail. + compare_out="$(compare_window "$LO" "$HI")" + read -r src_rows dst_rows src_checksum dst_checksum ok <<< "$compare_out" checked=$((checked + 1)) if [[ "$ok" == "1" ]]; then log "week $week ($LO .. $HI): OK (rows=$src_rows)" diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java index 5baca4fad84..89b78a4dce5 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java @@ -336,11 +336,14 @@ void bufferedCutoverPreservesEveryDeletionAcrossExchange() { // Reused-id delete scoped to projectId only — the copy under otherProjectId must survive. recordDeletionEvents(Set.of(reusedId.toString()), workspaceId, projectId.toString(), "user_request"); lightweightDeleteScoped(Set.of(reusedId.toString()), workspaceId, projectId); + // During-window instant from the SAME server clock as backfillStart (a later now64(6), so >= backfillStart) — + // NOT the JVM host clock, whose skew vs the container could put these below backfillStart and flake the delta. + var duringWindow = Instant.from(ClickHouseDateTimeFormat.MICROS.parse(nowMicros())); // Concurrent upserts: a newer version of a subset of survivors — caught by the delta's last_updated_at arm. var deltaUpserted = survivors.subList(0, DELTA_UPSERTS); - insertRows(deltaUpserted, workspaceId, projectId, deltaName(), _ -> Instant.now()); + insertRows(deltaUpserted, workspaceId, projectId, deltaName(), _ -> duringWindow); // Rows created during the window with a client-backdated last_updated_at — caught ONLY by the created_at arm. - var deltaLateCreated = mintNowIds(DELTA_LATE_CREATED); + var deltaLateCreated = mintIdsAt(DELTA_LATE_CREATED, duringWindow); insertRows(deltaLateCreated, workspaceId, projectId, "late", _ -> BACKDATED); // Delta-insert: created_at OR last_updated_at since backfill_start (see class Javadoc). @@ -802,8 +805,8 @@ INSERT INTO traces_local_v2 ( SELECT %s FROM traces - WHERE created_at >= parseDateTime64BestEffort(:week_lo, 9) - AND created_at < parseDateTime64BestEffort(:week_hi, 9) + WHERE created_at >= toDateTime64(:week_lo, 9, 'UTC') + AND created_at < toDateTime64(:week_hi, 9, 'UTC') SETTINGS max_insert_block_size = 100000 """.formatted(COPIED_COLUMNS, COPIED_SELECT), statement -> statement.bind("week_lo", weekLo).bind("week_hi", weekHi)); @@ -822,8 +825,8 @@ INSERT INTO traces_local_v2 ( SELECT %s FROM traces - WHERE created_at >= parseDateTime64BestEffort(:backfill_start, 6) - OR last_updated_at >= parseDateTime64BestEffort(:backfill_start, 6) + WHERE created_at >= toDateTime64(:backfill_start, 6) + OR last_updated_at >= toDateTime64(:backfill_start, 6) SETTINGS max_insert_block_size = 100000 """.formatted(COPIED_COLUMNS, COPIED_SELECT), statement -> statement.bind("backfill_start", backfillStart)); @@ -854,7 +857,7 @@ private long replayDeletions(String backfillStart) { toFixedString(deleted_id, 36) FROM deletion_events_local WHERE source_table = 'traces' - AND event_time >= parseDateTime64BestEffort(:backfill_start, 6) + AND event_time >= toDateTime64(:backfill_start, 6) AND project_id != '' ) AND (workspace_id, project_id, id) NOT IN ( @@ -867,7 +870,7 @@ WHERE id IN ( SELECT toFixedString(deleted_id, 36) FROM deletion_events_local WHERE source_table = 'traces' - AND event_time >= parseDateTime64BestEffort(:backfill_start, 6) + AND event_time >= toDateTime64(:backfill_start, 6) ) ) ) @@ -878,7 +881,7 @@ SELECT toFixedString(deleted_id, 36) toFixedString(deleted_id, 36) FROM deletion_events_local WHERE source_table = 'traces' - AND event_time >= parseDateTime64BestEffort(:backfill_start, 6) + AND event_time >= toDateTime64(:backfill_start, 6) AND project_id = '' ) AND (workspace_id, id) NOT IN ( @@ -890,7 +893,7 @@ WHERE id IN ( SELECT toFixedString(deleted_id, 36) FROM deletion_events_local WHERE source_table = 'traces' - AND event_time >= parseDateTime64BestEffort(:backfill_start, 6) + AND event_time >= toDateTime64(:backfill_start, 6) ) ) ) @@ -992,7 +995,7 @@ private void reverseReplay(String cutoverStart) { toFixedString(deleted_id, 36) FROM deletion_events_local WHERE source_table = 'traces' - AND event_time >= parseDateTime64BestEffort(:cutover_start, 6) + AND event_time >= toDateTime64(:cutover_start, 6) AND project_id != '' ) OR (workspace_id, id) IN ( @@ -1001,7 +1004,7 @@ private void reverseReplay(String cutoverStart) { toFixedString(deleted_id, 36) FROM deletion_events_local WHERE source_table = 'traces' - AND event_time >= parseDateTime64BestEffort(:cutover_start, 6) + AND event_time >= toDateTime64(:cutover_start, 6) AND project_id = '' ) SETTINGS allow_nondeterministic_mutations = 1, @@ -1441,10 +1444,9 @@ private List mintIds(int perWeek) { } /** Ids created "now" — used for rows written during the window, so their created_at is >= backfill_start. */ - private List mintNowIds(int count) { + private List mintIdsAt(int count, Instant createdAt) { var ids = new ArrayList(); for (int i = 0; i < count; i++) { - var createdAt = Instant.now(); ids.add(CategorizedId.builder().id(ID_GENERATOR.generateId(createdAt)).createdAt(createdAt).build()); } return ids; diff --git a/tests_load/tests/traces-local-v2-cutover/delete_traffic.py b/tests_load/tests/traces-local-v2-cutover/delete_traffic.py index 87a14393014..c1427a9c93b 100644 --- a/tests_load/tests/traces-local-v2-cutover/delete_traffic.py +++ b/tests_load/tests/traces-local-v2-cutover/delete_traffic.py @@ -19,6 +19,10 @@ _stop = False +# Consecutive empty refills tolerated before concluding the project is drained. An empty refill can be a transient +# delete-mask-visibility lag (the just-deleted top ids not yet hidden from search), not a truly empty project. +EMPTY_REFILL_LIMIT = 3 + def _handle_sigint(_signum, _frame): global _stop @@ -48,6 +52,7 @@ def main(project, tps, duration, batch): seen: set[str] = set() pool: list[str] = [] deleted = 0 + empty_refills = 0 started = time.time() LOGGER.info("delete traffic: project='%s' tps=%.2f batch=%d duration=%ss (Ctrl-C to stop)", project, tps, batch, duration or "∞") @@ -63,8 +68,16 @@ def main(project, tps, duration, batch): continue pool.extend(fetched) if not pool: - LOGGER.info("no more traces to delete; stopping") - break + # An empty refill can be transient: the delete mask may not be visible to search yet, so the newest 500 + # ids can all still be in `seen`/`pool`. Only stop after several consecutive empty refills, so a mask-lag + # blip doesn't end the run while thousands of lower-id traces remain undeleted. + empty_refills += 1 + if empty_refills >= EMPTY_REFILL_LIMIT: + LOGGER.info("no more traces to delete after %d empty refills; stopping", empty_refills) + break + time.sleep(interval if interval > 0 else 0.5) + continue + empty_refills = 0 ids = [pool.pop(0) for _ in range(min(batch, len(pool)))] seen.update(ids) client.rest_client.traces.delete_traces(ids=ids) diff --git a/tests_load/tests/traces-local-v2-cutover/requirements.txt b/tests_load/tests/traces-local-v2-cutover/requirements.txt index 982b6fe2b36..285d486e646 100644 --- a/tests_load/tests/traces-local-v2-cutover/requirements.txt +++ b/tests_load/tests/traces-local-v2-cutover/requirements.txt @@ -1,3 +1,6 @@ -opik +# NOTE: `opik` is intentionally NOT listed here. The README installs it editable from the local checkout +# (`pip install -e sdks/python`) so the rehearsal runs against the unreleased cutover SDK; listing it here would let +# `pip install -r requirements.txt` re-resolve it from PyPI and shadow that local install. click -clickhouse-connect +# clickhouse-connect had breaking get_client(...) changes across earlier minors; pin a floor with the current signature. +clickhouse-connect>=0.7 From 54729ab635b589284e26f52469dd958719004252 Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Mon, 27 Jul 2026 10:05:04 +0200 Subject: [PATCH 17/28] =?UTF-8?q?test(cutover):=20address=20JetoPistola=20?= =?UTF-8?q?review=20(remaining)=20=E2=80=94=20replay=20fail-safe,=20tiered?= =?UTF-8?q?=20gate,=20version-check,=20gate-scope=20note?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Deletion replay + reverse-replay: guard the deletion-events subqueries with length(deleted_id)=36 (and project_id) so a malformed bridge row is skipped rather than aborting the whole replay on toFixedString — a >36-char id can't match a real trace id, so skipping loses nothing. Mirrored in the gate test's inline replay to keep it aligned with the reference SQL. - backfill.sh: require --confirm-tiered-headroom when the destination storage_policy is tiered (multi-volume) or differs from the source — the whole-node free-space gate can't see per-volume hot headroom (an accurate auto-check isn't feasible), so force an explicit operator acknowledgment. - 000005 + verify.sh: add a version-collapse check — count old-table keys whose last_updated_at has more ns-distinct values than us-distinct, surfacing where the ns->us truncation could pick a different ReplacingMergeTree version than source FINAL (invisible to the us fingerprint). Reported, not pass/fail. - Gate test Javadoc: document that the inline SQL reimplementation is deliberate (an independent logic check); the drivers are the single-source reference-SQL path, validated end-to-end by the prod-clone QA. Kept inline aligned to the ref. Gate test 9/9 green on CH 26.3. Co-Authored-By: Claude Opus 4.8 --- .../scripts/backfill.sh | 20 ++++++++++++++++++ .../000002_delta_and_deletion_replay.sql | 9 ++++++++ .../000004_rollback_reverse_replay.sql | 3 +++ .../000005_verify_migration.sql | 18 ++++++++++++++++ .../traces-local-v2-cutover/scripts/verify.sh | 21 ++++++++++++++++++- .../TracesLocalV2CutoverTest.java | 16 ++++++++++++++ 6 files changed, 86 insertions(+), 1 deletion(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh index aa87f9b92b0..8aeb570af26 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh @@ -40,6 +40,10 @@ # --min-free-factor F abort at startup unless node free disk >= F x the current `traces` on-disk size (the # backfill writes a full second copy). Default 2.0. Pass 0 to skip the check. This is a # whole-node floor; on tiered storage validate per-volume (hot) headroom separately. +# --confirm-tiered-headroom REQUIRED when the destination storage_policy is tiered (multi-volume) or differs from the +# source's. The whole-node --min-free-factor check cannot see per-volume headroom, and new +# parts land on the hot volume before they tier; this asserts the operator validated hot +# headroom out of band. (No effect on a single-volume/default policy.) # --state-file PATH file the captured backfill_start is written to and reused from. On resume the ORIGINAL # anchor is kept; re-minting a later one would miss deletes that fired during the first run # against already-copied rows. Default ./traces_cutover_backfill_start. @@ -67,6 +71,7 @@ DIVERGENCE="0.0001" # fraction: max tolerated |src-dst|/src per settled wi PAUSE_SECONDS=0 # seconds: sleep after each inserted window so destination merges catch up. 30-60 at ~4 TB peak. MIN_FREE_FACTOR="2.0" # multiple of the current traces on-disk size that node free space must clear before starting. STATE_FILE="./traces_cutover_backfill_start" # backfill_start is persisted here and reused on resume (keeps one anchor). +CONFIRM_TIERED_HEADROOM=0 # required when the destination storage_policy is tiered/mismatched (see preflight_capacity). # Floor on adaptive splitting: never divide a window shorter than this. Guards against splitting forever on a single # hot instant; such a window is inserted whole (memory is still bounded by block squashing). @@ -83,6 +88,7 @@ while [[ $# -gt 0 ]]; do --divergence) DIVERGENCE="${2:?"$1 requires a value"}"; shift 2 ;; --pause-seconds) PAUSE_SECONDS="${2:?"$1 requires a value"}"; shift 2 ;; --min-free-factor) MIN_FREE_FACTOR="${2:?"$1 requires a value"}"; shift 2 ;; + --confirm-tiered-headroom) CONFIRM_TIERED_HEADROOM=1; shift ;; --state-file) STATE_FILE="${2:?"$1 requires a value"}"; shift 2 ;; *) echo "Unknown argument: $1" >&2; exit 2 ;; esac @@ -139,6 +145,20 @@ preflight_capacity() { if [[ "$src_policy" != "$dst_policy" ]]; then log "WARNING: storage_policy differs ($SRC_TABLE='$src_policy', $DST_TABLE='$dst_policy'). If $SRC_TABLE tiers to cold and $DST_TABLE does not, the whole backfill lands on the hot volume. Confirm this is intended." >&2 fi + # Tiered/mismatched storage_policy: the whole-node check above CANNOT see per-volume headroom (new parts land on the + # hot volume before they tier, so the node total can pass while hot fills mid-backfill — the likeliest prod failure). + # An accurate hot-headroom check isn't feasible in a preflight (it depends on tiering-vs-write rate), so require an + # explicit operator acknowledgment that per-volume headroom was validated out of band, rather than proceed silently. + local dst_volumes + dst_volumes="$(ch "SELECT uniqExact(volume_name) FROM system.storage_policies WHERE policy_name = '$dst_policy'")" + dst_volumes="${dst_volumes:-1}" + if [[ "$dst_volumes" -gt 1 || "$src_policy" != "$dst_policy" ]]; then + if [[ "$CONFIRM_TIERED_HEADROOM" != "1" ]]; then + log "ABORT: $DST_TABLE uses a tiered/mismatched storage_policy ('$dst_policy', $dst_volumes volume(s)). The whole-node free-space gate cannot see per-volume headroom — validate the HOT volume has room for the backfill out of band, then re-run with --confirm-tiered-headroom." >&2 + exit 1 + fi + log "Tiered/mismatched storage_policy acknowledged via --confirm-tiered-headroom (hot-volume headroom validated out of band)." + fi } # Live source rows in [lo, hi). count() honors the deleted-row mask, so masked rows are excluded (they must not copy). diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000002_delta_and_deletion_replay.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000002_delta_and_deletion_replay.sql index f7ae2b73875..7227a7f4e18 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000002_delta_and_deletion_replay.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000002_delta_and_deletion_replay.sql @@ -116,6 +116,10 @@ SETTINGS max_insert_block_size = ${MAX_INSERT_BLOCK_SIZE}, -- Uses ${BACKFILL_START}. Retention is disabled everywhere (see step 6), so this is user-scale volume — a single -- mutation. If it is ever large (e.g. retention enabled), bound each mutation by a partition predicate and loop the -- weeks, e.g. AND toMonday(id_at) = toDate(''). +-- length(...) = 36 guards: toFixedString(x, 36) THROWS on a value longer than 36 bytes, which would abort the whole +-- replay on a single malformed bridge row. For source_table='traces' the ids are 36-char UUIDs, so this is latent — but +-- a malformed (non-36-char) deleted_id/project_id can't match a real trace id anyway, so skipping it via the length +-- guard loses nothing and turns a hard abort mid-cutover into a benign no-op. Same guards in the reverse-replay. DELETE FROM ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 WHERE ( (workspace_id, project_id, id) IN ( @@ -127,6 +131,8 @@ WHERE ( WHERE source_table = 'traces' AND event_time >= toDateTime64('${BACKFILL_START}', 6) AND project_id != '' + AND length(project_id) = 36 + AND length(deleted_id) = 36 ) AND (workspace_id, project_id, id) NOT IN ( SELECT @@ -139,6 +145,7 @@ WHERE ( FROM ${ANALYTICS_DB_DATABASE_NAME}.deletion_events_local WHERE source_table = 'traces' AND event_time >= toDateTime64('${BACKFILL_START}', 6) + AND length(deleted_id) = 36 ) ) ) @@ -151,6 +158,7 @@ OR ( WHERE source_table = 'traces' AND event_time >= toDateTime64('${BACKFILL_START}', 6) AND project_id = '' + AND length(deleted_id) = 36 ) AND (workspace_id, id) NOT IN ( SELECT @@ -162,6 +170,7 @@ OR ( FROM ${ANALYTICS_DB_DATABASE_NAME}.deletion_events_local WHERE source_table = 'traces' AND event_time >= toDateTime64('${BACKFILL_START}', 6) + AND length(deleted_id) = 36 ) ) ) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_reverse_replay.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_reverse_replay.sql index 6573f710d1d..d580a6340cd 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_reverse_replay.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_reverse_replay.sql @@ -21,6 +21,8 @@ WHERE (workspace_id, project_id, id) IN ( WHERE source_table = 'traces' AND event_time >= toDateTime64('${CUTOVER_START}', 6) AND project_id != '' + AND length(project_id) = 36 + AND length(deleted_id) = 36 ) OR (workspace_id, id) IN ( SELECT @@ -30,6 +32,7 @@ OR (workspace_id, id) IN ( WHERE source_table = 'traces' AND event_time >= toDateTime64('${CUTOVER_START}', 6) AND project_id = '' + AND length(deleted_id) = 36 ) -- lightweight_deletes_sync = 2: wait for the mutation on every replica so the restored `traces` is consistent -- cluster-wide before the rollback is declared done (see 000002 for the rationale). diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000005_verify_migration.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000005_verify_migration.sql index 53979bb608c..ee9c1b6ff20 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000005_verify_migration.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000005_verify_migration.sql @@ -186,3 +186,21 @@ WHERE src_hash != dst_hash LIMIT 100 SETTINGS join_use_nulls = 1, use_skip_indexes_if_final = 1; -- >>> END drill-down + +-- >>> BEGIN version-check +-- ns->us version-selection guard. The successor's last_updated_at is DateTime64(6) but the source is DateTime64(9), and +-- last_updated_at is the ReplacingMergeTree version. Two source part-versions of the same key that differ ONLY in +-- sub-microsecond digits collapse to one version on the successor, which may then keep different column values than +-- source FINAL (which keeps the ns-max) — a divergence the microsecond-normalized fingerprint above cannot detect. This +-- surfaces the PRECONDITION: old-schema keys in the window with more distinct ns last_updated_at values than us ones. +-- 0 => the truncation cannot change version selection (safe). > 0 => investigate those keys before trusting the compare. +SELECT count() AS collapse_keys +FROM ( + SELECT workspace_id, project_id, id + FROM ${ANALYTICS_DB_DATABASE_NAME}.${OLD_TABLE} + WHERE created_at >= toDateTime64('${WINDOW_LO}', 9) + AND created_at < toDateTime64('${WINDOW_HI}', 9) + GROUP BY workspace_id, project_id, id + HAVING uniqExact(last_updated_at) > uniqExact(toDateTime64(last_updated_at, 6)) +); +-- >>> END version-check diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh index 08fbfe34a5f..7f568ed2c0c 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh @@ -96,6 +96,13 @@ compare_window() { clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_cutover:verify' --multiquery --query "$(render_block compare "$1" "$2")" } +# Count of old-table keys in the window whose last_updated_at has more ns-distinct values than us-distinct — i.e. rows +# where the ns->us truncation could make the successor pick a different ReplacingMergeTree version than source FINAL, a +# divergence the microsecond fingerprint cannot see. 0 => version selection is truncation-safe for the window. +version_check_window() { + clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_cutover:verify' --multiquery --query "$(render_block version-check "$1" "$2")" +} + # Per-key differences for one window (only run on a mismatch, under --drill-down). drill_down_window() { clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_cutover:verify' --multiquery --query "$(render_block drill-down "$1" "$2")" @@ -118,6 +125,7 @@ log "Verify: $OLD_TABLE vs $NEW_TABLE | weeks [$FROM_WEEK..$TO_WEEK] stride $WEE mismatches=0 checked=0 +version_collapse=0 for (( week=FROM_WEEK; week<=TO_WEEK; week+=WEEKS_STRIDE )); do LO="$(ch "SELECT toString(addWeeks(toDate('$ANCHOR'), $week))") 00:00:00" HI="$(ch "SELECT toString(addWeeks(toDate('$ANCHOR'), $((week + 1))))") 00:00:00" @@ -140,10 +148,21 @@ for (( week=FROM_WEEK; week<=TO_WEEK; week+=WEEKS_STRIDE )); do log " re-run with --drill-down to list the differing keys for this window" >&2 fi fi + + # ns->us version-selection guard (does not affect pass/fail — the fingerprint can't see it, so it is reported + # separately for the operator to investigate rather than silently trusted). + collapse_out="$(version_check_window "$LO" "$HI")" + if [[ -n "$collapse_out" && "$collapse_out" != "0" ]]; then + version_collapse=$((version_collapse + collapse_out)) + log " NOTE week $week: $collapse_out key(s) on $OLD_TABLE have sub-microsecond-distinct last_updated_at — the ns->us truncation may pick a different version than source FINAL for them; the fingerprint cannot detect it. Investigate." >&2 + fi done +if [[ "$version_collapse" != "0" ]]; then + log "WARNING: $version_collapse key(s) across the checked weeks have sub-microsecond-distinct last_updated_at on $OLD_TABLE. The microsecond fingerprint cannot see version-selection differences for them, so investigate those keys before trusting a PASS as zero-loss." >&2 +fi if [[ "$mismatches" != "0" ]]; then log "FAILED: $mismatches of $checked windows mismatched." >&2 exit 1 fi -log "PASSED: all $checked windows match (sample 1/$SAMPLE_MOD)." +log "PASSED: all $checked windows match (sample 1/$SAMPLE_MOD). Version-collapse keys: $version_collapse." diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java index 89b78a4dce5..7b3ff918271 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java @@ -44,6 +44,14 @@ * steps an operator runs from the {@code data-migrations/traces-local-v2-cutover} runbook — and pins the properties * the cutover's correctness depends on. * + *

Inline SQL, by design. This gate reimplements the cutover statements inline rather than executing the + * reference {@code .sql} files the drivers ship, so it can interleave seeding, per-step assertions, and the negative + * controls below — a deliberate choice. It is an independent validation of the cutover logic, not the + * single-source path: the driver scripts read the single-source reference SQL (that "no copy-paste drift" property is + * about the operator tools), and the shipped SQL itself is exercised end-to-end by running those drivers against a + * full-volume prod clone in the QA gate (OPIK-7405). The inline statements here are kept aligned with the reference SQL + * — identical functions, precision, and {@code 'UTC'} — so this gate and the shipped SQL stay in step. + * *

Deletions must survive the swap (the core property). A lightweight DELETE flips a hidden row mask; it does * not bump {@code last_updated_at} (the {@code ReplacingMergeTree} version column), so the version-based delta-insert is * blind to deletes that land while the table is being copied — the already-copied row stays alive on the destination @@ -859,6 +867,8 @@ private long replayDeletions(String backfillStart) { WHERE source_table = 'traces' AND event_time >= toDateTime64(:backfill_start, 6) AND project_id != '' + AND length(project_id) = 36 + AND length(deleted_id) = 36 ) AND (workspace_id, project_id, id) NOT IN ( SELECT @@ -871,6 +881,7 @@ SELECT toFixedString(deleted_id, 36) FROM deletion_events_local WHERE source_table = 'traces' AND event_time >= toDateTime64(:backfill_start, 6) + AND length(deleted_id) = 36 ) ) ) @@ -883,6 +894,7 @@ SELECT toFixedString(deleted_id, 36) WHERE source_table = 'traces' AND event_time >= toDateTime64(:backfill_start, 6) AND project_id = '' + AND length(deleted_id) = 36 ) AND (workspace_id, id) NOT IN ( SELECT @@ -894,6 +906,7 @@ SELECT toFixedString(deleted_id, 36) FROM deletion_events_local WHERE source_table = 'traces' AND event_time >= toDateTime64(:backfill_start, 6) + AND length(deleted_id) = 36 ) ) ) @@ -997,6 +1010,8 @@ private void reverseReplay(String cutoverStart) { WHERE source_table = 'traces' AND event_time >= toDateTime64(:cutover_start, 6) AND project_id != '' + AND length(project_id) = 36 + AND length(deleted_id) = 36 ) OR (workspace_id, id) IN ( SELECT @@ -1006,6 +1021,7 @@ private void reverseReplay(String cutoverStart) { WHERE source_table = 'traces' AND event_time >= toDateTime64(:cutover_start, 6) AND project_id = '' + AND length(deleted_id) = 36 ) SETTINGS allow_nondeterministic_mutations = 1, lightweight_deletes_sync = 2 From 1e888a1d86320baf53515dbea8b9e506013f2b1a Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Mon, 27 Jul 2026 10:17:35 +0200 Subject: [PATCH 18/28] docs(cutover): cross-reference the gate test from the reference SQL it mirrors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a one-line pointer to each reference .sql the gate reimplements inline (000001-000004 family) noting that TracesLocalV2CutoverTest mirrors it and the two must be kept in step. Closes the drift-awareness gap in the direction the test's Javadoc doesn't cover — a maintainer editing the SQL now sees the link without opening the test. 000005 is not pointed at (its fingerprint is validated independently, not reimplemented inline). Comment-only. Co-Authored-By: Claude Opus 4.8 --- .../scripts/db-app-analytics/000001_backfill_traces_local_v2.sql | 1 + .../db-app-analytics/000002_delta_and_deletion_replay.sql | 1 + .../scripts/db-app-analytics/000003_exchange_and_wrap.sql | 1 + .../scripts/db-app-analytics/000004_rollback_reverse_replay.sql | 1 + .../db-app-analytics/000004_rollback_stage_a_discard_shadow.sql | 1 + .../db-app-analytics/000004_rollback_stage_b_exchange_back.sql | 1 + .../000004_rollback_stage_c_promote_original.sql | 1 + 7 files changed, 7 insertions(+) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000001_backfill_traces_local_v2.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000001_backfill_traces_local_v2.sql index b983421ce02..0456fa17eaf 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000001_backfill_traces_local_v2.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000001_backfill_traces_local_v2.sql @@ -1,4 +1,5 @@ -- runbook traces-local-v2-cutover — step 1 of 3: backfill (reference statement) +-- The gate test TracesLocalV2CutoverTest reimplements this statement inline; keep the two in step (see its Javadoc). -- -- This file is the SINGLE source of the backfill INSERT; ../backfill.sh reads it, substitutes the ${...} placeholders -- (database, window bounds, block size) and runs it once per time sub-window — so the script and this reference never diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000002_delta_and_deletion_replay.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000002_delta_and_deletion_replay.sql index 7227a7f4e18..9eff4280af3 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000002_delta_and_deletion_replay.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000002_delta_and_deletion_replay.sql @@ -1,4 +1,5 @@ -- runbook traces-local-v2-cutover — step 2 of 3: delta-insert + deletion replay +-- The gate test TracesLocalV2CutoverTest reimplements these statements inline; keep the two in step (see its Javadoc). -- Run this only after the whole backfill (step 1) is complete and reconciled. -- Step 0: The SQL below (delta-insert + deletion replay) is the single source driven by ../delta_replay.sh, which reads diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000003_exchange_and_wrap.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000003_exchange_and_wrap.sql index 4d220ef72f3..24c3521e06f 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000003_exchange_and_wrap.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000003_exchange_and_wrap.sql @@ -1,4 +1,5 @@ -- runbook traces-local-v2-cutover — step 3 of 3: EXCHANGE + Distributed wrap (reference statements) +-- The gate test TracesLocalV2CutoverTest reimplements these statements inline; keep the two in step (see its Javadoc). -- -- ../exchange_and_wrap.sh drives this: it records cutover_start, runs the `exchange` block, and (unless --skip-wrap) -- the `wrap` block. Run it right after step 2's delta + replay, while the async-insert buffer is still holding writes. diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_reverse_replay.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_reverse_replay.sql index d580a6340cd..002a39b6047 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_reverse_replay.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_reverse_replay.sql @@ -1,4 +1,5 @@ -- runbook traces-local-v2-cutover — ROLLBACK reverse-replay (driven by ../rollback.sh after stage B or C) +-- The gate test TracesLocalV2CutoverTest reimplements this statement inline; keep the two in step (see its Javadoc). -- -- Re-applies the deletes that fired on the successor since cutover_start onto the restored original `traces`, so they do -- not resurrect. Two branches, like the forward replay in 000002: events with a project match the full key; events from diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_a_discard_shadow.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_a_discard_shadow.sql index 075e2a29019..aa64eb8e092 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_a_discard_shadow.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_a_discard_shadow.sql @@ -1,4 +1,5 @@ -- runbook traces-local-v2-cutover — ROLLBACK stage A: discard the shadow (driven by ../rollback.sh --stage A) +-- The gate test TracesLocalV2CutoverTest reimplements this rollback inline; keep the two in step (see its Javadoc). -- -- Use when the backfill/delta ran but the EXCHANGE did NOT. The live `traces` was never touched by the backfill, so -- there is nothing to restore; this only discards the disposable shadow copy so a re-attempt starts clean. backfill.sh diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_b_exchange_back.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_b_exchange_back.sql index 9b5eef81315..ff4d3f72b72 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_b_exchange_back.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_b_exchange_back.sql @@ -1,4 +1,5 @@ -- runbook traces-local-v2-cutover — ROLLBACK stage B: swap the tables back (driven by ../rollback.sh --stage B) +-- The gate test TracesLocalV2CutoverTest reimplements this rollback inline; keep the two in step (see its Javadoc). -- -- Use when the EXCHANGE ran but the wrap did NOT. `traces` holds the successor data and `traces_pre_cutover_backup` -- parks the original. A SINGLE atomic multi-target RENAME rotates both names back: the successor (`traces`) returns to diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql index 83312b0b82d..59bdbcc7373 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000004_rollback_stage_c_promote_original.sql @@ -1,4 +1,5 @@ -- runbook traces-local-v2-cutover — ROLLBACK stage C: promote the original back (driven by ../rollback.sh --stage C) +-- The gate test TracesLocalV2CutoverTest reimplements this rollback inline; keep the two in step (see its Javadoc). -- -- Use when the wrap ran. Post-wrap topology: `traces` is a Distributed wrapper over `traces_local` (successor data); -- `traces_pre_cutover_backup` parks the original. Promote the original back to `traces` GAPLESSLY with a single atomic From c5787132c9108abd052e3598febf3c62080c8226 Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Mon, 27 Jul 2026 10:49:01 +0200 Subject: [PATCH 19/28] =?UTF-8?q?test(cutover):=20address=20Baz=20?= =?UTF-8?q?=E2=80=94=20bad-id=20partition=20msg,=20liveCount=20FINAL,=20em?= =?UTF-8?q?pty-old-table=20guard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - seed_history.py: the bad-id log said id_at ~2201, but the successor's 32-bit DateTime wraps to ~2065 (OPIK-7456) — point operators at the ~2065 partition. - liveCount / liveCountScoped: SELECT ... FROM %s FINAL so leak/survivor assertions count ReplacingMergeTree winners (mask-applied), aligning with 000005; 9/9 green. - verify.sh: an empty OLD_TABLE now fails if NEW_TABLE has rows (unexplained successor data) instead of exiting 0 "nothing to verify"; both-empty still exits 0. Co-Authored-By: Claude Opus 4.8 --- .../traces-local-v2-cutover/scripts/verify.sh | 11 +++++++++-- .../opik/infrastructure/TracesLocalV2CutoverTest.java | 4 ++-- .../tests/traces-local-v2-cutover/seed_history.py | 5 ++++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh index 7f568ed2c0c..87b9259179c 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh @@ -110,8 +110,15 @@ drill_down_window() { ROWS="$(ch "SELECT count() FROM $OLD_TABLE")" if [[ "$ROWS" == "0" ]]; then - log "Old-schema table '$OLD_TABLE' is empty — nothing to verify." - exit 0 + # An empty old table is only "nothing to verify" if the new table is ALSO empty. If the successor has rows the + # source doesn't, that's an unexplained divergence (extra destination rows) — fail rather than declare success. + NEW_ROWS="$(ch "SELECT count() FROM $NEW_TABLE")" + if [[ "$NEW_ROWS" == "0" ]]; then + log "Both '$OLD_TABLE' and '$NEW_TABLE' are empty — nothing to verify." + exit 0 + fi + log "FAILED: '$OLD_TABLE' is empty but '$NEW_TABLE' has $NEW_ROWS row(s) — the successor holds data the source does not." >&2 + exit 1 fi # Week range from the old table's created_at (bounded and real; covers rows whose id_at is far-future from the bad-id bug diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java index 7b3ff918271..4837885d737 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java @@ -1228,7 +1228,7 @@ private long liveCount(String table, Set ids, String workspaceId) { } var sql = """ SELECT uniqExact(id) AS c - FROM %s + FROM %s FINAL WHERE workspace_id = :workspace_id AND id IN :ids """.formatted(table); @@ -1245,7 +1245,7 @@ SELECT uniqExact(id) AS c private long liveCountScoped(String table, Set ids, String workspaceId, UUID projectId) { var sql = """ SELECT uniqExact(id) AS c - FROM %s + FROM %s FINAL WHERE workspace_id = :workspace_id AND project_id = :project_id AND id IN :ids diff --git a/tests_load/tests/traces-local-v2-cutover/seed_history.py b/tests_load/tests/traces-local-v2-cutover/seed_history.py index 7da4bcf30d0..faaf803e205 100644 --- a/tests_load/tests/traces-local-v2-cutover/seed_history.py +++ b/tests_load/tests/traces-local-v2-cutover/seed_history.py @@ -172,7 +172,10 @@ def main(project, weeks, per_week, bad_ids, batch, workspace_id, project_id): LOGGER.info("Per-week seeded (created_at week -> count): %s", {k: per_week_counts[k] for k in sorted(per_week_counts)}) if bad_ids: - LOGGER.info("Plus %d far-future-id rows (id_at ~2201) to exercise the bad-id partition path.", bad_ids) + LOGGER.info( + "Plus %d far-future-id rows (litellm UUIDv7 ~2201) to exercise the bad-id partition path. NOTE: the " + "successor's 32-bit DateTime id_at wraps ~2201 to ~2065, so look for these rows in the ~2065 weekly " + "partition, not 2201.", bad_ids) if __name__ == "__main__": From 936f107a03805b27af492051b1e3f47a2f49802d Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Mon, 27 Jul 2026 11:08:32 +0200 Subject: [PATCH 20/28] test(cutover): final deletion replay before EXCHANGE; version-collapse fails REVIEW-REQUIRED MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #687 — close the delete coverage hole: delta_replay.sh replays deletes only up to when it runs, but cutover_start is captured later in exchange_and_wrap.sh, so deletes bridged in [final delta_replay, cutover_start) were covered by neither the forward replay nor the rollback reverse-replay and leaked live across the swap. exchange_and_wrap.sh now takes --backfill-start (required for every EXCHANGE path) and, right after capturing cutover_start, runs a final deletion replay — reusing the single-source 000002 deletion-replay block (now marked so it can be extracted without re-running the delta INSERT; writes in the gap are carried by the async buffer). Runbook + a gate-test case added; residual is the inherent in-flight micro-window, documented. #717 — verify.sh no longer only warns on version-collapse: a non-zero count is now a REVIEW-REQUIRED failure (exit 3, distinct from a fidelity mismatch exit 1) unless the operator reviews and passes --allow-version-collapse; --drill-down lists the offending keys (new 000005 version-check-drill block). Gate test 10/10 green on CH 26.3. Co-Authored-By: Claude Opus 4.8 --- .../traces-local-v2-cutover/README.md | 13 ++++-- .../000002_delta_and_deletion_replay.sql | 4 ++ .../000005_verify_migration.sql | 11 +++++ .../scripts/exchange_and_wrap.sh | 33 +++++++++++++++ .../traces-local-v2-cutover/scripts/verify.sh | 39 +++++++++++++---- .../TracesLocalV2CutoverTest.java | 42 +++++++++++++++++++ 6 files changed, 130 insertions(+), 12 deletions(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md index cda93600945..d4c7022f5b6 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md @@ -198,12 +198,17 @@ and the `EXCHANGE` completing must stay within the buffer hold**. So run the tai 2. Do the QA verify on an **earlier** pass (it can take minutes on a large table — do not let it be the last thing before the swap). 3. Run a **final** `delta_replay.sh` as the last write-facing step. -4. Run `exchange_and_wrap.sh` **immediately** after it (the settle gate + `EXCHANGE` are fast and metadata-only). +4. Run `exchange_and_wrap.sh --backfill-start '' …` **immediately** after it (the settle gate + `EXCHANGE` are + fast and metadata-only). It captures `cutover_start`, then runs a **final deletion replay** from `backfill_start` + right before the swap — so deletes bridged in the `[final delta_replay, cutover_start)` gap are masked on the + successor rather than leaking (that gap is covered by neither the earlier forward replay nor the rollback + reverse-replay, which starts at `cutover_start`). Deletions only; the buffer carries the writes. 5. Restore the buffer ceiling; parked inserts flush into the successor. -Keep step 3→4 short. If load is high enough that a size-triggered flush is plausible in that gap, re-run the final delta -once more right before the swap. Everything up to `cutover_start` is covered by the delta+replay; the buffer covers the -instant of the flip. +Keep step 3→4 short. **Deletes** up to `cutover_start` are covered by step 4's final deletion replay; **writes** in the +gap are covered by the buffer (which flushes into the successor after the flip). The one residual is a delete whose +bridge row commits after that final replay's read but with `event_time < cutover_start` — the same inherent micro-window +as a size-triggered buffer flush; if delete load is high, quiesce user deletes for the final seconds. **The `traceColumnsNonNullable` flip (mandatory, and why it goes first).** The successor stores `end_time`/`ttft` as non-nullable epoch/NaN sentinels; the app must bind those sentinels — not `null` — the moment that schema is live under diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000002_delta_and_deletion_replay.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000002_delta_and_deletion_replay.sql index 9eff4280af3..eda061f574f 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000002_delta_and_deletion_replay.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000002_delta_and_deletion_replay.sql @@ -32,6 +32,7 @@ -- (a) created_at >= backfill_start -- batch by created_at sub-windows -- (b) last_updated_at >= backfill_start AND created_at < backfill_start -- the updates-to-old-rows arm; batch by -- last_updated_at sub-windows. (a) ∪ (b) equals the OR below, with no overlap. +-- >>> BEGIN delta-insert INSERT INTO ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 ( id, workspace_id, @@ -86,6 +87,7 @@ WHERE created_at >= toDateTime64('${BACKFILL_START}', 6) OR last_updated_at >= toDateTime64('${BACKFILL_START}', 6) SETTINGS max_insert_block_size = ${MAX_INSERT_BLOCK_SIZE}, log_comment = 'traces_local_v2_cutover:delta_insert'; +-- >>> END delta-insert -- Step 4: Deletion replay — remove from the destination every row that was deleted on the source since backfill_start -- AND is still deleted there. Two branches, mirroring the product's two delete paths (TraceService.delete): a delete @@ -121,6 +123,7 @@ SETTINGS max_insert_block_size = ${MAX_INSERT_BLOCK_SIZE}, -- replay on a single malformed bridge row. For source_table='traces' the ids are 36-char UUIDs, so this is latent — but -- a malformed (non-36-char) deleted_id/project_id can't match a real trace id anyway, so skipping it via the length -- guard loses nothing and turns a hard abort mid-cutover into a benign no-op. Same guards in the reverse-replay. +-- >>> BEGIN deletion-replay DELETE FROM ${ANALYTICS_DB_DATABASE_NAME}.traces_local_v2 WHERE ( (workspace_id, project_id, id) IN ( @@ -178,6 +181,7 @@ OR ( SETTINGS allow_nondeterministic_mutations = 1, lightweight_deletes_sync = 2, log_comment = 'traces_local_v2_cutover:deletion_replay'; +-- >>> END deletion-replay -- Step 5: Measure the replay. Compare its wall time against the buffer window (must fit with margin — acceptance -- criterion). Re-run steps 3-4 if new rows/deletes accumulated during the replay itself; convergence is fast because diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000005_verify_migration.sql b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000005_verify_migration.sql index ee9c1b6ff20..87f5bec61ca 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000005_verify_migration.sql +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/db-app-analytics/000005_verify_migration.sql @@ -204,3 +204,14 @@ FROM ( HAVING uniqExact(last_updated_at) > uniqExact(toDateTime64(last_updated_at, 6)) ); -- >>> END version-check + +-- >>> BEGIN version-check-drill +-- Up to 100 of the keys behind a non-zero version-check, for investigation (verify.sh --drill-down). +SELECT workspace_id, project_id, id +FROM ${ANALYTICS_DB_DATABASE_NAME}.${OLD_TABLE} +WHERE created_at >= toDateTime64('${WINDOW_LO}', 9) + AND created_at < toDateTime64('${WINDOW_HI}', 9) +GROUP BY workspace_id, project_id, id +HAVING uniqExact(last_updated_at) > uniqExact(toDateTime64(last_updated_at, 6)) +LIMIT 100; +-- >>> END version-check-drill diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh index f68fab0028c..9350b51e2ec 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh @@ -19,6 +19,10 @@ # # Options: # --database NAME analytics database (e.g. opik). Required. +# --backfill-start TS the anchor printed by backfill.sh. REQUIRED for every EXCHANGE path (not --wrap-only): just +# before the swap this runs a final deletion replay from that anchor, so deletes bridged since the +# last delta_replay.sh don't leak live across the EXCHANGE (they'd be covered by neither the forward +# replay nor the rollback reverse-replay otherwise). # (default) run ONLY the EXCHANGE (the data cutover), then stop — leaves `traces` a MergeTree where deletes # still work. The Distributed wrap is deferred (see above). # --with-wrap also apply the Distributed wrap in the same run (EXCHANGE + wrap). Use only once the delete/read @@ -48,8 +52,10 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SQL_FILE="$SCRIPT_DIR/db-app-analytics/000003_exchange_and_wrap.sql" +DELTA_SQL_FILE="$SCRIPT_DIR/db-app-analytics/000002_delta_and_deletion_replay.sql" DATABASE="" +BACKFILL_START="" SKIP_WRAP=0 WITH_WRAP=0 WRAP_ONLY=0 @@ -61,6 +67,7 @@ CONFIRM_BUFFER_RAISED=0 while [[ $# -gt 0 ]]; do case "$1" in --database) DATABASE="${2:?"$1 requires a value"}"; shift 2 ;; + --backfill-start) BACKFILL_START="${2:?"$1 requires a value"}"; shift 2 ;; --skip-wrap) SKIP_WRAP=1; shift ;; --with-wrap) WITH_WRAP=1; shift ;; --wrap-only) WRAP_ONLY=1; shift ;; @@ -76,6 +83,9 @@ done # --database is interpolated into the reference SQL; require a plain ClickHouse identifier so it cannot alter the query. [[ "$DATABASE" =~ ^[A-Za-z0-9_]+$ ]] || { echo "ERROR: --database must be a ClickHouse identifier (letters, digits, underscore)." >&2; exit 2; } [[ -f "$SQL_FILE" ]] || { echo "ERROR: cannot find $SQL_FILE" >&2; exit 2; } +[[ -f "$DELTA_SQL_FILE" ]] || { echo "ERROR: cannot find $DELTA_SQL_FILE" >&2; exit 2; } +# --backfill-start (the anchor printed by backfill.sh) is interpolated into the final deletion replay; validate its shape. +[[ -z "$BACKFILL_START" || "$BACKFILL_START" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}\ [0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?$ ]] || { echo "ERROR: --backfill-start must be 'YYYY-MM-DD HH:MM:SS[.ffffff]'." >&2; exit 2; } # At most one wrap mode. Default (none set) is EXCHANGE only. if (( SKIP_WRAP + WITH_WRAP + WRAP_ONLY > 1 )); then echo "ERROR: --skip-wrap, --with-wrap and --wrap-only are mutually exclusive" >&2; exit 2 @@ -108,6 +118,12 @@ if [[ "$WRAP_ONLY" != "1" && "$CONFIRM_BUFFER_RAISED" != "1" ]]; then echo " window can commit to the old table and be lost after the EXCHANGE — then re-run with the flag." >&2 exit 2 fi +# The EXCHANGE runs a final deletion replay first (see below), to mask deletes bridged since the last delta_replay so +# they don't leak live across the swap — that needs the same backfill_start anchor delta_replay.sh used. +if [[ "$WRAP_ONLY" != "1" && -z "$BACKFILL_START" ]]; then + echo "ERROR: the EXCHANGE requires --backfill-start (the anchor printed by backfill.sh) for the final deletion replay." >&2 + exit 2 +fi ch() { clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_cutover:exchange_and_wrap' --query "$1" @@ -233,6 +249,20 @@ run_block() { clickhouse-client --database "$DATABASE" --multiquery --query "$sql" } +# Final deletion replay before the EXCHANGE. delta_replay.sh (step 2) replayed deletes only up to when it ran; cutover_start +# is captured HERE, so a delete bridged in that final gap would be covered by neither the forward replay nor the rollback +# reverse-replay (which starts at cutover_start) and would leak live across the swap. Re-running the deletion-replay block +# (from the single-source 000002) right after capturing cutover_start extends forward coverage to it — the arm is +# idempotent and user-scale (retention off), so it is cheap. Deletions only: writes in the gap are held by the async +# buffer and flush onto the successor after the swap. +run_final_deletion_replay() { + local sql + sql="$(awk -v begin="-- >>> BEGIN deletion-replay" -v end="-- >>> END deletion-replay" '$0 == begin {f = 1; next} $0 == end {f = 0} f' "$DELTA_SQL_FILE")" + sql="${sql//'${ANALYTICS_DB_DATABASE_NAME}'/$DATABASE}" + sql="${sql//'${BACKFILL_START}'/$BACKFILL_START}" + clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_cutover:exchange_and_wrap:final_deletion_replay' --multiquery --query "$sql" +} + if [[ "$WRAP_ONLY" == "1" ]]; then # Deferred second half: the EXCHANGE already happened in a prior --skip-wrap run, so `traces` is the live # partitioned data. Do not re-EXCHANGE (that would swap the parked original back in) and do not capture a new @@ -252,6 +282,9 @@ fi CUTOVER_START="$(clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_cutover:exchange_and_wrap' --query "SELECT toString(now64(6))")" echo "RECORD cutover_start=$CUTOVER_START (pass to rollback.sh --cutover-start if you roll back after this point)" +echo "Final deletion replay: masking deletes bridged since the last delta_replay so none leak across the swap..." +run_final_deletion_replay + run_block exchange echo "EXCHANGE done: 'traces' is now the partitioned data; the old data is parked as 'traces_pre_cutover_backup'." diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh index 87b9259179c..c2fe1e19544 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/verify.sh @@ -26,7 +26,11 @@ # --from-week N start at week offset N (0-based from the anchor Monday). Default 0. # --to-week M stop after week offset M (inclusive). Default: last week with data. # --weeks-stride S compare every S-th week (S>1 samples partitions for a quick pass). Default 1. -# --drill-down on a mismatched week, also print up to 100 keys that differ or exist on one side only. +# --drill-down on a mismatched week, also print up to 100 keys that differ or exist on one side only (also +# lists the version-collapse keys per week). +# --allow-version-collapse proceed (exit 0) despite version-collapse keys after reviewing them. Without it, a non-zero +# version-collapse count is a REVIEW-REQUIRED failure (exit 3) — distinct from a fidelity mismatch +# (exit 1) — because the microsecond fingerprint cannot verify version selection for those keys. set -euo pipefail @@ -41,6 +45,7 @@ FROM_WEEK=0 TO_WEEK="" WEEKS_STRIDE=1 # 1 = every week; S skips to every S-th weekly partition for a quick, pruned pass DRILL_DOWN=0 +ALLOW_VERSION_COLLAPSE=0 while [[ $# -gt 0 ]]; do case "$1" in @@ -52,6 +57,7 @@ while [[ $# -gt 0 ]]; do --to-week) TO_WEEK="${2:?"$1 requires a value"}"; shift 2 ;; --weeks-stride) WEEKS_STRIDE="${2:?"$1 requires a value"}"; shift 2 ;; --drill-down) DRILL_DOWN=1; shift ;; + --allow-version-collapse) ALLOW_VERSION_COLLAPSE=1; shift ;; *) echo "Unknown argument: $1" >&2; exit 2 ;; esac done @@ -103,6 +109,11 @@ version_check_window() { clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_cutover:verify' --multiquery --query "$(render_block version-check "$1" "$2")" } +# Up to 100 keys behind a non-zero version_check (only under --drill-down). +version_check_drill_window() { + clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_cutover:verify' --multiquery --query "$(render_block version-check-drill "$1" "$2")" +} + # Per-key differences for one window (only run on a mismatch, under --drill-down). drill_down_window() { clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_cutover:verify' --multiquery --query "$(render_block drill-down "$1" "$2")" @@ -156,20 +167,32 @@ for (( week=FROM_WEEK; week<=TO_WEEK; week+=WEEKS_STRIDE )); do fi fi - # ns->us version-selection guard (does not affect pass/fail — the fingerprint can't see it, so it is reported - # separately for the operator to investigate rather than silently trusted). + # ns->us version-selection guard. Separate from the fidelity verdict: the fingerprint can't see it, so it is + # surfaced and gates the run separately (REVIEW REQUIRED) rather than being silently trusted. collapse_out="$(version_check_window "$LO" "$HI")" if [[ -n "$collapse_out" && "$collapse_out" != "0" ]]; then version_collapse=$((version_collapse + collapse_out)) - log " NOTE week $week: $collapse_out key(s) on $OLD_TABLE have sub-microsecond-distinct last_updated_at — the ns->us truncation may pick a different version than source FINAL for them; the fingerprint cannot detect it. Investigate." >&2 + log " NOTE week $week: $collapse_out key(s) on $OLD_TABLE have sub-microsecond-distinct last_updated_at — the ns->us truncation may pick a different version than source FINAL for them; the fingerprint cannot detect it." >&2 + if [[ "$DRILL_DOWN" == "1" ]]; then + log " version-collapse keys (workspace_id, project_id, id):" >&2 + version_check_drill_window "$LO" "$HI" >&2 + fi fi done -if [[ "$version_collapse" != "0" ]]; then - log "WARNING: $version_collapse key(s) across the checked weeks have sub-microsecond-distinct last_updated_at on $OLD_TABLE. The microsecond fingerprint cannot see version-selection differences for them, so investigate those keys before trusting a PASS as zero-loss." >&2 -fi +# Fidelity mismatch is the hard failure (exit 1). Check it first so a real data difference takes precedence. if [[ "$mismatches" != "0" ]]; then log "FAILED: $mismatches of $checked windows mismatched." >&2 exit 1 fi -log "PASSED: all $checked windows match (sample 1/$SAMPLE_MOD). Version-collapse keys: $version_collapse." +# Version-collapse is NOT a proven mismatch — it is an unverifiable region. Fail REVIEW-REQUIRED (exit 3, distinct from +# a fidelity FAILED=1) unless the operator has reviewed the keys and accepts them via --allow-version-collapse. +if [[ "$version_collapse" != "0" ]]; then + if [[ "$ALLOW_VERSION_COLLAPSE" == "1" ]]; then + log "PASSED (with review override): all $checked windows match; $version_collapse version-collapse key(s) accepted via --allow-version-collapse." + exit 0 + fi + log "REVIEW REQUIRED: all $checked windows match, but $version_collapse key(s) on $OLD_TABLE have sub-microsecond-distinct last_updated_at — the microsecond fingerprint cannot verify version selection for them (NOT a data mismatch). Re-run with --drill-down to list them; once confirmed benign, re-run with --allow-version-collapse." >&2 + exit 3 +fi +log "PASSED: all $checked windows match (sample 1/$SAMPLE_MOD). Version-collapse keys: 0." diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java index 4837885d737..5e2ff06d72e 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java @@ -731,6 +731,48 @@ void workspaceScopedReplaySparesLiveCrossProjectRow() { .isEqualTo(1L); } + /** + * A delete bridged AFTER the main (step-2) replay but before the EXCHANGE must be masked by the final deletion replay + * that {@code exchange_and_wrap.sh} runs right after capturing {@code cutover_start}. Otherwise it is covered by + * neither the forward replay (already ran) nor the rollback reverse-replay ({@code event_time >= cutover_start}) and + * leaks live across the swap. This pins that final-replay step (mirrors the driver's fold-in of the 000002 + * deletion-replay block into the exchange step). + */ + @Test + void finalReplayBeforeExchangeMasksDeletesBridgedAfterTheMainReplay() { + var workspaceId = UUID.randomUUID().toString(); + var projectId = ID_GENERATOR.generateId(); + var survivors = mintIds(SURVIVORS_PER_WEEK); + var gapDeleted = mintIds(3); // deleted in the [main replay, EXCHANGE] gap + seedTraces(survivors, workspaceId, projectId); + seedTraces(gapDeleted, workspaceId, projectId); + + var backfillStart = nowMicros(); + for (int week = 0; week < SEED_WEEKS; week++) { + backfillWeek(week); + } + deltaInsert(backfillStart); + replayDeletions(backfillStart); // step 2 (delta_replay.sh) — runs before the gap delete below + + // A delete lands AFTER the main replay. Without a final replay before the swap it leaks onto the successor. + recordDeletionEvents(idStrings(gapDeleted), workspaceId, projectId.toString(), "user_request"); + lightweightDelete(idStrings(gapDeleted), workspaceId); + assertThat(liveCount("traces_local_v2", idStrings(gapDeleted), workspaceId)) + .as("negative control: the gap delete has leaked onto the successor before the final replay") + .isEqualTo(gapDeleted.size()); + + // exchange_and_wrap.sh runs this final deletion replay right after capturing cutover_start, before the EXCHANGE. + replayDeletions(backfillStart); + exchangeTables(); + + assertThat(liveCount("traces", idStrings(gapDeleted), workspaceId)) + .as("final deletion replay masks the gap delete — 0 leaks across the swap") + .isZero(); + assertThat(liveCount("traces", idStrings(survivors), workspaceId)) + .as("survivors intact after the final replay + EXCHANGE") + .isEqualTo(survivors.size()); + } + /** * Schema-drift guard. The cutover copies a fixed column list, and the fidelity fingerprint also lists fixed * columns — so a base column added to {@code traces} by a future migration would be silently left uncopied, with no From 9f85b7447d8597d9dab29baba0a78f99c63eeb46 Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Mon, 27 Jul 2026 11:50:03 +0200 Subject: [PATCH 21/28] test(cutover): server-clock resurrection instant + explicit container teardown - deleteThenResurrectSurvivesTheReplay: source the recreated row's last_updated_at from the CH server clock (now64(6) -> Instant) instead of JVM Instant.now(), so host<->container skew can't put it below backfillStart and make the delta miss the resurrection path (same fix already applied to the main test). - Add an @AfterAll (PER_CLASS, non-static) that stops the ClickHouse + ZooKeeper containers and closes the network, so the dedicated non-reused containers don't rely solely on the Ryuk reaper across reruns / a shared JVM. Gate test 10/10 green. Co-Authored-By: Claude Opus 4.8 --- .../infrastructure/TracesLocalV2CutoverTest.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java index 5e2ff06d72e..f92d04c6279 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java @@ -10,6 +10,7 @@ import io.r2dbc.spi.Statement; import lombok.Builder; import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -215,6 +216,15 @@ class TracesLocalV2CutoverTest { ClickHouseContainerUtils.newDatabaseAnalyticsFactory(clickHouseContainer, DATABASE_NAME).build()); } + // Dedicated (non-reused) containers, so tear them down explicitly rather than relying only on the Ryuk reaper — + // keeps reruns and a shared JVM from accumulating stopped-but-lingering resources. PER_CLASS lets this be non-static. + @AfterAll + void stopContainers() { + clickHouseContainer.stop(); + zookeeperContainer.stop(); + network.close(); + } + /** * Restore the canonical baseline (traces = original schema, traces_local_v2 = successor schema, both empty; no stray * wrap/rename artifacts) before every test, independent of what the previous test left behind. A green run always @@ -666,7 +676,10 @@ void deleteThenResurrectSurvivesTheReplay(boolean resolvedProject) { lightweightDelete(idStrings(resurrected), workspaceId); recordDeletionEvents(idStrings(stayDeleted), workspaceId, captureProject, "user_request"); lightweightDelete(idStrings(stayDeleted), workspaceId); - insertRows(resurrected, workspaceId, projectId, "resurrected", _ -> Instant.now()); + // Recreate with a server-clock last_updated_at (a later now64(6), so >= backfillStart) — NOT the JVM clock, + // whose skew vs the container could put it below backfillStart and make the delta miss the resurrection path. + var resurrectedAt = Instant.from(ClickHouseDateTimeFormat.MICROS.parse(nowMicros())); + insertRows(resurrected, workspaceId, projectId, "resurrected", _ -> resurrectedAt); deltaInsert(backfillStart); // Run the replay twice: it must be idempotent (re-runnable to convergence) and must not drop the resurrected From c731512a528c7fcab860ace00beb12bf14410e13 Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Mon, 27 Jul 2026 12:01:29 +0200 Subject: [PATCH 22/28] fix(cutover): gate EXCHANGE/rollback on retention-paused; make delete_traffic best-effort MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address Baz round: - exchange_and_wrap.sh / rollback.sh: require --confirm-retention-paused on every EXCHANGE path and on rollback stages B/C. Retention deletes bypass the deletion bridge, so a sweep in the cutover/rollback window leaks live across the swap (forward) or resurrects a deleted row from the backup (reverse-replay). It is an operator assertion — a backend setting the script can't read — so the README Go/No-Go keeps the real "actually paused on every backend" verification. - delete_traffic.py (#851): larger newest-page refill and more generous empty-refill tolerance so mask-visibility lag doesn't end the run early; docstring states it is a best-effort traffic generator, not a guaranteed full-drain. Co-Authored-By: Claude Opus 4.8 --- .../traces-local-v2-cutover/README.md | 14 ++++++++++---- .../scripts/exchange_and_wrap.sh | 16 ++++++++++++++++ .../traces-local-v2-cutover/scripts/rollback.sh | 17 +++++++++++++++-- .../traces-local-v2-cutover/delete_traffic.py | 15 ++++++++++++--- 4 files changed, 53 insertions(+), 9 deletions(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md index d4c7022f5b6..ac281d42511 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md @@ -144,10 +144,13 @@ new table before the EXCHANGE. The replay matches the **full key**, not `id` alo leaving `traces` a `MergeTree` where deletes still work); the `RENAME` + `Distributed` wrap runs only with `--with-wrap`. Restore the buffer ceiling and verify. ```bash - CLICKHOUSE_HOST= CLICKHOUSE_PASSWORD= ./scripts/exchange_and_wrap.sh --database opik --confirm-buffer-raised + CLICKHOUSE_HOST= CLICKHOUSE_PASSWORD= ./scripts/exchange_and_wrap.sh --database opik \ + --backfill-start '' --confirm-buffer-raised --confirm-retention-paused ``` - `--confirm-buffer-raised` is required on every EXCHANGE path: it asserts the async-insert buffer is raised so writes in - the final window survive the swap (add `--with-wrap --confirm-daos-retargeted` only once the DAOs target `traces_local`). + Every EXCHANGE path requires: `--backfill-start` (for the final deletion replay), `--confirm-buffer-raised` (writes in + the final window survive the swap), and `--confirm-retention-paused` (retention deletes bypass the bridge, so a + retention sweep in the window would leak across the swap). Add `--with-wrap --confirm-daos-retargeted` only once the + DAOs target `traces_local`. > **HARD PREREQUISITE for the wrap (step 4, part 2): the delete/mutation DAO must target `traces_local` first.** A > `Distributed` table supports `SELECT` and `INSERT` but **not** mutations. Verified on ClickHouse 26.3: @@ -613,7 +616,10 @@ cheap (stage A); the bridge stays enabled so nothing is lost on a retry. - [ ] **Async-insert ceiling confirmed** — raising `asyncInsertBusyTimeoutMaxMs` demonstrably widens the adaptive buffer under load, not just the cap. `exchange_and_wrap.sh` enforces the acknowledgment via `--confirm-buffer-raised`, but that is an assertion only — this checklist item is the actual "it took effect under load" verification. -- [ ] **Data Retention confirmed disabled** for the cutover window (`RETENTION_ENABLED=false`). +- [ ] **Data Retention confirmed disabled** for the cutover window (`RETENTION_ENABLED=false`). Retention deletes bypass + the deletion bridge, so a sweep in the window would leak/resurrect across the swap; `exchange_and_wrap.sh` and + `rollback.sh` (stages B/C) enforce `--confirm-retention-paused`, but that is an assertion — this item is the real + "it is actually paused on every backend" verification. - [ ] **Reconciliation clean** — per-window source/dest counts within 0.01% across the whole backfill. - [ ] **Replication settled before the EXCHANGE** — `replication_queue` empty and the deletion-replay mutation `is_done` on **all** replicas (`exchange_and_wrap.sh` gates on this; do not `--force` past it in production). diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh index 9350b51e2ec..160bc94683f 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/exchange_and_wrap.sh @@ -47,6 +47,10 @@ # writes across the swap so they land on the new table; at the default, a write in the final window # can commit to the old table and be lost after the EXCHANGE. It's a backend setting the script can't # read, so the operator must assert it. +# --confirm-retention-paused REQUIRED for every EXCHANGE path. Retention deletes (deleteForRetention*) bypass the +# deletion bridge and are never replayed onto the successor, so a retention sweep during the cutover +# window leaks live across the swap. Asserts retention is paused (RETENTION_ENABLED=false on every +# backend) for the whole window — a backend setting the script can't read. set -euo pipefail @@ -63,6 +67,7 @@ FORCE=0 CONFIRM_MAINTENANCE=0 CONFIRM_DAOS_RETARGETED=0 CONFIRM_BUFFER_RAISED=0 +CONFIRM_RETENTION_PAUSED=0 while [[ $# -gt 0 ]]; do case "$1" in @@ -75,6 +80,7 @@ while [[ $# -gt 0 ]]; do --confirm-maintenance) CONFIRM_MAINTENANCE=1; shift ;; --confirm-daos-retargeted) CONFIRM_DAOS_RETARGETED=1; shift ;; --confirm-buffer-raised) CONFIRM_BUFFER_RAISED=1; shift ;; + --confirm-retention-paused) CONFIRM_RETENTION_PAUSED=1; shift ;; *) echo "Unknown argument: $1" >&2; exit 2 ;; esac done @@ -124,6 +130,16 @@ if [[ "$WRAP_ONLY" != "1" && -z "$BACKFILL_START" ]]; then echo "ERROR: the EXCHANGE requires --backfill-start (the anchor printed by backfill.sh) for the final deletion replay." >&2 exit 2 fi +# Retention deletes (TraceDAO.deleteForRetention*) bypass the deletion bridge, so they are never replayed onto the +# successor — if any backend still has RETENTION_ENABLED=true, a retention sweep in the cutover window leaks live across +# the swap. Retention is a backend setting the script can't read, so require the operator to assert it is paused for the +# whole window. Applies to every EXCHANGE path (not --wrap-only, which does no data cutover). +if [[ "$WRAP_ONLY" != "1" && "$CONFIRM_RETENTION_PAUSED" != "1" ]]; then + echo "ERROR: the EXCHANGE requires --confirm-retention-paused. Retention deletes bypass the deletion bridge, so a" >&2 + echo " retention sweep during the cutover window would leak live across the swap. Pause retention" >&2 + echo " (RETENTION_ENABLED=false on every backend) for the whole window, then re-run with the flag." >&2 + exit 2 +fi ch() { clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_cutover:exchange_and_wrap' --query "$1" diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh index 792f36571e2..1f8e3202417 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh @@ -7,8 +7,9 @@ # --stage A backfill/delta ran but the EXCHANGE did not — discard the shadow (live `traces` is untouched). # --stage B the EXCHANGE ran but not the wrap — swap the tables back, then reverse-replay. # --stage C the wrap ran — drop the wrapper, promote the parked original, then reverse-replay. -# Stages B and C need --cutover-start (printed by exchange_and_wrap.sh) to bound the reverse-replay. Keep the deletion -# bridge enabled through the rollback so no delete is lost. +# Stages B and C need --cutover-start (printed by exchange_and_wrap.sh) to bound the reverse-replay, and +# --confirm-retention-paused (retention deletes bypass the bridge, so a retention sweep in the rollback window would +# resurrect a deleted row from the backup). Keep the deletion bridge enabled through the rollback so no delete is lost. # # SAFETY: the stages are mutually exclusive and each lives in its OWN file, so no single file mixes a TRUNCATE with an # EXCHANGE/DROP — running any file does exactly one stage. Before running, this asserts the live `traces` topology matches @@ -26,12 +27,14 @@ SQL_DIR="$SCRIPT_DIR/db-app-analytics" DATABASE="" STAGE="" CUTOVER_START="" +CONFIRM_RETENTION_PAUSED=0 while [[ $# -gt 0 ]]; do case "$1" in --database) DATABASE="${2:?"$1 requires a value"}"; shift 2 ;; --stage) STAGE="${2:?"$1 requires a value"}"; shift 2 ;; --cutover-start) CUTOVER_START="${2:?"$1 requires a value"}"; shift 2 ;; + --confirm-retention-paused) CONFIRM_RETENTION_PAUSED=1; shift ;; *) echo "Unknown argument: $1" >&2; exit 2 ;; esac done @@ -44,6 +47,16 @@ case "$STAGE" in A|B|C) ;; *) echo "ERROR: --stage must be A, B or C" >&2; exit 2 ;; esac +# Stages B/C run the reverse-replay, which — like the forward replay — only re-applies bridged deletes. Retention deletes +# (deleteForRetention*) bypass the bridge, so a retention sweep during the rollback window would restore a legitimately +# deleted row from the backup and resurrect it. Retention is a backend setting the script can't read; require the +# operator to assert it is paused. Stage A does no reverse-replay, so it is exempt. +if [[ ( "$STAGE" == "B" || "$STAGE" == "C" ) && "$CONFIRM_RETENTION_PAUSED" != "1" ]]; then + echo "ERROR: rollback --stage $STAGE requires --confirm-retention-paused. The reverse-replay only re-applies bridged" >&2 + echo " deletes; a retention sweep in the rollback window would resurrect a deleted row from the backup." >&2 + echo " Pause retention (RETENTION_ENABLED=false on every backend), then re-run with the flag." >&2 + exit 2 +fi ch() { clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_rollback' --query "$1" diff --git a/tests_load/tests/traces-local-v2-cutover/delete_traffic.py b/tests_load/tests/traces-local-v2-cutover/delete_traffic.py index c1427a9c93b..b902b624913 100644 --- a/tests_load/tests/traces-local-v2-cutover/delete_traffic.py +++ b/tests_load/tests/traces-local-v2-cutover/delete_traffic.py @@ -7,6 +7,11 @@ It pulls a pool of existing trace ids via search and deletes them at the target rate, refilling as it drains. Run it during or after the backfill so the traces it deletes have already been copied — that is the leak the bridge prevents. +This is a best-effort TRAFFIC GENERATOR (deletes newest-first for the run's --duration), NOT a guaranteed full-drain: +search returns the newest page, so during delete-mask visibility lag a refill can transiently return only ids already +in `seen`/`pool`; the loop tolerates a few such empty refills (EMPTY_REFILL_LIMIT) before concluding the project is +drained. It does not assert every trace was deleted — its job is to exercise the deletion bridge, not to empty the table. + Prerequisites: `OPIK_URL_OVERRIDE` pointing at the local install. Run `python delete_traffic.py --help` for options. """ @@ -20,8 +25,12 @@ _stop = False # Consecutive empty refills tolerated before concluding the project is drained. An empty refill can be a transient -# delete-mask-visibility lag (the just-deleted top ids not yet hidden from search), not a truly empty project. -EMPTY_REFILL_LIMIT = 3 +# delete-mask-visibility lag (the just-deleted top ids not yet hidden from search), not a truly empty project — so give +# the mask several beats to propagate before stopping. +EMPTY_REFILL_LIMIT = 5 +# Newest-page size to pull per refill. Larger reaches past the just-deleted (still-visible) top ids to undeleted ones +# during mask lag, so the run keeps finding work instead of stopping early. +REFILL_FETCH = 2000 def _handle_sigint(_signum, _frame): @@ -61,7 +70,7 @@ def main(project, tps, duration, batch): tick = time.time() if len(pool) < batch: # Exclude both already-deleted ids and those still queued in `pool`, so a refill can't requeue an in-flight id. - fetched = _fetch_ids(client, project, want=500, exclude=seen | set(pool)) + fetched = _fetch_ids(client, project, want=REFILL_FETCH, exclude=seen | set(pool)) if fetched is None: # transient search failure — back off and retry rather than mistaking it for "no more traces". time.sleep(interval if interval > 0 else 0.5) From adfd0f1e9c2c9bb6cb4b62ef60f96e4c4ac449f6 Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Mon, 27 Jul 2026 12:30:41 +0200 Subject: [PATCH 23/28] fix(cutover): resume only on dst>=src; residual full-compare gate; test hygiene MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address Baz round (items 1-5): - backfill.sh (#916, high): resume skips a window only when dst >= src. DIVERGENCE is no longer a resume criterion — a partially-copied window within tolerance would be skipped and its pre-anchor rows lost forever (delta only re-copies at/after backfill_start). The INSERT is idempotent, so re-copying a short window is safe; DIVERGENCE still governs the post-copy abort. - README (#666, high): the pre-EXCHANGE fidelity gate must be a full compare (--sample-mod 1 --weeks-stride 1, no week narrowing); a single cross-project workspace-scoped residual row can be hashed out or week-skipped by sampling. - delete_traffic.py (#890): comment referenced newest 500; now references REFILL_FETCH. - TracesLocalV2CutoverTest (#901): wrapInDistributed() uses a text block + .formatted(). - _common.py (#922): delete the cutover-anchor trace in a finally block so discovery leaves no extra live trace skewing seeded counts. Gate test 10/10. Co-Authored-By: Claude Opus 4.8 --- .../traces-local-v2-cutover/README.md | 7 ++-- .../scripts/backfill.sh | 10 ++++-- .../TracesLocalV2CutoverTest.java | 9 ++--- .../tests/traces-local-v2-cutover/_common.py | 34 ++++++++++++------- .../traces-local-v2-cutover/delete_traffic.py | 6 ++-- 5 files changed, 42 insertions(+), 24 deletions(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md index ac281d42511..5b42f9fee00 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md @@ -629,8 +629,11 @@ cheap (stage A); the bridge stays enabled so nothing is lost on a retry. - [ ] **Schema-parity guards green** — `cutoverCopiesEveryBaseColumn` and `successorMaterializedColumnsMatchSource` pass on the release, so the cutover copies every base column of `traces` and the two tables' base and materialized columns match. -- [ ] **Fidelity verified** — `verify.sh` passes (full, or a documented representative sample) between source and - destination before the EXCHANGE. Re-run `delta_replay.sh` then `verify.sh` until it PASSES: while the buffer holds +- [ ] **Fidelity verified** — `verify.sh` passes between source and destination before the EXCHANGE. This gate MUST be a + **full compare** (`--sample-mod 1 --weeks-stride 1`, no `--from-week`/`--to-week` narrowing): a cross-project + workspace-scoped residual row is a single key, so any sampling (`--sample-mod > 1`), week stride, or week narrowing + can hash it out or skip its week and still report `ok=1`. Reserve sampling/ranged runs for follow-up confidence + *after* the full gate passes. Re-run `delta_replay.sh` then `verify.sh` until it PASSES: while the buffer holds writes (or, on a rehearsal without it, once traffic is quiescent) the last delta must catch every in-flight write. - [ ] **`Distributed` wrap gated on app-readiness** — apply the wrap (step 4, part 2) only when the delete/read DAOs are sharding-aware; otherwise stop after the `EXCHANGE`, since a lightweight `DELETE` against a `Distributed` `traces` diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh index 8aeb570af26..990ad1402ff 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/backfill.sh @@ -210,9 +210,13 @@ insert_window() { src="$(count_src_uniq "$lo" "$hi")" dst="$(count_dst_uniq "$lo" "$hi")" - # Resume: destination already has this window (exact, ahead due to concurrent deletes, or within tolerance). - if [[ "$dst" != "0" ]] \ - && [[ "$(awk -v s="$src" -v d="$dst" -v p="$DIVERGENCE" 'BEGIN { print (d >= s || (s - d) / s <= p) ? 1 : 0 }')" == "1" ]]; then + # Resume: skip only when the destination already holds at least as many logical rows as the source (exact, or ahead + # because concurrent deletes shrank the source). DIVERGENCE is NOT a resume criterion: a partially-copied window can + # sit a hair short of src yet within tolerance, and skipping it would leave those rows missing forever — the delta + # step only re-copies rows at/after backfill_start, so a pre-anchor gap is unrepairable. The backfill INSERT is + # idempotent (ReplacingMergeTree, mask-honoring), so re-copying a short window is safe and cheap. DIVERGENCE governs + # only the post-copy abort below. + if [[ "$dst" != "0" && "$dst" -ge "$src" ]]; then log "$label ($lo .. $hi): already present (src_uniq=$src dst_uniq=$dst), skipping" return fi diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java index f92d04c6279..c4d4ce45218 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java @@ -987,10 +987,11 @@ private void exchangeTables() { // need not exist yet), then one atomic multi-target RENAME rotates the data to traces_local and the wrapper into // traces (the name freed by the first clause), so traces is never absent on a node. private void wrapInDistributed() { - execute(("CREATE TABLE traces_dist ON CLUSTER '{cluster}' AS traces " - + "ENGINE = Distributed('{cluster}', '" + DATABASE_NAME + "', 'traces_local', sipHash64(project_id))"), - _ -> { - }); + execute(""" + CREATE TABLE traces_dist ON CLUSTER '{cluster}' AS traces + ENGINE = Distributed('{cluster}', '%s', 'traces_local', sipHash64(project_id)) + """.formatted(DATABASE_NAME), _ -> { + }); execute(""" RENAME TABLE traces TO traces_local, diff --git a/tests_load/tests/traces-local-v2-cutover/_common.py b/tests_load/tests/traces-local-v2-cutover/_common.py index eec373c6c72..c29cbaf197d 100644 --- a/tests_load/tests/traces-local-v2-cutover/_common.py +++ b/tests_load/tests/traces-local-v2-cutover/_common.py @@ -62,15 +62,25 @@ def discover_workspace_and_project( opik_client.trace(id=anchor_id, name="cutover-anchor", project_name=project_name, input={"anchor": True}).end() opik_client.flush() - deadline = time.time() + timeout_s - while time.time() < deadline: - rows = ch.query( - "SELECT workspace_id, toString(project_id) FROM traces WHERE id = {id:String} LIMIT 1", - parameters={"id": anchor_id}, - ).result_rows - if rows: - workspace_id, project_id = rows[0] - LOGGER.info("Resolved project '%s': workspace_id=%s project_id=%s", project_name, workspace_id, project_id) - return workspace_id, project_id - time.sleep(0.5) - raise TimeoutError(f"anchor trace for project '{project_name}' did not appear in ClickHouse within {timeout_s}s") + # Always remove the anchor: leaving it behind would add one live trace to the target project and skew seeded + # counts and delete/cutover verification. Cleanup failures are logged, not raised, so they don't mask a real error. + try: + deadline = time.time() + timeout_s + while time.time() < deadline: + rows = ch.query( + "SELECT workspace_id, toString(project_id) FROM traces WHERE id = {id:String} LIMIT 1", + parameters={"id": anchor_id}, + ).result_rows + if rows: + workspace_id, project_id = rows[0] + LOGGER.info( + "Resolved project '%s': workspace_id=%s project_id=%s", project_name, workspace_id, project_id + ) + return workspace_id, project_id + time.sleep(0.5) + raise TimeoutError(f"anchor trace for project '{project_name}' did not appear in ClickHouse within {timeout_s}s") + finally: + try: + opik_client.rest_client.traces.delete_traces(ids=[anchor_id]) + except Exception as exc: # noqa: BLE001 + LOGGER.warning("could not delete cutover-anchor trace %s: %s", anchor_id, exc) diff --git a/tests_load/tests/traces-local-v2-cutover/delete_traffic.py b/tests_load/tests/traces-local-v2-cutover/delete_traffic.py index b902b624913..03282236fcf 100644 --- a/tests_load/tests/traces-local-v2-cutover/delete_traffic.py +++ b/tests_load/tests/traces-local-v2-cutover/delete_traffic.py @@ -77,9 +77,9 @@ def main(project, tps, duration, batch): continue pool.extend(fetched) if not pool: - # An empty refill can be transient: the delete mask may not be visible to search yet, so the newest 500 - # ids can all still be in `seen`/`pool`. Only stop after several consecutive empty refills, so a mask-lag - # blip doesn't end the run while thousands of lower-id traces remain undeleted. + # An empty refill can be transient: the delete mask may not be visible to search yet, so the newest + # REFILL_FETCH ids can all still be in `seen`/`pool`. Only stop after several consecutive empty refills, + # so a mask-lag blip doesn't end the run while thousands of lower-id traces remain undeleted. empty_refills += 1 if empty_refills >= EMPTY_REFILL_LIMIT: LOGGER.info("no more traces to delete after %d empty refills; stopping", empty_refills) From 8755d6a79c02b35c534e7c81d90cb5718a09ccd6 Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Mon, 27 Jul 2026 12:39:08 +0200 Subject: [PATCH 24/28] fix(cutover): gate rollback stages B/C on --accept-post-cutover-write-loss MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address Baz #919 (high, rollback safety). Promoting the frozen pre-cutover backup makes traces the successor accepted after cutover_start non-live. They are not destroyed — the successor is parked as traces_local_v2 until finalize.sh, recoverable during the soak — but the live table stops serving them. This is inherent to promoting a point-in-time backup and is deliberately not auto-repaired (merging the successor's post-cutover writes back would re-import the data the rollback exists to discard). - rollback.sh: stages B/C now require --accept-post-cutover-write-loss and print the recovery pointer before the promote. Named --accept-* (not --confirm-*) because it acknowledges an unavoidable consequence rather than a precondition the operator fixes. - README: scope the "no data loss" claim to "no data-bearing table dropped"; add the post-cutover-write caveat with the recovery path; show the flag in the B/C examples. Co-Authored-By: Claude Opus 4.8 --- .../traces-local-v2-cutover/README.md | 40 ++++++++++++------- .../scripts/rollback.sh | 29 +++++++++++++- 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md index 5b42f9fee00..6fb17cd65aa 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md @@ -419,26 +419,36 @@ as one file per stage (`000004_rollback_stage_a_discard_shadow.sql`, `…_stage_ `…_stage_c_promote_original.sql`, and the shared `000004_rollback_reverse_replay.sql`) and driven by [`scripts/rollback.sh`](scripts/rollback.sh), so no one authors it under pressure. -**No data loss by construction.** The stages are mutually exclusive, so each lives in its **own file** — no single file -mixes the `TRUNCATE` (stage A only) with the `EXCHANGE`/`DROP` of the others, and running any file does exactly one -stage. No statement drops a data-bearing table: swaps are atomic `EXCHANGE`/`RENAME`, and the only `DROP` targets the -`Distributed` wrapper, which stores no data (it is a routing definition over `traces_local`). Before running, `rollback.sh` -**asserts the live `traces` topology matches the requested stage and aborts otherwise** — so a wrong-stage run (the only -way a `TRUNCATE`/`DROP` could hit the wrong table) makes no change. Every stage lands in the same **canonical state**: -`traces` = the original data (live), `traces_local_v2` = the successor data (parked backup). No leftover `*_new` names. -The parked backup is dropped only later, by `finalize.sh`, after the soak. +**No data-bearing table is dropped by construction.** The stages are mutually exclusive, so each lives in its **own +file** — no single file mixes the `TRUNCATE` (stage A only) with the `EXCHANGE`/`DROP` of the others, and running any +file does exactly one stage. No statement drops a data-bearing table: swaps are atomic `EXCHANGE`/`RENAME`, and the only +`DROP` targets the `Distributed` wrapper, which stores no data (it is a routing definition over `traces_local`). Before +running, `rollback.sh` **asserts the live `traces` topology matches the requested stage and aborts otherwise** — so a +wrong-stage run (the only way a `TRUNCATE`/`DROP` could hit the wrong table) makes no change. Every stage lands in the +same **canonical state**: `traces` = the original data (live), `traces_local_v2` = the successor data (parked backup). +No leftover `*_new` names. The parked backup is dropped only later, by `finalize.sh`, after the soak. + +> **Stages B/C make post-cutover writes non-live — an accepted, acknowledged trade-off.** Promoting the frozen +> `traces_pre_cutover_backup` means traces the successor accepted **after** `cutover_start` stop being served by the live +> table (the reverse-replay carries post-cutover *deletes* forward, but not *writes*). They are **not destroyed**: the +> successor is parked as `traces_local_v2` and retained until `finalize.sh`, so recover them from there during the soak +> if the rollback is later judged unnecessary. This is inherent to promoting a point-in-time backup and is *not* auto-repaired +> — merging the successor's post-cutover writes back would re-import the very data the rollback exists to discard. Because +> it is irreversible in the moment, stages B/C require `--accept-post-cutover-write-loss`, and `rollback.sh` prints the +> recovery pointer before the promote. Pick the stage by how far the cutover got (`cutover_start` is the value `exchange_and_wrap.sh` printed): - **Stage A — before EXCHANGE:** `./scripts/rollback.sh --database opik --stage A`. Discards the disposable shadow `traces_local_v2`; the live `traces` was never touched. (Guarded: aborts unless `traces` is still the original schema.) -- **Stage B — after EXCHANGE, before wrap:** `./scripts/rollback.sh --database opik --stage B --cutover-start ''`. - `EXCHANGE` `traces_pre_cutover_backup` back to live `traces`, rename the now-parked successor back to - `traces_local_v2`, then the reverse replay. (Guarded: aborts if `traces` is `Distributed` — use C.) -- **Stage C — after wrap:** `./scripts/rollback.sh --database opik --stage C --cutover-start ''`. Drops the - `Distributed` wrapper, then one atomic `RENAME` promotes the original (`traces_pre_cutover_backup`) back to `traces` - and parks the successor under `traces_local_v2`, then the reverse replay. (Guarded: aborts unless `traces` is - `Distributed`.) +- **Stage B — after EXCHANGE, before wrap:** `./scripts/rollback.sh --database opik --stage B --cutover-start '' + --confirm-retention-paused --accept-post-cutover-write-loss`. `EXCHANGE` `traces_pre_cutover_backup` back to live + `traces`, rename the now-parked successor back to `traces_local_v2`, then the reverse replay. (Guarded: aborts if + `traces` is `Distributed` — use C.) +- **Stage C — after wrap:** `./scripts/rollback.sh --database opik --stage C --cutover-start '' + --confirm-retention-paused --accept-post-cutover-write-loss`. Drops the `Distributed` wrapper, then one atomic + `RENAME` promotes the original (`traces_pre_cutover_backup`) back to `traces` and parks the successor under + `traces_local_v2`, then the reverse replay. (Guarded: aborts unless `traces` is `Distributed`.) After a stage B or C rollback, `traces` is the Nullable original again — **revert `traceColumnsNonNullable` to `false` AND roll-restart every backend instance**. The flag is read from a **startup snapshot** of `OpikConfiguration` (bound via diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh index 1f8e3202417..831e494b2ff 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh @@ -7,9 +7,17 @@ # --stage A backfill/delta ran but the EXCHANGE did not — discard the shadow (live `traces` is untouched). # --stage B the EXCHANGE ran but not the wrap — swap the tables back, then reverse-replay. # --stage C the wrap ran — drop the wrapper, promote the parked original, then reverse-replay. -# Stages B and C need --cutover-start (printed by exchange_and_wrap.sh) to bound the reverse-replay, and +# Stages B and C need --cutover-start (printed by exchange_and_wrap.sh) to bound the reverse-replay, # --confirm-retention-paused (retention deletes bypass the bridge, so a retention sweep in the rollback window would -# resurrect a deleted row from the backup). Keep the deletion bridge enabled through the rollback so no delete is lost. +# resurrect a deleted row from the backup), and --accept-post-cutover-write-loss (see below). Keep the deletion bridge +# enabled through the rollback so no delete is lost. +# +# POST-CUTOVER WRITES: stages B/C promote the frozen pre-cutover backup back to live `traces`, so traces WRITTEN to the +# successor after cutover_start stop being live. They are NOT destroyed — the successor is parked as traces_local_v2 and +# retained until finalize.sh, so they can be recovered from there during the soak — but the live table no longer serves +# them. This is inherent to promoting a point-in-time backup and cannot be "fixed" (auto-merging the successor's writes +# would re-import the very data the rollback is discarding); --accept-post-cutover-write-loss makes the operator +# acknowledge it before the promote. # # SAFETY: the stages are mutually exclusive and each lives in its OWN file, so no single file mixes a TRUNCATE with an # EXCHANGE/DROP — running any file does exactly one stage. Before running, this asserts the live `traces` topology matches @@ -28,6 +36,7 @@ DATABASE="" STAGE="" CUTOVER_START="" CONFIRM_RETENTION_PAUSED=0 +ACCEPT_WRITE_LOSS=0 while [[ $# -gt 0 ]]; do case "$1" in @@ -35,6 +44,7 @@ while [[ $# -gt 0 ]]; do --stage) STAGE="${2:?"$1 requires a value"}"; shift 2 ;; --cutover-start) CUTOVER_START="${2:?"$1 requires a value"}"; shift 2 ;; --confirm-retention-paused) CONFIRM_RETENTION_PAUSED=1; shift ;; + --accept-post-cutover-write-loss) ACCEPT_WRITE_LOSS=1; shift ;; *) echo "Unknown argument: $1" >&2; exit 2 ;; esac done @@ -57,6 +67,16 @@ if [[ ( "$STAGE" == "B" || "$STAGE" == "C" ) && "$CONFIRM_RETENTION_PAUSED" != " echo " Pause retention (RETENTION_ENABLED=false on every backend), then re-run with the flag." >&2 exit 2 fi +# Stages B/C promote the frozen pre-cutover backup, so writes the successor accepted after cutover_start stop being live +# (they are preserved in the parked traces_local_v2 until finalize.sh, recoverable during the soak). This is unavoidable +# when promoting a point-in-time backup — require the operator to acknowledge it, unlike a precondition they could fix. +if [[ ( "$STAGE" == "B" || "$STAGE" == "C" ) && "$ACCEPT_WRITE_LOSS" != "1" ]]; then + echo "ERROR: rollback --stage $STAGE requires --accept-post-cutover-write-loss. Promoting the frozen backup makes" >&2 + echo " traces written to the successor after cutover_start non-live. They are NOT destroyed — the successor is" >&2 + echo " parked as traces_local_v2 until finalize.sh, so recover them from there during the soak — but the live" >&2 + echo " table will no longer serve them. Re-run with the flag once you accept this." >&2 + exit 2 +fi ch() { clickhouse-client --database "$DATABASE" --log_comment 'traces_local_v2_rollback' --query "$1" @@ -124,6 +144,11 @@ run_file() { assert_topology +if [[ "$STAGE" == "B" || "$STAGE" == "C" ]]; then + echo "NOTE: promoting the frozen backup now. Traces the successor accepted after cutover_start ($CUTOVER_START) will" >&2 + echo " stop being live; recover them from the parked traces_local_v2 (kept until finalize.sh) if needed." >&2 +fi + case "$STAGE" in A) run_file 000004_rollback_stage_a_discard_shadow.sql From 91b7983b442c0c859989bc37f885c77aca2d93ee Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Mon, 27 Jul 2026 13:25:47 +0200 Subject: [PATCH 25/28] fix(cutover): server-clock resurrection stamp; scope loss claim; confirm anchor cleanup Address Baz round: - TracesLocalV2CutoverTest (#808): the workspace-scoped resurrection row stamped last_updated_at from Instant.now() (JVM clock) while backfillStart uses server now64(6); host/container skew could put it below backfillStart and flake the delta. Now parses nowMicros() like the other two window inserts. Swept the file: this was the last host-clock stamp feeding a server-clock comparison. Gate test 10/10. - README (#792): the opening claimed "zero deletion loss" while the cutover-tail section documents a bounded residual micro-window; reworded to "near-zero deletion loss" with a pointer to that section's mitigation. - _common.py (#789): delete_traces returns before ClickHouse applies the mask, so the #922 cleanup could still race the seeder; poll for the anchor's disappearance (bounded, best-effort) before returning. Co-Authored-By: Claude Opus 4.8 --- .../traces-local-v2-cutover/README.md | 4 +++- .../infrastructure/TracesLocalV2CutoverTest.java | 5 ++++- tests_load/tests/traces-local-v2-cutover/_common.py | 13 +++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md index 6fb17cd65aa..3c5547885e3 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md @@ -2,7 +2,9 @@ Operator runbook for the buffered cutover of the ClickHouse `traces` table: it migrates the live, unpartitioned `traces` table to `traces_local_v2` (weekly-partitioned, denullified, `is_deleted`-ready) with **near-zero downtime** -and **zero deletion loss**, then wraps it in a sharding-ready `Distributed` table. +and **near-zero deletion loss** — the deletion bridge replays every captured delete before the swap, leaving only a +bounded residual micro-window (see "The final cutover window" below, which also gives the mitigation) — then wraps it +in a sharding-ready `Distributed` table. The mechanism is **backfill + delta + deletion replay + EXCHANGE**, using the ingestion async-insert buffer to absorb the brief cutover window instead of a dual-write path. diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java index c4d4ce45218..fdcacd0c52b 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java @@ -728,7 +728,10 @@ void workspaceScopedReplaySparesLiveCrossProjectRow() { // wins under FINAL), so it is live again on the source in B and caught by the delta's last_updated_at arm. recordDeletionEvents(Set.of(reusedId.toString()), workspaceId, "", "user_request"); lightweightDelete(Set.of(reusedId.toString()), workspaceId); - insertRows(reused, workspaceId, projectB, "resurrected", _ -> Instant.now()); + // Server-clock last_updated_at (a later now64(6), so >= backfillStart) — NOT the JVM clock, whose skew vs the + // container could put it below backfillStart and make the delta's last_updated_at arm miss the resurrection. + var resurrectedAt = Instant.from(ClickHouseDateTimeFormat.MICROS.parse(nowMicros())); + insertRows(reused, workspaceId, projectB, "resurrected", _ -> resurrectedAt); deltaInsert(backfillStart); replayDeletions(backfillStart); diff --git a/tests_load/tests/traces-local-v2-cutover/_common.py b/tests_load/tests/traces-local-v2-cutover/_common.py index c29cbaf197d..f82ec6e8b0e 100644 --- a/tests_load/tests/traces-local-v2-cutover/_common.py +++ b/tests_load/tests/traces-local-v2-cutover/_common.py @@ -82,5 +82,18 @@ def discover_workspace_and_project( finally: try: opik_client.rest_client.traces.delete_traces(ids=[anchor_id]) + # delete_traces returns before ClickHouse applies the delete mask; poll until the anchor is actually gone so + # the seeder that runs next doesn't count it and skew backfill/delete/fidelity assertions (best-effort). + cleanup_deadline = time.time() + timeout_s + while time.time() < cleanup_deadline: + still_present = ch.query( + "SELECT 1 FROM traces WHERE id = {id:String} LIMIT 1", + parameters={"id": anchor_id}, + ).result_rows + if not still_present: + break + time.sleep(0.5) + else: + LOGGER.warning("cutover-anchor trace %s still visible %ss after delete; may skew counts", anchor_id, timeout_s) except Exception as exc: # noqa: BLE001 LOGGER.warning("could not delete cutover-anchor trace %s: %s", anchor_id, exc) From 90f5fba1b26733907094f59d8b70009fcf193348 Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Mon, 27 Jul 2026 13:50:09 +0200 Subject: [PATCH 26/28] fix(cutover): add --reverse-replay-only restart path; guide partial-EXCHANGE rollback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address Baz round (rollback robustness): - #801: a stage B/C promote and its reverse-replay run as two statements; if the replay is interrupted, the promote has already restored the canonical shape, so re-running the stage is refused by the topology guard — stranding post-cutover deletes unreplayed (they resurrect). This hole is symmetric across B and C. Add --reverse-replay-only: re-applies just 000004_rollback_reverse_replay.sql against the live traces (asserts non-Distributed MergeTree; requires --cutover-start and --confirm-retention-paused). Idempotent, safe to re-run, closes both stages. - #798: if the forward EXCHANGE succeeded but its post-swap RENAME did not, the parked original is still under traces_local_v2 and stage B aborts "backup not found". Stage B now detects that state and prints the one-line RENAME to finish it (same remediation exchange_and_wrap.sh gives), then proceeds. Not auto-completed: rollback does exactly one thing per run. - README: document both recovery paths in the rollback section. Co-Authored-By: Claude Opus 4.8 --- .../traces-local-v2-cutover/README.md | 13 ++++ .../scripts/rollback.sh | 69 +++++++++++++++---- 2 files changed, 70 insertions(+), 12 deletions(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md index 3c5547885e3..95f5b83d5e4 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md @@ -452,6 +452,19 @@ Pick the stage by how far the cutover got (`cutover_start` is the value `exchang `RENAME` promotes the original (`traces_pre_cutover_backup`) back to `traces` and parks the successor under `traces_local_v2`, then the reverse replay. (Guarded: aborts unless `traces` is `Distributed`.) +**Recovering from an interrupted rollback.** Each promote stage runs its table-swap and then the reverse-replay as two +statements, so a failure *between* them needs a restart path: + +- **Reverse-replay interrupted (stage B or C).** The promote already restored the original, so `traces` is back in the + canonical shape and re-running the stage is (correctly) refused by the topology guard — which would otherwise leave the + post-cutover deletes unreplayed and let them resurrect. Re-apply just the replay: + `./scripts/rollback.sh --database opik --reverse-replay-only --cutover-start '' --confirm-retention-paused`. It runs + only `000004_rollback_reverse_replay.sql` against the live `traces` (asserts it is a non-`Distributed` `MergeTree`) and + is idempotent, so it is safe to run once or repeatedly. +- **Forward EXCHANGE half-done (stage B says the backup is missing).** If the forward `EXCHANGE` succeeded but its + post-swap `RENAME` did not, the parked original is still under `traces_local_v2` and stage B aborts pointing at the + one-line `RENAME` that finishes it (`traces_local_v2` → `traces_pre_cutover_backup`); run that, then re-run stage B. + After a stage B or C rollback, `traces` is the Nullable original again — **revert `traceColumnsNonNullable` to `false` AND roll-restart every backend instance**. The flag is read from a **startup snapshot** of `OpikConfiguration` (bound via `toInstance`), so a config change does **not** take effect until each instance restarts — exactly like the forward diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh index 831e494b2ff..9f23c2020bd 100755 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/scripts/rollback.sh @@ -7,6 +7,10 @@ # --stage A backfill/delta ran but the EXCHANGE did not — discard the shadow (live `traces` is untouched). # --stage B the EXCHANGE ran but not the wrap — swap the tables back, then reverse-replay. # --stage C the wrap ran — drop the wrapper, promote the parked original, then reverse-replay. +# Or --reverse-replay-only: re-apply just the reverse deletion replay against the current live `traces`. Use it when a +# stage B/C run's promote succeeded but its reverse-replay was interrupted — the promote leaves `traces` in the restored +# canonical shape, so re-running the stage is (correctly) rejected by the topology guard, which would otherwise strand +# the post-cutover deletes unreplayed and let them resurrect. The replay is idempotent, so this is always safe to re-run. # Stages B and C need --cutover-start (printed by exchange_and_wrap.sh) to bound the reverse-replay, # --confirm-retention-paused (retention deletes bypass the bridge, so a retention sweep in the rollback window would # resurrect a deleted row from the backup), and --accept-post-cutover-write-loss (see below). Keep the deletion bridge @@ -37,6 +41,7 @@ STAGE="" CUTOVER_START="" CONFIRM_RETENTION_PAUSED=0 ACCEPT_WRITE_LOSS=0 +REVERSE_REPLAY_ONLY=0 while [[ $# -gt 0 ]]; do case "$1" in @@ -45,6 +50,7 @@ while [[ $# -gt 0 ]]; do --cutover-start) CUTOVER_START="${2:?"$1 requires a value"}"; shift 2 ;; --confirm-retention-paused) CONFIRM_RETENTION_PAUSED=1; shift ;; --accept-post-cutover-write-loss) ACCEPT_WRITE_LOSS=1; shift ;; + --reverse-replay-only) REVERSE_REPLAY_ONLY=1; shift ;; *) echo "Unknown argument: $1" >&2; exit 2 ;; esac done @@ -53,17 +59,22 @@ done # --database and --cutover-start are interpolated into the reference SQL; validate their shapes so neither can alter it. [[ "$DATABASE" =~ ^[A-Za-z0-9_]+$ ]] || { echo "ERROR: --database must be a ClickHouse identifier (letters, digits, underscore)." >&2; exit 2; } [[ -z "$CUTOVER_START" || "$CUTOVER_START" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}\ [0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?$ ]] || { echo "ERROR: --cutover-start must be 'YYYY-MM-DD HH:MM:SS[.ffffff]'." >&2; exit 2; } -case "$STAGE" in - A|B|C) ;; - *) echo "ERROR: --stage must be A, B or C" >&2; exit 2 ;; -esac -# Stages B/C run the reverse-replay, which — like the forward replay — only re-applies bridged deletes. Retention deletes -# (deleteForRetention*) bypass the bridge, so a retention sweep during the rollback window would restore a legitimately -# deleted row from the backup and resurrect it. Retention is a backend setting the script can't read; require the -# operator to assert it is paused. Stage A does no reverse-replay, so it is exempt. -if [[ ( "$STAGE" == "B" || "$STAGE" == "C" ) && "$CONFIRM_RETENTION_PAUSED" != "1" ]]; then - echo "ERROR: rollback --stage $STAGE requires --confirm-retention-paused. The reverse-replay only re-applies bridged" >&2 - echo " deletes; a retention sweep in the rollback window would resurrect a deleted row from the backup." >&2 +# Exactly one of --stage / --reverse-replay-only. +if [[ "$REVERSE_REPLAY_ONLY" == "1" ]]; then + [[ -z "$STAGE" ]] || { echo "ERROR: --reverse-replay-only cannot be combined with --stage." >&2; exit 2; } +else + case "$STAGE" in + A|B|C) ;; + *) echo "ERROR: --stage must be A, B or C (or pass --reverse-replay-only)" >&2; exit 2 ;; + esac +fi +# The reverse-replay runs for stages B/C and for --reverse-replay-only, and — like the forward replay — only re-applies +# bridged deletes. Retention deletes (deleteForRetention*) bypass the bridge, so a retention sweep during the rollback +# window would restore a legitimately deleted row from the backup and resurrect it. Retention is a backend setting the +# script can't read; require the operator to assert it is paused. Stage A does no reverse-replay, so it is exempt. +if [[ ( "$STAGE" == "B" || "$STAGE" == "C" || "$REVERSE_REPLAY_ONLY" == "1" ) && "$CONFIRM_RETENTION_PAUSED" != "1" ]]; then + echo "ERROR: this rollback runs the reverse-replay and requires --confirm-retention-paused. It re-applies only" >&2 + echo " bridged deletes; a retention sweep in the rollback window would resurrect a deleted row from the backup." >&2 echo " Pause retention (RETENTION_ENABLED=false on every backend), then re-run with the flag." >&2 exit 2 fi @@ -119,7 +130,20 @@ assert_topology() { echo "ERROR: stage B expects traces to hold the successor schema, but end_time is Nullable (the EXCHANGE has not run). Nothing to roll back; use stage A to discard the shadow." >&2 exit 1 } - [[ -n "$(traces_engine traces_pre_cutover_backup)" ]] || { echo "ERROR: 'traces_pre_cutover_backup' (parked original) not found; cannot swap back." >&2; exit 1; } + if [[ -z "$(traces_engine traces_pre_cutover_backup)" ]]; then + if [[ -n "$(traces_engine traces_local_v2)" ]]; then + # State X: the forward EXCHANGE succeeded but its post-swap RENAME did not, so the parked original is + # still under traces_local_v2. Finish that RENAME (the same remediation exchange_and_wrap.sh prints), + # then stage B proceeds normally. Not auto-completed here: rollback does exactly one thing per run. + echo "ERROR: 'traces_pre_cutover_backup' not found but 'traces_local_v2' still exists — the forward" >&2 + echo " EXCHANGE's post-swap RENAME did not complete, so the parked original is still under" >&2 + echo " 'traces_local_v2'. Finish that RENAME, then re-run stage B:" >&2 + echo " clickhouse-client --database $DATABASE --query \"RENAME TABLE $DATABASE.traces_local_v2 TO $DATABASE.traces_pre_cutover_backup ON CLUSTER '{cluster}'\"" >&2 + else + echo "ERROR: 'traces_pre_cutover_backup' (parked original) not found; cannot swap back." >&2 + fi + exit 1 + fi ;; C) [[ "$engine" == "Distributed" ]] || { @@ -142,6 +166,27 @@ run_file() { clickhouse-client --database "$DATABASE" --multiquery --query "$sql" } +# Recovery mode: re-apply only the reverse-replay against the current live `traces`. The promote (stage B/C) already ran, +# so `traces` is the restored original MergeTree; the replay is idempotent, so re-running it just re-masks any deletes it +# missed. Assert `traces` is present and NOT Distributed (a lightweight DELETE is unsupported on Distributed, and the +# replay only makes sense on the restored original — a Distributed `traces` means the wrap is still up; roll it back with +# stage C first). +if [[ "$REVERSE_REPLAY_ONLY" == "1" ]]; then + [[ -n "$CUTOVER_START" ]] || { echo "ERROR: --cutover-start is required for --reverse-replay-only" >&2; exit 2; } + reverse_replay_engine="$(traces_engine traces)" + [[ -n "$reverse_replay_engine" ]] || { echo "ERROR: no 'traces' table found in database '$DATABASE'." >&2; exit 1; } + [[ "$reverse_replay_engine" != "Distributed" ]] || { + echo "ERROR: 'traces' is Distributed, so the wrap is still applied — the reverse-replay runs on the restored" >&2 + echo " original MergeTree. Roll the wrap back first with --stage C." >&2 + exit 1 + } + echo "NOTE: re-applying the reverse deletion replay only (no table swap) for deletes since cutover_start" >&2 + echo " ($CUTOVER_START). Idempotent; use this after a stage B/C run whose reverse-replay was interrupted." >&2 + run_file 000004_rollback_reverse_replay.sql + echo "Reverse-replay-only done: bridged deletes since cutover_start re-applied to the live 'traces'." + exit 0 +fi + assert_topology if [[ "$STAGE" == "B" || "$STAGE" == "C" ]]; then From ff6e8fc1ca66ce19e8469b8cae3c4aede1440275 Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Mon, 27 Jul 2026 14:19:36 +0200 Subject: [PATCH 27/28] fix(cutover): drop flaky replay-time gate; disclose workspace-replay residual Address Baz round: - TracesLocalV2CutoverTest (#444): the replay wall-time assertion (< 60s) gated a non-correctness, environment-sensitive property (container startup, CI contention), making it a flaky gate; correctness is already asserted (the mask is applied). Drop the hard bound and the unused REPLAY_BUDGET_SECONDS; keep the measurement as a log. - README (#447): the workspace-scoped replay branch was called a "faithful mirror" that "ends matching the source" with no caveat, but 000002 documents a known residual (the (workspace_id, id) resurrection guard lets a live copy in one project shield the same reused id's delete in another). Disclose it, note 000005 flags it (ok=0), and point at OPIK-7483 which retires the arm. Gate test 10/10. Co-Authored-By: Claude Opus 4.8 --- .../traces-local-v2-cutover/README.md | 15 ++++++++++----- .../infrastructure/TracesLocalV2CutoverTest.java | 8 +++----- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md index 95f5b83d5e4..6203662fdc7 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md @@ -343,10 +343,14 @@ resolve fall back to a **workspace-scoped** delete — `TraceDAO.DELETE_BY_ID` w second with an **empty `project_id`** (`DeletionEventDAO`: "project_id is empty for workspace-scoped source tables"). The replay mirrors both: full-key events delete by `(workspace_id, project_id, id)` (exact; prunes on the destination primary key — correct even though trace ids are not globally unique), and empty-project events delete by `(workspace_id, id)`. -The second branch is a **faithful mirror, not an over-delete**: the workspace-scoped fallback fires only for ids the -resolver found no live row for in any project, and the source delete it replays already removed every `(workspace_id, -id)` row — so the destination ends matching the source. Without it, those deletions would **silently leak** across the -swap. +The second branch is a **mirror, not an over-delete**: the workspace-scoped fallback fires only for ids the resolver +found no live row for in any project, and the source delete it replays already removed every `(workspace_id, id)` row. +Without it, those deletions would **silently leak** across the swap. It is faithful in the common case, with **one known +residual** (see 000002): because its resurrection guard keys on `(workspace_id, id)` (no project), an id that is live in +one project shields the delete of the *same reused id* in another — leaving an extra destination row. It needs id reuse +across projects **plus** a workspace-scoped delete **plus** a resurrection in the window, so it is rare; the `000005` +`FINAL` fingerprint flags it (`ok=0`) rather than passing silently, and OPIK-7483 retires the arm entirely by making +deletes always carry `project_id`. **Resurrection guard.** A trace can be deleted and then re-created/updated under the **same id** during the window (ids are client-supplied; the delete is a mask, and a newer insert wins under `FINAL`). Such an id is bridged as deleted @@ -599,7 +603,8 @@ mismatched window). reverse-replays so a post-cutover delete does not resurrect; - **wrong-stage rollback guard** — the topology signals `rollback.sh` keys on (the `traces` engine and `end_time` nullability) are distinct in each cutover state, so a mis-targeted stage aborts instead of touching the wrong table; -- the replay wall time is measured and bounded. +- the replay wall time is measured and logged (not asserted — it is environment-sensitive; the buffer-window sizing is + done in the cutover rehearsal, not in CI). Run it with: `mvn -o test -Dtest=TracesLocalV2CutoverTest` from `apps/opik-backend`. diff --git a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java index fdcacd0c52b..b3e65ff791b 100644 --- a/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java +++ b/apps/opik-backend/src/test/java/com/comet/opik/infrastructure/TracesLocalV2CutoverTest.java @@ -132,8 +132,6 @@ class TracesLocalV2CutoverTest { private static final int DELTA_UPSERTS = 20; private static final int DELTA_LATE_CREATED = 10; - private static final int REPLAY_BUDGET_SECONDS = 60; - private static final String[] FIDELITY_SOURCES = {"sdk", "experiment", "playground", "optimization", "evaluator"}; private static final String[] FIDELITY_ENVIRONMENTS = {"production", "staging", "dev", ""}; @@ -381,11 +379,11 @@ void bufferedCutoverPreservesEveryDeletionAcrossExchange() { // Deletion replay: read the bridge for the window and re-issue the deletes against the destination, matched on // the full key so a reused id in another project is untouched. + // Measured and logged (not asserted): replay wall time is environment-sensitive (container startup, CI + // contention), so a hard bound here would be a flaky gate on a non-correctness property. The runbook sizes it + // against the buffer window during the cutover rehearsal; correctness is asserted below (the mask is applied). var replayMillis = replayDeletions(backfillStart); log.info("Deletion replay covered {} ids in {} ms", leakedIds.size() + 1, replayMillis); - assertThat(replayMillis) - .as("replay wall time smoke bound — the runbook compares this against the buffer window") - .isLessThan(REPLAY_BUDGET_SECONDS * 1_000L); // After replay, the leak is closed on the destination, before the swap. assertThat(liveCount("traces_local_v2", leakedIds, workspaceId)) From 994479e1da93ec713f3131cecdcc238338283025 Mon Sep 17 00:00:00 2001 From: Andres Cruz Date: Mon, 27 Jul 2026 14:28:00 +0200 Subject: [PATCH 28/28] docs(cutover): fix grammar in workspace-scoped replay explanation Address Baz #3657021887: "the source delete it replays already removed" -> "the source deletion that it replays already removed". Co-Authored-By: Claude Opus 4.8 --- .../data-migrations/traces-local-v2-cutover/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md index 6203662fdc7..24c771aaa0c 100644 --- a/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md +++ b/apps/opik-backend/data-migrations/traces-local-v2-cutover/README.md @@ -344,7 +344,8 @@ second with an **empty `project_id`** (`DeletionEventDAO`: "project_id is empty replay mirrors both: full-key events delete by `(workspace_id, project_id, id)` (exact; prunes on the destination primary key — correct even though trace ids are not globally unique), and empty-project events delete by `(workspace_id, id)`. The second branch is a **mirror, not an over-delete**: the workspace-scoped fallback fires only for ids the resolver -found no live row for in any project, and the source delete it replays already removed every `(workspace_id, id)` row. +found no live row for in any project, and the source deletion that it replays already removed every `(workspace_id, id)` +row. Without it, those deletions would **silently leak** across the swap. It is faithful in the common case, with **one known residual** (see 000002): because its resurrection guard keys on `(workspace_id, id)` (no project), an id that is live in one project shields the delete of the *same reused id* in another — leaving an extra destination row. It needs id reuse