[codex] Support TiDB Cloud Native e2e provisioning

e2e/AGENTS.md

@@ -40,6 +40,13 @@ export DRIVE9_BASE="https://api.drive9.ai"
 
 Use the dev value unless the environment owner announces a new endpoint.
 
+For `tidb_cloud_native` endpoints that do not provide server-side default TiDB
+Cloud credentials, set `DRIVE9_TIDBCLOUD_PUBLIC_KEY` and
+`DRIVE9_TIDBCLOUD_PRIVATE_KEY`. The common e2e provisioning helper sends them in
+`/v1/provision` for the standard smoke scripts while preserving empty-body
+provisioning when they are unset. Set `DRIVE9_TIDBCLOUD_SPENDING_LIMIT` only
+when the run must override the create-time TiDB Cloud spending limit.
+
 #### Run smoke scripts
 
 ```bash
@@ -88,6 +95,13 @@ bash e2e/posix-permission-smoke-test.sh
 # Run the default smoke-all sequence once
 bash e2e/smoke-all.sh
 
+# Scheduled hosted smoke runs should opt into registry-based cleanup. Cleanup
+# only deletes tenants/forks registered by this local drive9-e2e run; it does not
+# scan server tenants.
+DRIVE9_E2E_TMPDIR="$HOME/.cache/drive9-smoke/tmp" \
+DRIVE9_E2E_CLEANUP=always \
+bash e2e/smoke-all.sh
+
 # Or enable optional Git coverage in that same single smoke-all run.
 # Set either variable to 1 as needed; setting both includes both Git suites.
 RUN_GIT_OPS_SMOKE=1 RUN_GIT_WORKSPACE_SMOKE=1 bash e2e/smoke-all.sh
@@ -104,6 +118,41 @@ DRIVE9_TIDBCLOUD_PRIVATE_KEY="$DRIVE9_TIDBCLOUD_PRIVATE_KEY" \
 bash e2e/native-smoke-test.sh
 ```
 
+### Scheduled smoke-all failure handling
+
+When a scheduled `smoke-all.sh` run finishes with any failed suite or assertion,
+do the failure analysis before creating or changing repo issues:
+
+1. Identify the failed suite, failed case/assertion, smoke endpoint, command,
+   exit status, cleanup result, and full log path.
+2. Analyze the failed case enough to state a likely root cause or the concrete
+   blocker that prevents a root-cause call. Use the repo, smoke logs, and
+   relevant code paths; do not report only the aggregate `PASS=N FAIL=M` line.
+   If service/app logs are available through local tooling, include them in the
+   analysis. If they are not available from the current machine or agent, state
+   that limitation explicitly and continue the local-log/code-path analysis.
+   Keep machine-specific log access details in local runbooks or agent memory,
+   not in this repo-level policy.
+3. Report the findings first in the Slock/Raft smoke-test thread that requested
+   or owns the run. Include the failed case, evidence, log path, impact, and
+   root-cause assessment.
+4. Include an investigation-context block with non-secret identifiers needed for
+   external/cloud-side debugging when they are available: run id, branch/commit,
+   endpoint/region, parent/fork/admin tenant ids, Drive9 tenant status/kind,
+   TiDB Cloud cluster id/state, branch id/state/display name/user prefix/public
+   endpoint, relevant create/update/failure timestamps, failed checks, cleanup
+   registry path, pending-cleanup state, and retained-resource decision. Never
+   include API keys, private keys, DB passwords, bearer tokens, full DSNs, or
+   credential files in chat or issues.
+5. If the failure is a code problem, search existing GitHub issues before filing
+   a new one. Create or update the matching issue with: background, reproduction
+   steps, evidence logs, impact, root-cause analysis or suspicious code path,
+   suggested fix direction, and validation plan.
+6. If the failure is due to environment, credentials, runner setup, endpoint
+   availability, or another non-code operational condition, report that in the
+   Slock/Raft thread instead of filing a code bug, unless durable repo tracking
+   is needed because the condition is recurring or needs engineering follow-up.
+
 ### Local via `drive9-server-local`
 
 When the task is specifically about local validation on this machine, prefer
@@ -534,6 +583,7 @@ Manual-only: requires TiDB Cloud API credentials. Not wired into CI.
 | Variable | Default | Used by |
 |----------|---------|---------|
 | `DRIVE9_BASE` | `http://127.0.0.1:9009` | all scripts |
+| `DRIVE9_E2E_TMPDIR` | `${TMPDIR:-/tmp}` | shared e2e temp root; scripts export `TMPDIR` from it and default FUSE roots to it |
 | `DRIVE9_IMAGE_FIXTURE_PATH` | `e2e/fixtures/cat03.jpg` | `api-smoke-test.sh`, `cli-smoke-test.sh` |
 | `DRIVE9_API_KEY` | - | `api-smoke-test-existing-key.sh` |
 | `DRIVE9_API_KEY` | - | `fuse-smoke-test.sh` (optional; skip provision when set) |
@@ -565,7 +615,7 @@ Manual-only: requires TiDB Cloud API credentials. Not wired into CI.
 | `CLI_RELEASE_VERSION` | *(latest)* | `cli-smoke-test.sh`, `fuse-smoke-test.sh`, `fuse-correctness-workload.sh`, `fuse-sqlite-correctness.sh`, `fuse-concurrency-stress.sh`, `fuse-performance-baseline.sh` |
 | `MOUNT_READY_TIMEOUT_S` | `20` | `fuse-smoke-test.sh`, `fuse-correctness-workload.sh`, `fuse-sqlite-correctness.sh`, `fuse-concurrency-stress.sh`, `fuse-performance-baseline.sh` |
 | `MOUNT_READY_INTERVAL_S` | `1` | `fuse-smoke-test.sh`, `fuse-correctness-workload.sh`, `fuse-sqlite-correctness.sh`, `fuse-concurrency-stress.sh`, `fuse-performance-baseline.sh` |
-| `FUSE_MOUNT_ROOT` | `/tmp` | `fuse-smoke-test.sh`, `fuse-correctness-workload.sh`, `fuse-sqlite-correctness.sh`, `fuse-concurrency-stress.sh`, `fuse-performance-baseline.sh` |
+| `FUSE_MOUNT_ROOT` | `$DRIVE9_E2E_TMPDIR` | `fuse-smoke-test.sh`, `fuse-correctness-workload.sh`, `fuse-sqlite-correctness.sh`, `fuse-concurrency-stress.sh`, `fuse-performance-baseline.sh` |
 | `CLI_MAX_RETRIES` | `8` | `fuse-smoke-test.sh` |
 | `CLI_RETRY_SLEEP_S` | `2` | `fuse-smoke-test.sh` |
 | `FUSE_STRICT_PREREQS` | `0` (`1` in release gate) | `fuse-smoke-test.sh`, `fuse-correctness-workload.sh`, `fuse-sqlite-correctness.sh`, `fuse-concurrency-stress.sh`, `fuse-performance-baseline.sh` |

e2e/README.md

@@ -9,13 +9,64 @@ including local single-tenant validation via `drive9-server-local`.
 - `jq` installed
 - Bash 4+
 
+For `tidb_cloud_native` endpoints that do not have server-side default TiDB
+Cloud credentials, set `DRIVE9_TIDBCLOUD_PUBLIC_KEY` and
+`DRIVE9_TIDBCLOUD_PRIVATE_KEY`. The general provisioning smoke scripts then send
+those fields in `/v1/provision`; when unset, they keep the existing empty-body
+provisioning behavior for anonymous/default-key endpoints. Optionally set
+`DRIVE9_TIDBCLOUD_SPENDING_LIMIT` to include `tidbcloud_spending_limit`.
+
+Use `DRIVE9_E2E_TMPDIR` to move generated e2e local artifacts off `/tmp`.
+Scripts create the directory if needed, export `TMPDIR` from it so `mktemp`
+uses the same root, and default FUSE run roots to it unless `FUSE_MOUNT_ROOT`
+is set explicitly.
+
+## Hosted cleanup
+
+Hosted smoke suites create real Drive9 tenants/spaces. By default cleanup is
+disabled to preserve the historical debugging behavior:
+
+```bash
+DRIVE9_E2E_CLEANUP=0 bash e2e/smoke-all.sh
+```
+
+For scheduled online checks, enable registry-based cleanup explicitly:
+
+```bash
+DRIVE9_E2E_TMPDIR="$HOME/.cache/drive9-smoke/tmp" \
+DRIVE9_E2E_CLEANUP=always \
+bash e2e/smoke-all.sh
+```
+
+`smoke-all.sh` creates a `drive9-e2e-*` run registry under
+`~/.cache/drive9-smoke/cleanup-pending/`. Each provisioned tenant/fork is
+registered immediately after creation; the finalizer deletes only those
+registered resources and never scans the server for tenants. Registries are
+owner-readable only because they contain API keys. Successful cleanup writes a
+sanitized summary under `~/.cache/drive9-smoke/cleanup-completed/`; incomplete
+cleanup leaves the raw registry pending for retry by `e2e/cleanup-pending.sh`.
+Standalone suites fail before provisioning if cleanup is enabled without a
+registry prepared by `smoke-all.sh`.
+If a pending registry was created with TiDBCloud-native credentials, retries must
+run with the same `DRIVE9_TIDBCLOUD_PUBLIC_KEY` and
+`DRIVE9_TIDBCLOUD_PRIVATE_KEY` environment available; the registry records only
+that credentials are required, not the credential values themselves.
+
+Supported modes:
+
+- `DRIVE9_E2E_CLEANUP=0`: register/delete disabled.
+- `DRIVE9_E2E_CLEANUP=always`: delete registered resources on success or failure.
+- `DRIVE9_E2E_CLEANUP=success`: delete only after a fully successful run; failed
+  runs move the registry to `cleanup-retained/` for manual debugging.
+
 ## Scripts
 
 | Script | What it validates |
 |--------|--------------------|
 | `api-smoke-test.sh` | Fresh provisioning, status polling, nested+batch file ops, hardlink/copy/rename/delete checks, grep/find checks, semantic text recall, image-associated recall, sql checks, large multipart upload+download |
 | `api-smoke-test-existing-key.sh` | Existing API key status/list checks |
 | `cli-smoke-test.sh` | End-to-end CLI workflow including `fs symlink`, `fs hardlink`, default-slot `pack`/`unpack`, `fs grep`/`fs find`, semantic/image-associated recall checks, image `fs cp`+`fs find`, and large multipart `fs cp` upload/download |
+| `cleanup-pending.sh` | Manual retry helper for pending hosted cleanup registries left by interrupted `smoke-all.sh` runs; `smoke-all.sh` also invokes the same pending cleanup path before starting a new cleanup-enabled run |
 | `layer-fs-smoke-test.sh` | Layer filesystem API+CLI+FUSE workflow: create by name/tag, diff/checkpoint lookup, rollback, commit, scope rejection, conflict detection, mkdir/upsert/whiteout/rename/symlink/chmod entries, and checkpoint/full restore into fresh local roots |
 | `layer-fs-smoke-test-realdev.sh` | Manual realdev wrapper for `layer-fs-smoke-test.sh`; defaults to the shared dev endpoint and enables strict FUSE restore coverage without being wired into `local-e2e` CI |
 | `portable-pack-unpack-e2e.sh` | Portable profile pack/unpack over a deterministic local repo: offline npm `file:` install creates `node_modules`, Git staged/unstaged/untracked status changes are captured, pack writes the default hidden archive, fresh local-root unpack restores overlay files, symlinks, `.git`, `node_modules`, branch, HEAD, and `git status` |
@@ -48,7 +99,7 @@ without adding it to `.github/workflows/local-e2e.yml`.
 | Post-merge | `push` to `main` (local-e2e, coalesced via concurrency group) | PR gate + concurrency stress, POSIX/fsx, sqlite WAL/churn/concurrency, full `smoke-all.sh` (journal, posix-permission, git-workspace), git feature matrix |
 | Nightly | cron 20:17 UTC (local-e2e) | Post-merge set + FUSE performance baseline/archive/compare (compare is report-only; hosted-runner noise) |
 | Manual all | `e2e-all` workflow (`Run workflow` button) | Everything above + pjdfstest POSIX suite (best-effort) via `run_all_e2e=1` |
-| Manual only | not wired, run by hand | `layer-fs-smoke-test-realdev.sh` (shared dev endpoint), `verify-description-e2e.sh` (Docker + Ollama), `verify-description-tidb-zero-e2e.sh` (TiDB Cloud Zero), `native-smoke-test.sh` (TiDB Cloud Native — requires credentials), `local-smoke.sh` (`make e2e-local` wrapper) |
+| Manual only | not wired, run by hand | `layer-fs-smoke-test-realdev.sh` (shared dev endpoint), `verify-description-e2e.sh` (Docker + Ollama), `verify-description-tidb-zero-e2e.sh` (TiDB Cloud Zero), `native-smoke-test.sh` (TiDB Cloud Native — requires credentials), `cleanup-pending.sh` (manual pending cleanup retry helper; also invoked by cleanup-enabled `smoke-all.sh` before a run), `local-smoke.sh` (`make e2e-local` wrapper) |
 
 Scheduled and post-merge failures auto-file/append to a `ci-e2e-failure`
 GitHub issue, since GitHub only notifies the workflow author otherwise.
@@ -193,6 +244,11 @@ DRIVE9_TIDBCLOUD_PUBLIC_KEY=xxx DRIVE9_TIDBCLOUD_PRIVATE_KEY=xxx bash e2e/native
 
 bash e2e/smoke-all.sh
 
+# Use TiDB Cloud credentials for tidb_cloud_native endpoints without a
+# server-side default key.
+DRIVE9_TIDBCLOUD_PUBLIC_KEY=xxx DRIVE9_TIDBCLOUD_PRIVATE_KEY=xxx \
+  bash e2e/smoke-all.sh
+
 # Include portable profile pack/unpack coverage in smoke-all.
 RUN_PORTABLE_PACK_E2E=1 bash e2e/smoke-all.sh
 ```
@@ -406,7 +462,8 @@ CLI_SOURCE=official bash e2e/git-workspace-smoke-test.sh
 - CLI batch small-file coverage can be tuned with `CLI_BATCH_SMALL_FILE_COUNT` (default `10`).
 - API retry knobs for throttling are `REQUEST_MAX_RETRIES` and `REQUEST_RETRY_SLEEP_S`; the FUSE correctness/SQLite/concurrency workloads use these for provisioning/status and CLI retry loops.
 - CLI retry knobs for `cli-smoke-test.sh` and `fuse-smoke-test.sh` throttling are `CLI_MAX_RETRIES` and `CLI_RETRY_SLEEP_S`.
-- FUSE mount readiness knobs are `MOUNT_READY_TIMEOUT_S`, `MOUNT_READY_INTERVAL_S`, and `FUSE_MOUNT_ROOT`.
+- Temporary artifact root is `DRIVE9_E2E_TMPDIR` (default `${TMPDIR:-/tmp}`); scripts export `TMPDIR` from it and use it for named large fixtures, local work roots, cleanup run dirs, and default FUSE roots.
+- FUSE mount readiness knobs are `MOUNT_READY_TIMEOUT_S`, `MOUNT_READY_INTERVAL_S`, and `FUSE_MOUNT_ROOT` (default `$DRIVE9_E2E_TMPDIR`).
 - FUSE correctness workload knobs are `FUSE_CORRECTNESS_LARGE_MB` and `FUSE_CORRECTNESS_KEEP_ARTIFACTS`.
 - FUSE SQLite correctness workload knobs are `FUSE_SQLITE_ROWS`, `FUSE_SQLITE_CHURN_ROUNDS`, `FUSE_SQLITE_CONCURRENCY_READERS`, `FUSE_SQLITE_CONCURRENCY_WRITES`, `FUSE_SQLITE_WORKLOAD_TIMEOUT_S`, `FUSE_SQLITE_KEEP_ARTIFACTS`, `RUN_FUSE_SQLITE_WAL`, `RUN_FUSE_SQLITE_CHURN`, and `RUN_FUSE_SQLITE_CONCURRENCY`.
 - FUSE concurrency workload knobs are `FUSE_CONCURRENCY_WORKERS`, `FUSE_CONCURRENCY_FILES_PER_WORKER`, `FUSE_CONCURRENCY_READER_WORKERS`, `FUSE_CONCURRENCY_PAYLOAD_KB`, `FUSE_CONCURRENCY_TIMEOUT_S`, and `FUSE_CONCURRENCY_KEEP_ARTIFACTS`.

e2e/_feature-matrix-runner.sh

@@ -14,6 +14,9 @@
 
 set -euo pipefail
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+. "$SCRIPT_DIR/provision-helper.sh"
+
 BASE="${DRIVE9_BASE:-http://127.0.0.1:9009}"
 DRIVE9_API_KEY="${DRIVE9_API_KEY:-}"
 POLL_TIMEOUT_S="${POLL_TIMEOUT_S:-120}"
@@ -22,7 +25,7 @@ MOUNT_READY_TIMEOUT_S="${MOUNT_READY_TIMEOUT_S:-25}"
 MOUNT_READY_INTERVAL_S="${MOUNT_READY_INTERVAL_S:-1}"
 REMOTE_VISIBILITY_TIMEOUT_S="${REMOTE_VISIBILITY_TIMEOUT_S:-20}"
 REMOTE_VISIBILITY_INTERVAL_S="${REMOTE_VISIBILITY_INTERVAL_S:-0.5}"
-FUSE_MOUNT_ROOT="${FUSE_MOUNT_ROOT:-/tmp}"
+FUSE_MOUNT_ROOT="${FUSE_MOUNT_ROOT:-$DRIVE9_E2E_TMPDIR}"
 FUSE_STRICT_PREREQS="${FUSE_STRICT_PREREQS:-0}"
 FUSE_UMOUNT_TIMEOUT="${FUSE_UMOUNT_TIMEOUT:-60s}"
 CLI_SOURCE="${CLI_SOURCE:-build}"
@@ -42,7 +45,6 @@ PJDFSTEST_TIMEOUT_S="${PJDFSTEST_TIMEOUT_S:-900}"
 PJDFSTEST_ALLOW_NONROOT="${PJDFSTEST_ALLOW_NONROOT:-0}"
 PJDFSTEST_MOUNT_ALLOW_OTHER="${PJDFSTEST_MOUNT_ALLOW_OTHER:-auto}"
 
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
 FEATURE_MATRIX_REPORT_DIR="${FEATURE_MATRIX_REPORT_DIR:-$REPO_ROOT/e2e/reports}"
 TS="$(date +%Y%m%d-%H%M%S)"
@@ -1284,7 +1286,7 @@ main() {
     record "PASS" "Provisioning" "use provided DRIVE9_API_KEY" "provided"
   else
     local resp code body
-    resp=$(curl_body_code POST "$BASE/v1/provision")
+    resp=$(drive9_provision_curl_body_code "$BASE" || true)
     code=$(http_code "$resp")
     body=$(json_body "$resp")
     if [ "$code" = "202" ]; then

e2e/api-smoke-test-existing-key.sh

@@ -3,6 +3,10 @@
 
 set -euo pipefail
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+. "$SCRIPT_DIR/tmp-helper.sh"
+drive9_e2e_init_tmpdir
+
 BASE="${DRIVE9_BASE:-http://127.0.0.1:9009}"
 API_KEY="${DRIVE9_API_KEY:-}"
 POLL_TIMEOUT_S="${POLL_TIMEOUT_S:-60}"

e2e/api-smoke-test.sh

@@ -21,6 +21,7 @@
 set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+. "$SCRIPT_DIR/provision-helper.sh"
 BASE="${DRIVE9_BASE:-http://127.0.0.1:9009}"
 DRIVE9_IMAGE_FIXTURE_PATH="${DRIVE9_IMAGE_FIXTURE_PATH:-$SCRIPT_DIR/fixtures/cat03.jpg}"
 POLL_TIMEOUT_S="${POLL_TIMEOUT_S:-120}"
@@ -222,18 +223,18 @@ BACKEND_DIR="${ROOT_DIR}/backend/go"
 FRONTEND_DIR="${ROOT_DIR}/frontend/web"
 BATCH_DIR="${ROOT_DIR}/batch"
 LARGE_FILE_BYTES=$((LARGE_FILE_MB * 1024 * 1024))
-LARGE_FILE_LOCAL="/tmp/drive9-e2e-large-${TS}.bin"
-LARGE_FILE_DOWNLOADED="/tmp/drive9-e2e-large-${TS}.download.bin"
+LARGE_FILE_LOCAL="$(drive9_e2e_tmp_path "drive9-e2e-large-${TS}.bin")"
+LARGE_FILE_DOWNLOADED="$(drive9_e2e_tmp_path "drive9-e2e-large-${TS}.download.bin")"
 LARGE_REMOTE_DIR="${ROOT_DIR}/large"
 LARGE_REMOTE_FILE="${LARGE_REMOTE_DIR}/blob-${LARGE_FILE_MB}m.bin"
-IMAGE_LOCAL="/tmp/drive9-e2e-image-${TS}.jpg"
+IMAGE_LOCAL="$(drive9_e2e_tmp_path "drive9-e2e-image-${TS}.jpg")"
 IMAGE_REMOTE="${ROOT_DIR}/assets/icon-${TS}.jpg"
 SEM_TEXT_TARGET="${ROOT_DIR}/notes/cat-story-${TS}.txt"
 SEM_TEXT_OTHER="${ROOT_DIR}/notes/dog-story-${TS}.txt"
 IMAGE_CAPTION_REMOTE="${ROOT_DIR}/assets/icon-${TS}.caption.txt"
 
 step "1" "Provision tenant"
-resp=$(curl_body_code POST "$BASE/v1/provision")
+resp=$(drive9_provision_curl_body_code "$BASE" || true)
 code=$(http_code "$resp")
 body=$(json_body "$resp")
 check_eq "POST /v1/provision returns 202" "$code" "202"