From 79cd6fc47325c0350b3f949135e321088f628ced Mon Sep 17 00:00:00 2001 From: HoangDucBach Date: Fri, 11 Sep 2026 11:17:17 +0700 Subject: [PATCH 01/70] docs: show recall() scoringWeights on SKILL.md (WALM-616) The TypeScript gap already shipped in 0.1.6. Agent docs still listed the old RecallOptions, which is how GH #708 concluded recency ranking was unreachable. --- SKILL.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/SKILL.md b/SKILL.md index 062f727e4..892904d17 100644 --- a/SKILL.md +++ b/SKILL.md @@ -152,7 +152,7 @@ const stored = await memwal.waitForRememberJob(accepted.job_id, { |---|---|---| | `remember(text, namespace?)` | Accept one memory job immediately | `{ job_id, status }` | | `rememberAndWait(text, namespace?, opts?)` | Store one memory and wait for completion | `{ id, job_id, blob_id, owner, namespace }` | -| `recall({ query, limit?, topK?, namespace?, maxDistance? })` *(preferred)* or `recall(query, limit?, namespace?)` | Semantic search for memories | `{ results: [{ blob_id, text, distance }], total }` | +| `recall({ query, limit?, topK?, namespace?, maxDistance?, sort?, scoringWeights? })` *(preferred)* or `recall(query, limit?, namespace?)` | Semantic search for memories | `{ results: [{ blob_id, text, distance }], total }` | | `analyze(text, namespace?)` | Extract facts and accept one memory job per fact | `{ job_ids, facts, fact_count, status, owner }` | | `analyzeAndWait(text, namespace?, opts?)` | Extract facts and wait for all fact jobs to complete | `{ results, facts, total, succeeded, failed, owner }` | | `restore(namespace, limit?)` | Rebuild missing index entries from Walrus | `{ restored, skipped, failed, total, namespace, owner, truncated }` | @@ -208,6 +208,8 @@ interface RecallOptions { topK?: number; // alias of limit; if both are set, topK wins namespace?: string; maxDistance?: number; + sort?: "relevance" | "recent"; + scoringWeights?: ScoringWeights; // sent as scoring_weights; omit to keep cosine order } // `topK` and `limit` are aliases for the same value; if both are provided, `topK` takes precedence. From 89e29ac7a540c6cd13bf66d5e4efe72dec15e4f6 Mon Sep 17 00:00:00 2001 From: HoangDucBach Date: Fri, 11 Sep 2026 15:38:01 +0700 Subject: [PATCH 02/70] docs: define ScoringWeights on SKILL.md (WALM-616) Agents reading only this file still could not see the weight fields. --- SKILL.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/SKILL.md b/SKILL.md index 892904d17..de3272a0b 100644 --- a/SKILL.md +++ b/SKILL.md @@ -212,7 +212,15 @@ interface RecallOptions { scoringWeights?: ScoringWeights; // sent as scoring_weights; omit to keep cosine order } +interface ScoringWeights { + semantic?: number; // default 1 + recency?: number; // default 0 + recencyHalfLifeDays?: number; // default 30 + importance?: number; // default 0 +} + // `topK` and `limit` are aliases for the same value; if both are provided, `topK` takes precedence. +// `sort: "recent"` is newest-wins (over-fetch then write-time). `scoringWeights` only re-ranks the cosine window. interface RememberBulkAcceptedResult { job_ids: string[]; From 7aec0197f37921b01945f778516d9a8426ff69ed Mon Sep 17 00:00:00 2001 From: Le Tien Phat <91601109+Niko1444@users.noreply.github.com> Date: Thu, 24 Sep 2026 13:58:24 +0700 Subject: [PATCH 03/70] fix(relayer): restart a failed same-key job on the insert-race path too (WALM-670) --- services/server/src/routes/remember.rs | 390 ++++++++++++++----------- 1 file changed, 217 insertions(+), 173 deletions(-) diff --git a/services/server/src/routes/remember.rs b/services/server/src/routes/remember.rs index 4085479e0..6d78518ac 100644 --- a/services/server/src/routes/remember.rs +++ b/services/server/src/routes/remember.rs @@ -800,123 +800,17 @@ pub async fn remember( )); } } - // A previous attempt under this key permanently failed. Idempotency - // should suppress duplicate *successes*, not pin *failures* — BUT only - // restart a failure that never minted a paid blob. A `failed` row that - // still carries a `blob_id` was marked failed *after* a successful - // mint (a recovery-handoff failure or the stale sweeper); restarting - // it would clear + re-upload and DISCARD the paid blob. So: reset only - // when there's no blob; otherwise leave the row (preserve the mint for - // recovery) and return it as-is. - if existing_status == "failed" && existing_blob_id.is_none() { - let claimed = claim_remember_preparation(state.db.pool(), &existing_id).await?; - if let Some(token) = claimed { - spawn_persisted_remember_preparation( - Arc::clone(&state), - existing_id.clone(), - token, - text, - owner_owned, - auth.account_id.clone(), - namespace_owned, - auth.public_key.clone(), - ); - return Ok(( - StatusCode::ACCEPTED, - Json(RememberAcceptedResponse { - job_id: existing_id, - status: "pending".to_string(), - }), - )); - } - - // Losing the claim now means a concurrent retry took it, not - // that the TTL blocked us — `failed` rows are re-claimable - // immediately. Report whatever that winner left behind rather - // than asserting "pending" on its behalf: answering with a - // state we did not reach is what told callers a dead job was - // queued. - let actual: String = sqlx::query_scalar( - "SELECT status FROM remember_jobs WHERE id = $1", - ) - .bind(&existing_id) - .fetch_optional(state.db.pool()) - .await - .map_err(|e| AppError::Internal(format!("Failed to re-read job status: {}", e)))? - .unwrap_or_else(|| existing_status.clone()); - - return Ok(( - StatusCode::ACCEPTED, - Json(RememberAcceptedResponse { - job_id: existing_id, - status: actual, - }), - )); - } - - if existing_status == "pending" && existing_blob_id.is_none() { - if let Some(token) = - claim_remember_preparation(state.db.pool(), &existing_id).await? - { - spawn_persisted_remember_preparation( - Arc::clone(&state), - existing_id.clone(), - token, - text, - owner_owned, - auth.account_id.clone(), - namespace_owned, - auth.public_key.clone(), - ); - } - return Ok(( - StatusCode::ACCEPTED, - Json(RememberAcceptedResponse { - job_id: existing_id, - status: "pending".to_string(), - }), - )); - } - - if matches!(existing_status.as_str(), "failed" | "uploaded") - && existing_blob_id.is_some() - { - // Claim first, enqueue second, publish `uploaded` last. If the - // process dies before enqueue, the lease expires and a later - // retry reclaims it; no durable state falsely says recovered. - if let Some(token) = claim_paid_recovery(state.db.pool(), &existing_id).await? { - enqueue_persisted_paid_recovery(&state, &existing_id, owner, namespace).await?; - sqlx::query( - "UPDATE remember_jobs SET status = 'uploaded', error_msg = NULL, updated_at = NOW() WHERE id = $1 AND recovery_claim_token = $2 AND blob_id IS NOT NULL AND status IN ('failed', 'uploaded')", - ) - .bind(&existing_id) - .bind(&token) - .execute(state.db.pool()) - .await - .map_err(|e| AppError::Internal(format!("Failed to publish paid recovery: {}", e)))?; - } - return Ok(( - StatusCode::ACCEPTED, - Json(RememberAcceptedResponse { - job_id: existing_id, - status: "uploaded".to_string(), - }), - )); - } - tracing::info!( - "remember idempotent hit: job_id={} owner={} ns={} status={}", + return collapse_onto_existing_job( + &state, + &auth, existing_id, - owner, - namespace, existing_status, - ); - return Ok(( - StatusCode::ACCEPTED, - Json(RememberAcceptedResponse { - job_id: existing_id, - status: existing_status, - }), - )); + existing_blob_id.is_some(), + text, + owner_owned, + namespace_owned, + ) + .await; } } @@ -957,7 +851,7 @@ pub async fn remember( // job rather than spawning a duplicate write. if inserted.rows_affected() == 0 { if let Some(key) = body.idempotency_key.as_deref() { - if let Some((existing_id, existing_status, _blob_id, existing_fingerprint)) = + if let Some((existing_id, existing_status, existing_blob_id, existing_fingerprint)) = find_remember_job_by_key(state.db.pool(), owner, key).await? { // Lost the race to a concurrent same-key request with DIFFERENT @@ -976,29 +870,17 @@ pub async fn remember( owner, namespace, ); - if existing_status == "pending" { - if let Some(token) = - claim_remember_preparation(state.db.pool(), &existing_id).await? - { - spawn_persisted_remember_preparation( - Arc::clone(&state), - existing_id.clone(), - token, - text, - owner_owned, - auth.account_id.clone(), - namespace_owned, - auth.public_key.clone(), - ); - } - } - return Ok(( - StatusCode::ACCEPTED, - Json(RememberAcceptedResponse { - job_id: existing_id, - status: existing_status, - }), - )); + return collapse_onto_existing_job( + &state, + &auth, + existing_id, + existing_status, + existing_blob_id.is_some(), + text, + owner_owned, + namespace_owned, + ) + .await; } } return Err(AppError::Internal( @@ -1036,6 +918,134 @@ pub async fn remember( )) } +/// What a same-key POST does with the row already bound to that key. Shared by +/// the pre-insert lookup and the insert-race loser, so a request that loses the +/// race to a row that has already failed still restarts or recovers it. +#[derive(Debug, PartialEq, Eq)] +enum ExistingJobAction { + /// Failed before minting: restart in place under the same job id. + RestartFailed, + /// Pending with no blob: re-drive preparation if nothing holds the claim. + RedrivePending, + /// A paid blob exists: re-drive metadata/transfer, never re-upload. + RecoverPaidBlob, + /// In flight or done: hand back the job as it is. + ReturnAsIs, +} + +fn existing_job_action(status: &str, has_blob: bool) -> ExistingJobAction { + match (status, has_blob) { + ("failed", false) => ExistingJobAction::RestartFailed, + ("pending", false) => ExistingJobAction::RedrivePending, + ("failed" | "uploaded", true) => ExistingJobAction::RecoverPaidBlob, + _ => ExistingJobAction::ReturnAsIs, + } +} + +/// Answer a same-key POST from the row already bound to the key. +#[allow(clippy::too_many_arguments)] +async fn collapse_onto_existing_job( + state: &Arc, + auth: &AuthInfo, + existing_id: String, + existing_status: String, + has_blob: bool, + text: String, + owner: String, + namespace: String, +) -> Result<(StatusCode, Json), AppError> { + let accepted = |job_id: String, status: String| { + Ok(( + StatusCode::ACCEPTED, + Json(RememberAcceptedResponse { job_id, status }), + )) + }; + match existing_job_action(&existing_status, has_blob) { + ExistingJobAction::RestartFailed => { + // A previous attempt under this key permanently failed. Idempotency + // should suppress duplicate *successes*, not pin *failures* — BUT only + // restart a failure that never minted a paid blob. A `failed` row that + // still carries a `blob_id` was marked failed *after* a successful + // mint (a recovery-handoff failure or the stale sweeper); restarting + // it would clear + re-upload and DISCARD the paid blob. So: reset only + // when there's no blob; otherwise leave the row (preserve the mint for + // recovery) and return it as-is. + if let Some(token) = claim_remember_preparation(state.db.pool(), &existing_id).await? { + spawn_persisted_remember_preparation( + Arc::clone(state), + existing_id.clone(), + token, + text, + owner, + auth.account_id.clone(), + namespace, + auth.public_key.clone(), + ); + return accepted(existing_id, "pending".to_string()); + } + + // Losing the claim now means a concurrent retry took it, not + // that the TTL blocked us — `failed` rows are re-claimable + // immediately. Report whatever that winner left behind rather + // than asserting "pending" on its behalf: answering with a + // state we did not reach is what told callers a dead job was + // queued. + let actual: String = + sqlx::query_scalar("SELECT status FROM remember_jobs WHERE id = $1") + .bind(&existing_id) + .fetch_optional(state.db.pool()) + .await + .map_err(|e| { + AppError::Internal(format!("Failed to re-read job status: {}", e)) + })? + .unwrap_or_else(|| existing_status.clone()); + accepted(existing_id, actual) + } + ExistingJobAction::RedrivePending => { + if let Some(token) = claim_remember_preparation(state.db.pool(), &existing_id).await? { + spawn_persisted_remember_preparation( + Arc::clone(state), + existing_id.clone(), + token, + text, + owner, + auth.account_id.clone(), + namespace, + auth.public_key.clone(), + ); + } + accepted(existing_id, "pending".to_string()) + } + ExistingJobAction::RecoverPaidBlob => { + // Claim first, enqueue second, publish `uploaded` last. If the + // process dies before enqueue, the lease expires and a later + // retry reclaims it; no durable state falsely says recovered. + if let Some(token) = claim_paid_recovery(state.db.pool(), &existing_id).await? { + enqueue_persisted_paid_recovery(state, &existing_id, &owner, &namespace).await?; + sqlx::query( + "UPDATE remember_jobs SET status = 'uploaded', error_msg = NULL, updated_at = NOW() WHERE id = $1 AND recovery_claim_token = $2 AND blob_id IS NOT NULL AND status IN ('failed', 'uploaded')", + ) + .bind(&existing_id) + .bind(&token) + .execute(state.db.pool()) + .await + .map_err(|e| AppError::Internal(format!("Failed to publish paid recovery: {}", e)))?; + } + accepted(existing_id, "uploaded".to_string()) + } + ExistingJobAction::ReturnAsIs => { + tracing::info!( + "remember idempotent hit: job_id={} owner={} ns={} status={}", + existing_id, + owner, + namespace, + existing_status, + ); + accepted(existing_id, existing_status) + } + } +} + /// Whether to spawn a fresh background write after attempting to reset a failed /// job. Spawn only when THIS request flipped exactly one row out of `failed` /// (the reset UPDATE is guarded by `AND status='failed'`), so a concurrent @@ -1624,11 +1634,12 @@ pub async fn remember_manual( mod tests { use super::{ batch_summary_inputs, build_bulk_status_results, claim_remember_preparation, - find_remember_job_by_key, paid_recovery_operation, redact_hex_addresses, - request_fingerprint, sanitize_job_error_for_client, should_spawn_after_reset, - split_text_chunks, summarize_for_embedding, validate_idempotency_key, - INFRA_JOB_ERROR_MESSAGE, INFRA_JOB_RETRYING_MESSAGE, MAX_IDEMPOTENCY_KEY_BYTES, - MAX_REMEMBER_TEXT_BYTES, SUMMARIZE_BATCH_INPUT_BYTES, SUMMARIZE_CHUNK_BYTES, + existing_job_action, find_remember_job_by_key, paid_recovery_operation, + redact_hex_addresses, request_fingerprint, sanitize_job_error_for_client, + should_spawn_after_reset, split_text_chunks, summarize_for_embedding, + validate_idempotency_key, ExistingJobAction, INFRA_JOB_ERROR_MESSAGE, + INFRA_JOB_RETRYING_MESSAGE, MAX_IDEMPOTENCY_KEY_BYTES, MAX_REMEMBER_TEXT_BYTES, + SUMMARIZE_BATCH_INPUT_BYTES, SUMMARIZE_CHUNK_BYTES, }; use crate::jobs::WalletOperation; use crate::types::AppError; @@ -2131,19 +2142,31 @@ mod tests { .await; } - // GH #477 RC-5: a same-key retry of a FAILED job restarts it in place rather - // than pinning `failed` forever. This pins the reset SQL: it flips only a - // `failed` row (guarded by AND status='failed'), clearing blob columns/error. + #[test] + fn same_key_action_matches_row_state() { + use ExistingJobAction::*; + assert_eq!(existing_job_action("failed", false), RestartFailed); + assert_eq!(existing_job_action("pending", false), RedrivePending); + assert_eq!(existing_job_action("failed", true), RecoverPaidBlob); + assert_eq!(existing_job_action("uploaded", true), RecoverPaidBlob); + assert_eq!(existing_job_action("running", false), ReturnAsIs); + assert_eq!(existing_job_action("running", true), ReturnAsIs); + assert_eq!(existing_job_action("done", true), ReturnAsIs); + assert_eq!(existing_job_action("uploaded", false), ReturnAsIs); + } + + // GH #966: a sequential same-key retry after a terminal failure restarts the + // SAME job instead of pinning it, and a retry while that restart is in + // flight collapses onto it without a second row. #[tokio::test] - async fn failed_job_reset_flips_only_failed_no_blob_and_clears_state() { + async fn sequential_retry_after_terminal_failure_restarts_the_same_job() { let pool = idem_test_pool().await; let owner = format!("0xowner-{}", uuid::Uuid::new_v4()); let key = "retry-key"; let job_id = format!("remember-job-{}", uuid::Uuid::new_v4()); - - // A failed job with NO blob (genuine pre-mint failure) + an error message. sqlx::query( - "INSERT INTO remember_jobs (id, owner, namespace, status, idempotency_key, error_msg) VALUES ($1, $2, 'ns', 'failed', $3, 'boom')", + "INSERT INTO remember_jobs (id, owner, namespace, status, idempotency_key, error_msg, prepare_claimed_at, prepare_claim_token) + VALUES ($1, $2, 'ns', 'failed', $3, 'walrus upload failed: exhausted', NOW(), 'dead-token')", ) .bind(&job_id) .bind(&owner) @@ -2152,19 +2175,21 @@ mod tests { .await .unwrap(); - // The handler's reset SQL: only a failed row with NO blob is restarted. - let reset = sqlx::query( - "UPDATE remember_jobs SET status = 'running', blob_object_id = NULL, error_msg = NULL, updated_at = NOW() WHERE id = $1 AND status = 'failed' AND blob_id IS NULL", - ) - .bind(&job_id) - .execute(&pool) - .await - .unwrap(); + // Retry #1 after the terminal failure: the key resolves to the failed + // row, which is restartable now, not after the TTL. + let (found_id, status, blob, _) = find_remember_job_by_key(&pool, &owner, key) + .await + .unwrap() + .expect("row bound to key"); + assert_eq!(found_id, job_id); assert_eq!( - reset.rows_affected(), - 1, - "a failed no-blob row must be reset" + existing_job_action(&status, blob.is_some()), + ExistingJobAction::RestartFailed ); + assert!(claim_remember_preparation(&pool, &job_id) + .await + .unwrap() + .is_some()); let row: (String, Option) = sqlx::query_as("SELECT status, error_msg FROM remember_jobs WHERE id = $1") @@ -2172,22 +2197,41 @@ mod tests { .fetch_one(&pool) .await .unwrap(); - assert_eq!(row.0, "running"); - assert_eq!(row.1, None, "error_msg cleared"); + assert_eq!( + row, + ("pending".to_string(), None), + "restarted in place, error cleared" + ); - // Re-running on the now-`running` row is a no-op → no double-spawn. - let again = sqlx::query( - "UPDATE remember_jobs SET status = 'running' WHERE id = $1 AND status = 'failed' AND blob_id IS NULL", + // Retry #2 while that restart is in flight: same row, no new claim, and + // the insert path cannot mint a second job for the key. + let (again_id, again_status, again_blob, _) = find_remember_job_by_key(&pool, &owner, key) + .await + .unwrap() + .unwrap(); + assert_eq!(again_id, job_id); + assert_eq!( + existing_job_action(&again_status, again_blob.is_some()), + ExistingJobAction::RedrivePending + ); + assert!( + claim_remember_preparation(&pool, &job_id) + .await + .unwrap() + .is_none(), + "a live restart keeps its claim" + ); + let dup = sqlx::query( + "INSERT INTO remember_jobs (id, owner, namespace, status, idempotency_key) VALUES ($1, $2, 'ns', 'pending', $3) + ON CONFLICT (owner, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING", ) - .bind(&job_id) + .bind(uuid::Uuid::new_v4().to_string()) + .bind(&owner) + .bind(key) .execute(&pool) .await .unwrap(); - assert_eq!( - again.rows_affected(), - 0, - "reset must not re-fire on a live row" - ); + assert_eq!(dup.rows_affected(), 0, "no second paid job under the key"); let _ = sqlx::query("DELETE FROM remember_jobs WHERE owner = $1") .bind(&owner) From 25656ad45c389a8b03a81fef4975a6ac83e0f35d Mon Sep 17 00:00:00 2001 From: Le Tien Phat <91601109+Niko1444@users.noreply.github.com> Date: Thu, 24 Sep 2026 14:00:22 +0700 Subject: [PATCH 04/70] fix(sdk): say what the relayer last reported when a remember wait times out (WALM-670) --- packages/sdk/CHANGELOG.md | 1 + packages/sdk/src/memwal.ts | 39 +++++-- .../sdk/test/remember-idempotency.test.mjs | 100 ++++++++++++++++++ packages/sdk/test/remember-job-poll.test.mjs | 19 +++- 4 files changed, 151 insertions(+), 8 deletions(-) diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index 4d9234d4f..61e49d6c4 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -5,6 +5,7 @@ ### Fixed - `rememberBulkAndWait` / `waitForRememberJobs` no longer resolve a batch they could not read as if it were still uploading. The status endpoint counts against the delegate-key budget, and a 429 on every poll used to be retried silently until the wait ran out, leaving each item as `timeout` with "polling timed out", even when nothing had been stored. When no poll got through, the wait now makes one confirming read. If that is rate-limited too, it throws `MemWalRateLimited` (`status: 429`, `jobIds`, `retryAfterSeconds`). An item still unsettled at the deadline names its last known status, or says no read got through. (WALM-671, #967) +- A `rememberAndWait` / `waitForRememberJob` that runs out of time now raises `MemWalRememberJobTimeout` (still `status: 504`) carrying `lastStatus` and `serverError`, the last state the relayer reported. Under upload congestion the relayer keeps a job `running` for up to ~25 minutes and says so, but the 504 used to drop that, so a job that was still retrying looked dead and callers dropped their idempotency key to force a new paid write. When no poll got through, the message says the state is unknown instead. (WALM-670, #966) ## 0.1.9 diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index 22c7365b0..b619479ea 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -236,6 +236,33 @@ function requestTimeoutError(method: string, path: string, ms: number): Error { return err; } +/** + * The 504 a remember wait raises when its budget runs out. + * + * Under upload congestion the relayer keeps a job `running` for tens of + * minutes, far past a 60s wait, and says so in the job's `error`. Carrying the + * last status the relayer reported is what lets a caller tell "still retrying, + * wait on the same key" from "we never got an answer" (GH #966). + */ +function rememberJobTimeoutError( + jobId: string, + timeoutMs: number, + lastSeen: RememberJobStatus | undefined, + lastRefusal: number | undefined, + sawRateLimit: boolean, +): Error { + const serverError = lastSeen?.error ? redactInternalUrls(lastSeen.error) : undefined; + // "no status read got through (last poll: HTTP 429)" already names the rate limit. + const rateLimited = sawRateLimit && (lastSeen || lastRefusal !== 429) ? "; wait hit a rate limit (429)" : ""; + const detail = lastSeen + ? `last status: ${lastSeen.status}${serverError ? ` (${serverError})` : ""}. ` + + `Retry with the same idempotency key to keep waiting on this job; rememberAndWait does this for you.` + : `no status read got through (last poll: HTTP ${lastRefusal ?? 0}), so the job's state is unknown.`; + const err = new Error(`remember job timed out after ${timeoutMs}ms (job_id=${jobId})${rateLimited}; ${detail}`); + err.name = "MemWalRememberJobTimeout"; + return Object.assign(err, { status: 504, jobId, lastStatus: lastSeen?.status, serverError }); +} + function isTransientPollingStatus(status: number): boolean { return status === 0 || status === 429 || status >= 500; } @@ -499,6 +526,8 @@ export class MemWal { let attempt = 0; let retryAfterMs = 0; let sawRateLimit = false; + let lastSeen: RememberJobStatus | undefined; + let lastRefusal: number | undefined; while (Date.now() < deadline) { // A retry-after the server just gave us wins over our own curve; @@ -529,6 +558,7 @@ export class MemWal { } catch (err) { const httpStatus = (err as { status?: number }).status ?? 0; if (isTransientPollingStatus(httpStatus)) { + lastRefusal = httpStatus; if (httpStatus === 429) sawRateLimit = true; retryAfterMs = retryAfterDelayMs(err, deadline); continue; @@ -542,6 +572,7 @@ export class MemWal { jobId, }); } + lastSeen = status; if (status.status === "done") { return { @@ -562,13 +593,7 @@ export class MemWal { } } - throw Object.assign( - new Error( - `remember job timed out after ${timeoutMs}ms (job_id=${jobId})` + - (sawRateLimit ? "; wait hit a rate limit (429)" : ""), - ), - { status: 504, jobId }, - ); + throw rememberJobTimeoutError(jobId, timeoutMs, lastSeen, lastRefusal, sawRateLimit); } /** diff --git a/packages/sdk/test/remember-idempotency.test.mjs b/packages/sdk/test/remember-idempotency.test.mjs index 52428714f..1461ee189 100644 --- a/packages/sdk/test/remember-idempotency.test.mjs +++ b/packages/sdk/test/remember-idempotency.test.mjs @@ -62,3 +62,103 @@ test("rememberAndWait reuses its generated key and job after polling timeout", a assert.equal(posted.length, 2); assert.equal(posted[0].idempotency_key, posted[1].idempotency_key); }); + +function relayer(routes) { + return async (url, init = {}) => { + const path = new URL(url).pathname; + if (path === "/version") { + return Response.json({ + apiVersion: "1.0.0", + relayerVersion: "1.0.0", + minSupportedSdk: { typescript: "0.0.4" }, + }); + } + if (path === "/api/config") return Response.json({ packageId: "0x1", network: "testnet" }); + const handler = routes[path]; + if (!handler) throw new Error(`unexpected request ${path}`); + return handler(init); + }; +} + +function client() { + const c = MemWal.create({ + key: new Uint8Array(32).fill(1), + accountId: "0x1", + serverUrl: "https://relayer.example", + }); + c.buildSealSession = async () => "test-session"; + return c; +} + +const RETRYING = + "Temporary capacity limit on Walrus Memory infrastructure; the hosted relayer is still retrying this upload."; + +test("a timed-out wait reports the job's last status and the relayer's message", async () => { + globalThis.fetch = relayer({ + "/api/remember/slow-job": () => + Response.json({ job_id: "slow-job", status: "running", error: RETRYING }), + }); + await assert.rejects( + client().waitForRememberJob("slow-job", { pollIntervalMs: 0, timeoutMs: 300 }), + (err) => { + assert.equal(err.name, "MemWalRememberJobTimeout"); + assert.equal(err.status, 504); + assert.equal(err.jobId, "slow-job"); + assert.equal(err.lastStatus, "running"); + assert.match(err.serverError, /still retrying/); + assert.match(err.message, /^remember job timed out after 300ms/); + assert.match(err.message, /last status: running/); + return true; + }, + ); +}); + +test("a timed-out wait whose every poll was refused says the state is unknown", async () => { + globalThis.fetch = relayer({ + "/api/remember/hidden-job": () => + Response.json({ error: "rate limited", retry_after_seconds: 0 }, { status: 429 }), + }); + await assert.rejects( + client().waitForRememberJob("hidden-job", { pollIntervalMs: 0, timeoutMs: 300 }), + (err) => { + assert.equal(err.status, 504); + assert.equal(err.lastStatus, undefined); + assert.match(err.message, /no status read got through \(last poll: HTTP 429\)/); + return true; + }, + ); +}); + +// GH #966: a retry after a terminal failure must reach the relayer under the +// same key, which restarts the same job instead of minting a second paid one. +test("rememberAndWait retried after a terminal failure restarts the same job under the same key", async () => { + const posted = []; + let phase = "first"; + globalThis.fetch = relayer({ + "/api/remember": (init) => { + posted.push(JSON.parse(init.body)); + return Response.json({ job_id: "job-966", status: "pending" }, { status: 202 }); + }, + "/api/remember/job-966": () => + phase === "first" + ? Response.json({ job_id: "job-966", status: "failed", error: "walrus upload failed" }) + : Response.json({ + job_id: "job-966", + status: "done", + blob_id: "blob-966", + owner: "0x1", + namespace: "default", + }), + }); + const c = client(); + await assert.rejects( + c.rememberAndWait("fact", undefined, { pollIntervalMs: 0, timeoutMs: 1_000 }), + /failed/, + ); + phase = "second"; + const result = await c.rememberAndWait("fact", undefined, { pollIntervalMs: 0, timeoutMs: 1_000 }); + assert.equal(result.job_id, "job-966"); + assert.equal(result.blob_id, "blob-966"); + assert.equal(posted.length, 2); + assert.equal(posted[0].idempotency_key, posted[1].idempotency_key); +}); diff --git a/packages/sdk/test/remember-job-poll.test.mjs b/packages/sdk/test/remember-job-poll.test.mjs index f0d962d64..4ed3f8ac4 100644 --- a/packages/sdk/test/remember-job-poll.test.mjs +++ b/packages/sdk/test/remember-job-poll.test.mjs @@ -58,11 +58,28 @@ test("a remember wait that only sees 429 names the rate limit on timeout", async const started = Date.now(); await assert.rejects( client().waitForRememberJob("job-1", { pollIntervalMs: 1, timeoutMs: 400 }), - /remember job timed out after 400ms \(job_id=job-1\); wait hit a rate limit \(429\)/, + /remember job timed out after 400ms \(job_id=job-1\); no status read got through \(last poll: HTTP 429\)/, ); assert.ok(Date.now() - started < 3_000, "a 600s Retry-After must stay inside the wait budget"); }); +test("a remember wait that saw the job, then 429, keeps both on timeout", async () => { + let calls = 0; + stub((path) => { + if (path !== "/api/remember/job-1") throw new Error(`unexpected request ${path}`); + calls += 1; + if (calls === 1) return Response.json({ job_id: "job-1", status: "running" }); + return rateLimit(); + }); + + await assert.rejects( + client().waitForRememberJob("job-1", { pollIntervalMs: 1, timeoutMs: 400 }), + (err) => + /\(job_id=job-1\); wait hit a rate limit \(429\); last status: running\./.test(err.message) + && err.lastStatus === "running", + ); +}); + test("a bulk remember wait that only sees 429 throws instead of timing out", async () => { stub((path) => { if (path === "/api/remember/bulk/status") return rateLimit(); From 8df38e50cabdca54b0daac2664c3aeaa1941de1f Mon Sep 17 00:00:00 2001 From: Le Tien Phat <91601109+Niko1444@users.noreply.github.com> Date: Thu, 24 Sep 2026 14:01:41 +0700 Subject: [PATCH 05/70] fix(python-sdk): say what the relayer last reported when a remember wait times out (WALM-670) --- packages/python-sdk-memwal/CHANGELOG.md | 1 + packages/python-sdk-memwal/memwal/client.py | 60 ++++++++++++++---- .../python-sdk-memwal/tests/test_client.py | 61 +++++++++++++++++++ 3 files changed, 109 insertions(+), 13 deletions(-) diff --git a/packages/python-sdk-memwal/CHANGELOG.md b/packages/python-sdk-memwal/CHANGELOG.md index 333e550a3..514f2f512 100644 --- a/packages/python-sdk-memwal/CHANGELOG.md +++ b/packages/python-sdk-memwal/CHANGELOG.md @@ -5,6 +5,7 @@ ### Fixed - `wait_for_remember_jobs` / `remember_bulk_and_wait` no longer return a batch they could not read as if it were still uploading. When every status poll is rate-limited, the wait makes one confirming read and raises `MemWalRateLimited` (`status` 429, `job_ids`, `retry_after`) if that is refused too. An item still unsettled at the deadline names its last known status, or says no read got through. (WALM-671, #967) +- `MemWalRememberJobTimeout` carries `last_status`, `server_error` and `last_refusal`, and its message names the relayer's last reported state. Under upload congestion a job stays `running` for up to ~25 minutes while the relayer retries, and the timeout used to hide that, so a job that was still retrying looked dead. (WALM-670, #966) ## 0.1.12 diff --git a/packages/python-sdk-memwal/memwal/client.py b/packages/python-sdk-memwal/memwal/client.py index b5d0e09d0..050d017a5 100644 --- a/packages/python-sdk-memwal/memwal/client.py +++ b/packages/python-sdk-memwal/memwal/client.py @@ -471,6 +471,9 @@ async def wait_for_remember_job( attempt = 0 saw_rate_limit = False retry_after_ms = 0 + last_status: Optional[str] = None + server_error: Optional[str] = None + last_refusal: Optional[int] = None while _now_ms() < deadline_ms: if retry_after_ms > 0: @@ -489,6 +492,7 @@ async def wait_for_remember_job( ) except _HttpStatusError as err: if _is_transient_polling_status(err.status): + last_refusal = err.status if err.status == 429: saw_rate_limit = True retry_after_ms = _clamped_retry_after_ms(err, deadline_ms) @@ -498,6 +502,8 @@ async def wait_for_remember_job( status_str = data.get("status") if status_str is None or status_str == "not_found": raise MemWalRememberJobNotFound(job_id) + last_status = status_str + server_error = data.get("error") if status_str == "done": return RememberResult( @@ -514,7 +520,14 @@ async def wait_for_remember_job( # pending / running / uploaded — keep polling. - raise _remember_job_timeout(job_id, timeout_ms, saw_rate_limit) + raise MemWalRememberJobTimeout( + job_id=job_id, + timeout_ms=timeout_ms, + last_status=last_status, + server_error=server_error, + last_refusal=last_refusal, + saw_rate_limit=saw_rate_limit, + ) async def remember_and_wait( self, @@ -1671,24 +1684,45 @@ def __init__(self, job_ids: Sequence[str], retry_after: Optional[str] = None) -> class MemWalRememberJobTimeout(MemWalError): - """Polling loop exceeded the configured timeout.""" + """Polling loop exceeded the configured timeout. - def __init__(self, job_id: str, timeout_ms: int) -> None: + Carries the last status the relayer reported (GH #966): under upload + congestion a job stays ``running`` for tens of minutes, and without it a + job that is still retrying looks dead. + """ + + def __init__( + self, + job_id: str, + timeout_ms: int, + last_status: Optional[str] = None, + server_error: Optional[str] = None, + last_refusal: Optional[int] = None, + saw_rate_limit: bool = False, + ) -> None: + # "no status read got through (last poll: HTTP 429)" already names the rate limit. + named = last_status is None and last_refusal == 429 + rate_limited = "; wait hit a rate limit (429)" if saw_rate_limit and not named else "" + if last_status is not None: + suffix = f" ({_redact_internal_urls(server_error)})" if server_error else "" + detail = ( + f"last status: {last_status}{suffix}. Retry with the same idempotency key " + "to keep waiting on this job; remember_and_wait does this for you." + ) + else: + detail = ( + f"no status read got through (last poll: HTTP {last_refusal or 0}), " + "so the job's state is unknown." + ) super().__init__( - f"remember job timed out after {timeout_ms}ms (job_id={job_id})" + f"remember job timed out after {timeout_ms}ms (job_id={job_id}){rate_limited}; {detail}" ) self.status = 504 self.job_id = job_id self.timeout_ms = timeout_ms - - -def _remember_job_timeout( - job_id: str, timeout_ms: int, saw_rate_limit: bool -) -> MemWalRememberJobTimeout: - exc = MemWalRememberJobTimeout(job_id, timeout_ms) - if saw_rate_limit: - exc.args = (f"{exc.args[0]}; wait hit a rate limit (429)",) - return exc + self.last_status = last_status + self.server_error = server_error + self.last_refusal = last_refusal async def _discard_http_client(memwal: MemWal) -> None: diff --git a/packages/python-sdk-memwal/tests/test_client.py b/packages/python-sdk-memwal/tests/test_client.py index 6df617345..cda1f479d 100644 --- a/packages/python-sdk-memwal/tests/test_client.py +++ b/packages/python-sdk-memwal/tests/test_client.py @@ -24,6 +24,7 @@ MemWalCompatibilityError, MemWalError, MemWalRateLimited, + MemWalRememberJobTimeout, MemWalSync, _HttpStatusError, ) @@ -558,6 +559,66 @@ async def then_refused(job_ids: Sequence[str]) -> RememberBulkStatusResult: ) +class TestRememberJobTimeoutDetail: + async def test_reports_last_status_and_server_message( + self, memwal_client: MemWal + ) -> None: + async def running(*_args: Any, **_kwargs: Any) -> dict: + return { + "job_id": "slow-job", + "status": "running", + "error": "the hosted relayer is still retrying this upload", + } + + memwal_client._signed_request = running # type: ignore[method-assign] + with pytest.raises(MemWalRememberJobTimeout) as exc: + await memwal_client.wait_for_remember_job( + "slow-job", poll_interval_ms=0, timeout_ms=300 + ) + assert exc.value.status == 504 + assert exc.value.last_status == "running" + assert "still retrying" in (exc.value.server_error or "") + assert str(exc.value).startswith("remember job timed out after 300ms") + assert "last status: running" in str(exc.value) + + async def test_every_poll_refused_reports_unknown_state( + self, memwal_client: MemWal + ) -> None: + async def refused(*_args: Any, **_kwargs: Any) -> dict: + raise _HttpStatusError(429, '{"error":"rate limited"}') + + memwal_client._signed_request = refused # type: ignore[method-assign] + with pytest.raises(MemWalRememberJobTimeout) as exc: + await memwal_client.wait_for_remember_job( + "hidden-job", poll_interval_ms=0, timeout_ms=300 + ) + assert exc.value.last_status is None + assert exc.value.last_refusal == 429 + assert "no status read got through (last poll: HTTP 429)" in str(exc.value) + + async def test_seen_then_rate_limited_keeps_both( + self, memwal_client: MemWal + ) -> None: + calls = 0 + + async def running_then_refused(*_args: Any, **_kwargs: Any) -> dict: + nonlocal calls + calls += 1 + if calls == 1: + return {"job_id": "slow-job", "status": "running"} + raise _HttpStatusError(429, '{"error":"rate limited"}') + + memwal_client._signed_request = running_then_refused # type: ignore[method-assign] + with pytest.raises(MemWalRememberJobTimeout) as exc: + await memwal_client.wait_for_remember_job( + "slow-job", poll_interval_ms=0, timeout_ms=300 + ) + assert exc.value.last_status == "running" + assert "(job_id=slow-job); wait hit a rate limit (429); last status: running." in str( + exc.value + ) + + # ============================================================ # recall() tests # ============================================================ From 85c334863b625421bb60511a84018744af854c98 Mon Sep 17 00:00:00 2001 From: Le Tien Phat <91601109+Niko1444@users.noreply.github.com> Date: Thu, 24 Sep 2026 14:10:32 +0700 Subject: [PATCH 06/70] fix(python-sdk): say the retry reuses the job only on the same client (WALM-670) --- packages/python-sdk-memwal/memwal/client.py | 3 ++- packages/python-sdk-memwal/tests/test_client.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/python-sdk-memwal/memwal/client.py b/packages/python-sdk-memwal/memwal/client.py index 050d017a5..79a946080 100644 --- a/packages/python-sdk-memwal/memwal/client.py +++ b/packages/python-sdk-memwal/memwal/client.py @@ -1707,7 +1707,8 @@ def __init__( suffix = f" ({_redact_internal_urls(server_error)})" if server_error else "" detail = ( f"last status: {last_status}{suffix}. Retry with the same idempotency key " - "to keep waiting on this job; remember_and_wait does this for you." + "to keep waiting on this job; remember_and_wait does this for you " + "on the same client instance." ) else: detail = ( diff --git a/packages/python-sdk-memwal/tests/test_client.py b/packages/python-sdk-memwal/tests/test_client.py index cda1f479d..b35e6d8cb 100644 --- a/packages/python-sdk-memwal/tests/test_client.py +++ b/packages/python-sdk-memwal/tests/test_client.py @@ -580,6 +580,9 @@ async def running(*_args: Any, **_kwargs: Any) -> dict: assert "still retrying" in (exc.value.server_error or "") assert str(exc.value).startswith("remember job timed out after 300ms") assert "last status: running" in str(exc.value) + # Python keys are per client instance (uuid4), so only a retry on the + # same client reuses the job; a new client would mint a second write. + assert "on the same client instance" in str(exc.value) async def test_every_poll_refused_reports_unknown_state( self, memwal_client: MemWal From aa4259669e740f18925b398247c6a03e8019dbbb Mon Sep 17 00:00:00 2001 From: Le Tien Phat <91601109+Niko1444@users.noreply.github.com> Date: Thu, 24 Sep 2026 16:12:46 +0700 Subject: [PATCH 07/70] feat(sdk): listMemories() pages every memory without a query (WALM-388) Recall is similarity-ranked and limit-bounded, so a write can't be confirmed and a namespace can't be counted. listMemories() wraps the existing GET /v1/owners/{owner}/memories and reuses resolveOwner(). Metadata only: no blob fetch, no decrypt. The relayer has no namespace filter, so the optional namespace is applied client-side; a filtered page can be empty while has_more is true. Tombstone fields default for relayers older than WALM-363. --- docs/sdk/api-reference.md | 48 ++++++ packages/sdk/src/index.ts | 4 + packages/sdk/src/memwal.ts | 56 +++++++ packages/sdk/src/types.ts | 60 +++++++ packages/sdk/test/e2e/live.e2e.mjs | 50 ++++++ packages/sdk/test/list-memories.test.mjs | 191 +++++++++++++++++++++++ 6 files changed, 409 insertions(+) create mode 100644 packages/sdk/test/list-memories.test.mjs diff --git a/docs/sdk/api-reference.md b/docs/sdk/api-reference.md index ec931610e..95e018a30 100644 --- a/docs/sdk/api-reference.md +++ b/docs/sdk/api-reference.md @@ -312,6 +312,54 @@ while (more) { } ``` +### `listMemories(options?): Promise` + +List the memories this account holds, without a semantic query. Use it to confirm a write landed, count a namespace exactly, or find a bulk write that went to the wrong namespace. Returns metadata only, with no blob fetch or decryption; use `recall()` for content. + +- `options.cursor`: The previous page's `next_cursor`, to continue a walk or poll incrementally +- `options.limit`: Page size, counted before the `namespace` filter; the relayer defaults to `100` and clamps to `500` +- `options.namespace`: Keep only this namespace. Filtered client-side, so a page can be short or empty while `has_more` is still `true` + +**Returns:** + +```ts +{ + memories: Array<{ + memory_id: string; + namespace_id: string; // the namespace name + blob_id: string; + created_at: string; + updated_at: string; + size: number; // encrypted bytes + agent_id: string | null; + package_id: string | null; + status: string; + end_epoch: number | null; + expires_at: string | null; + importance: number | null; + }>; + next_cursor: string | null; + has_more: boolean; + snapshot_version: number; + deleted: Array<{ memory_id: string; namespace_id: string; deleted_at: string }>; + must_resync: boolean; // cursor too old: restart without one +} +``` + +Paginate on `has_more`, not on page length: + +```ts +let cursor: string | undefined; +let more = true; +let count = 0; +while (more) { + const page = await memwal.listMemories({ cursor, namespace: "work" }); + count += page.memories.length; + cursor = page.next_cursor ?? undefined; + more = page.has_more; +} +``` + ### `health(): Promise` Check relayer health. Does not require authentication — a successful response confirms the relayer is reachable, not that your `key`/`accountId` are valid. A signed call (e.g. `remember()`, `recall()`) can still fail with `401` immediately after a passing `health()`. diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 612bddaae..293dc30f0 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -79,4 +79,8 @@ export type { NamespaceSummary, NamespacesResult, ListNamespacesOptions, + MemorySummary, + DeletedMemorySummary, + MemoriesResult, + ListMemoriesOptions, } from "./types.js"; diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index 22c7365b0..857da923b 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -47,6 +47,8 @@ import type { RestoreResult, NamespacesResult, ListNamespacesOptions, + MemoriesResult, + ListMemoriesOptions, RememberBulkItem, RememberBulkOptions, RememberBulkResult, @@ -1233,6 +1235,60 @@ export class MemWal { }); } + /** + * List the memories this account holds, without a semantic query. + * + * Recall is similarity-ranked and limit-bounded, so it cannot confirm a + * write or count a namespace. This walks the full inventory instead. + * Returns metadata only — no blob fetch, no decryption; use `recall()` + * for content. + * + * `namespace` is filtered client-side: the relayer lists every namespace, + * so a filtered page can be short or empty while `has_more` is true. + * Paginate with `has_more`, NOT page length. + * + * ```ts + * let cursor: string | undefined; + * let more = true; + * let count = 0; + * while (more) { + * const page = await memwal.listMemories({ cursor, namespace: "work" }); + * count += page.memories.length; + * cursor = page.next_cursor ?? undefined; + * more = page.has_more; + * } + * ``` + */ + async listMemories(options: ListMemoriesOptions = {}): Promise { + const owner = await this.resolveOwner(); + + const params = new URLSearchParams(); + if (options.cursor !== undefined) params.set("updated_after", options.cursor); + if (options.limit !== undefined) params.set("limit", String(options.limit)); + const query = params.toString(); + + // Query string must be part of the signed path (see listNamespaces). + const path = `/v1/owners/${owner}/memories${query ? `?${query}` : ""}`; + + const raw = await this.signedRequest("GET", path, {}, [200], { + includeDelegateKey: false, + }); + // Relayers older than WALM-363 omit the tombstone fields. + const page: MemoriesResult = { + ...raw, + deleted: raw.deleted ?? [], + must_resync: raw.must_resync ?? false, + }; + if (options.namespace === undefined) return page; + + const namespace = options.namespace; + return { + ...page, + memories: page.memories.filter((m) => m.namespace_id === namespace), + deleted: page.deleted.filter((d) => d.namespace_id === namespace), + }; + } + /** * Resolve this account's owner address, memoised for the client's life. * diff --git a/packages/sdk/src/types.ts b/packages/sdk/src/types.ts index e253c1090..8052b249e 100644 --- a/packages/sdk/src/types.ts +++ b/packages/sdk/src/types.ts @@ -462,6 +462,66 @@ export interface ListNamespacesOptions { limit?: number; } +/** One memory in a `listMemories()` page. Mirrors the relayer wire shape. */ +export interface MemorySummary { + memory_id: string; + /** The namespace name (the same string passed to `remember()` / `recall()`). */ + namespace_id: string; + blob_id: string; + created_at: string; + /** The row's `updated_at` — the value the keyset cursor is built from. */ + updated_at: string; + /** Encrypted blob size in bytes. */ + size: number; + agent_id: string | null; + package_id: string | null; + status: string; + end_epoch: number | null; + expires_at: string | null; + importance: number | null; +} + +/** A memory removed since the cursor. Never mixed into `memories`. */ +export interface DeletedMemorySummary { + memory_id: string; + namespace_id: string; + deleted_at: string; +} + +/** Result from listMemories() */ +export interface MemoriesResult { + memories: MemorySummary[]; + /** Watermark to hand back as `cursor` on the next call. */ + next_cursor: string | null; + /** + * Authoritative "keep paginating" signal. Do NOT infer this from page + * length: the server clamps `limit`, and a `namespace` filter can leave a + * page short or empty while more pages remain. + */ + has_more: boolean; + snapshot_version: number; + /** Tombstones since the cursor, for incremental sync. */ + deleted: DeletedMemorySummary[]; + /** + * True when the cursor is too old for the relayer to know what was + * deleted since; restart the walk without a cursor. + */ + must_resync: boolean; +} + +/** Options for listMemories() */ +export interface ListMemoriesOptions { + /** Previous page's `next_cursor`, to continue a walk or poll incrementally. */ + cursor?: string; + /** Page size, counted before any `namespace` filter. Server defaults to 100 and clamps to 500. */ + limit?: number; + /** + * Keep only this namespace. Applied client-side: the relayer lists every + * namespace, so a filtered page can be empty while `has_more` is true. + */ + namespace?: string; +} + /** Result from restore() */ export interface RestoreResult { restored: number; diff --git a/packages/sdk/test/e2e/live.e2e.mjs b/packages/sdk/test/e2e/live.e2e.mjs index c0f3a16f4..0960ac7bf 100644 --- a/packages/sdk/test/e2e/live.e2e.mjs +++ b/packages/sdk/test/e2e/live.e2e.mjs @@ -278,4 +278,54 @@ describe("authenticated", { skip: requiresKey, concurrency: 1 }, () => { assert.deepEqual(none.results, []); assert.equal(none.total, 0); }); + + test("listMemories counts a fresh namespace exactly", async () => { + const namespace = `${E2E_NAMESPACE}-list`; + const mw = client(namespace); + const unique = randomUUID().replaceAll("-", "").slice(0, 8); + + const written = await mw.rememberBulkAndWait( + ["alpha", "beta", "gamma"].map((word) => ({ text: `SDK e2e list ${unique}: ${word}` })), + { timeoutMs: REMEMBER_TIMEOUT_MS }, + ); + assert.equal(written.succeeded, 3, `expected 3 writes, got ${JSON.stringify(written)}`); + const blobIds = new Set(written.results.map((r) => r.blob_id)); + + // Walk every page, filtered to this namespace. + const listed = []; + let cursor; + let more = true; + while (more) { + const page = await mw.listMemories({ cursor, namespace, limit: 100 }); + for (const m of page.memories) { + assert.equal(m.namespace_id, namespace); + assert.equal(typeof m.memory_id, "string"); + assert.equal(typeof m.blob_id, "string"); + assert.equal(typeof m.size, "number"); + assert.ok(!Number.isNaN(Date.parse(m.updated_at))); + } + listed.push(...page.memories); + assert.ok(Array.isArray(page.deleted)); + assert.equal(typeof page.must_resync, "boolean"); + cursor = page.next_cursor ?? undefined; + more = page.has_more; + } + + assert.equal(listed.length, 3); + assert.deepEqual(new Set(listed.map((m) => m.blob_id)), blobIds); + + // Cross-check against the namespace rollup. + const namespaces = []; + cursor = undefined; + more = true; + while (more) { + const page = await mw.listNamespaces({ cursor }); + namespaces.push(...page.namespaces); + cursor = page.next_cursor ?? undefined; + more = page.has_more; + } + const summary = namespaces.find((ns) => ns.name === namespace); + assert.ok(summary, `listNamespaces must include ${namespace}`); + assert.equal(summary.memory_count, 3); + }); }); diff --git a/packages/sdk/test/list-memories.test.mjs b/packages/sdk/test/list-memories.test.mjs new file mode 100644 index 000000000..c1f30a1da --- /dev/null +++ b/packages/sdk/test/list-memories.test.mjs @@ -0,0 +1,191 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { MemWal } from "../dist/memwal.js"; + +const OWNER = "0xowner0000000000000000000000000000000000000000000000000000000001"; +const originalFetch = globalThis.fetch; + +function client() { + return MemWal.create({ + key: new Uint8Array(32).fill(1), + accountId: "0x1", + serverUrl: "https://relayer.example", + }); +} + +/** + * Stub the three calls a listMemories() round-trip makes: the compatibility + * preflight, the owner resolution, and the read itself. Records every request + * so tests can assert on paths, headers and call counts. + */ +function stubRelayer(memoriesBody) { + const calls = []; + globalThis.fetch = async (url, init = {}) => { + const u = new URL(url); + calls.push({ path: u.pathname, search: u.search, method: init.method ?? "GET", headers: init.headers ?? {} }); + + if (u.pathname === "/version") { + return Response.json({ + apiVersion: "1.0.0", + relayerVersion: "1.0.0", + minSupportedSdk: { typescript: "0.0.4" }, + }); + } + if (u.pathname === "/api/stats") { + return Response.json({ memory_count: 0, storage_bytes: 0, namespace: "default", owner: OWNER }); + } + if (u.pathname === `/v1/owners/${OWNER}/memories`) { + return Response.json(memoriesBody); + } + throw new Error(`unexpected request: ${u.pathname}`); + }; + return calls; +} + +function memory(id, namespace) { + return { + memory_id: id, + namespace_id: namespace, + blob_id: `blob-${id}`, + created_at: "2026-08-20T10:00:00Z", + updated_at: "2026-08-20T10:00:00Z", + size: 512, + agent_id: null, + package_id: "0x1", + status: "active", + end_epoch: 42, + expires_at: null, + importance: null, + }; +} + +const PAGE = { + memories: [memory("m1", "work"), memory("m2", "personal"), memory("m3", "work")], + next_cursor: "cursor-1", + has_more: true, + snapshot_version: 2, + deleted: [ + { memory_id: "m0", namespace_id: "personal", deleted_at: "2026-08-20T09:00:00Z" }, + { memory_id: "m9", namespace_id: "work", deleted_at: "2026-08-20T09:00:00Z" }, + ], + must_resync: false, +}; + +test.afterEach(() => { + globalThis.fetch = originalFetch; +}); + +test("listMemories reads the owner-scoped memories path", async () => { + const calls = stubRelayer(PAGE); + + await client().listMemories(); + + const read = calls.find((c) => c.path.endsWith("/memories")); + assert.ok(read, "expected a request to the memories endpoint"); + assert.equal(read.path, `/v1/owners/${OWNER}/memories`); + assert.equal(read.method, "GET"); +}); + +test("listMemories forwards cursor as updated_after and passes limit", async () => { + const calls = stubRelayer(PAGE); + + await client().listMemories({ cursor: "cursor-0", limit: 25 }); + + const params = new URLSearchParams(calls.find((c) => c.path.endsWith("/memories")).search); + assert.equal(params.get("updated_after"), "cursor-0"); + assert.equal(params.get("limit"), "25"); +}); + +test("listMemories never sends the namespace to the relayer", async () => { + const calls = stubRelayer(PAGE); + + await client().listMemories({ namespace: "work" }); + + const params = new URLSearchParams(calls.find((c) => c.path.endsWith("/memories")).search); + assert.equal(params.get("namespace"), null); +}); + +test("listMemories without a namespace returns the relayer's wire shape unchanged", async () => { + stubRelayer(PAGE); + + const result = await client().listMemories(); + + assert.deepEqual(result, PAGE); +}); + +test("listMemories with a namespace keeps only that namespace, pagination intact", async () => { + stubRelayer(PAGE); + + const result = await client().listMemories({ namespace: "work" }); + + assert.deepEqual(result.memories.map((m) => m.memory_id), ["m1", "m3"]); + assert.deepEqual(result.deleted.map((d) => d.memory_id), ["m9"]); + assert.equal(result.next_cursor, "cursor-1"); + assert.equal(result.has_more, true); +}); + +test("listMemories with a namespace can return an empty page that still has more", async () => { + stubRelayer(PAGE); + + const result = await client().listMemories({ namespace: "nowhere" }); + + assert.deepEqual(result.memories, []); + assert.equal(result.has_more, true, "an empty filtered page is not the end of the walk"); +}); + +test("listMemories defaults tombstone fields an older relayer omits", async () => { + const { deleted: _d, must_resync: _m, ...legacy } = PAGE; + stubRelayer(legacy); + + const result = await client().listMemories({ namespace: "work" }); + + assert.deepEqual(result.deleted, []); + assert.equal(result.must_resync, false); +}); + +test("listMemories walks every page and counts a namespace exactly", async () => { + const pages = [ + { ...PAGE, memories: [memory("a", "work"), memory("b", "other")], next_cursor: "c1", has_more: true, deleted: [] }, + { ...PAGE, memories: [memory("c", "other")], next_cursor: "c2", has_more: true, deleted: [] }, + { ...PAGE, memories: [memory("d", "work")], next_cursor: "c3", has_more: false, deleted: [] }, + ]; + const cursors = []; + globalThis.fetch = async (url) => { + const u = new URL(url); + if (u.pathname === "/version") { + return Response.json({ apiVersion: "1.0.0", relayerVersion: "1.0.0", minSupportedSdk: { typescript: "0.0.4" } }); + } + if (u.pathname === "/api/stats") return Response.json({ owner: OWNER }); + cursors.push(u.searchParams.get("updated_after")); + return Response.json(pages[cursors.length - 1]); + }; + + const memwal = client(); + let cursor; + let more = true; + let count = 0; + while (more) { + const page = await memwal.listMemories({ cursor, namespace: "work" }); + count += page.memories.length; + cursor = page.next_cursor ?? undefined; + more = page.has_more; + } + + assert.equal(count, 2); + assert.deepEqual(cursors, [null, "c1", "c2"]); +}); + +test("listMemories sends no SEAL session on a metadata-only read", async () => { + const calls = stubRelayer(PAGE); + + await client().listMemories(); + + for (const call of calls.filter((c) => c.path !== "/version")) { + assert.equal( + call.headers["x-seal-session"], + undefined, + `${call.path} must not build a decrypt credential for a metadata-only read`, + ); + } +}); From 82e55495ad72ae07fc15565a17884dfd898023e1 Mon Sep 17 00:00:00 2001 From: Le Tien Phat <91601109+Niko1444@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:09:26 +0700 Subject: [PATCH 08/70] fix(relayer): report the paid-recovery state a same-key retry actually reached (WALM-670) A same-key POST on a paid job always answered 202 `uploaded`, even when it lost the recovery claim. And the claim fenced `failed` rows for 60s, so a recovery that had just failed (the wallet job, or an enqueue error that returned before the publish) left a lease behind. The next retry enqueued nothing, still said `uploaded`, and GET kept saying `failed`: the poisoned-key shape for a blob that already exists. The claim now publishes `uploaded` in the same statement. A `failed` row is claimable at once, and only an `uploaded` row keeps the 60s fence. Two racing retries therefore enqueue one recovery, and the loser reports the row's real status. If enqueue fails, the claim is released back to `failed` with the lease cleared, so the next retry takes it immediately. --- services/server/src/routes/remember.rs | 219 +++++++++++++++++++------ 1 file changed, 171 insertions(+), 48 deletions(-) diff --git a/services/server/src/routes/remember.rs b/services/server/src/routes/remember.rs index 6d78518ac..eb7eed414 100644 --- a/services/server/src/routes/remember.rs +++ b/services/server/src/routes/remember.rs @@ -990,15 +990,7 @@ async fn collapse_onto_existing_job( // than asserting "pending" on its behalf: answering with a // state we did not reach is what told callers a dead job was // queued. - let actual: String = - sqlx::query_scalar("SELECT status FROM remember_jobs WHERE id = $1") - .bind(&existing_id) - .fetch_optional(state.db.pool()) - .await - .map_err(|e| { - AppError::Internal(format!("Failed to re-read job status: {}", e)) - })? - .unwrap_or_else(|| existing_status.clone()); + let actual = read_job_status(state.db.pool(), &existing_id, &existing_status).await?; accepted(existing_id, actual) } ExistingJobAction::RedrivePending => { @@ -1017,21 +1009,26 @@ async fn collapse_onto_existing_job( accepted(existing_id, "pending".to_string()) } ExistingJobAction::RecoverPaidBlob => { - // Claim first, enqueue second, publish `uploaded` last. If the - // process dies before enqueue, the lease expires and a later - // retry reclaims it; no durable state falsely says recovered. + // The claim flips the row to `uploaded` in the same statement, so a + // concurrent same-key POST sees the recovery as taken and cannot + // enqueue a second one. If enqueue then fails, hand the row back + // (`failed`, lease cleared) so the next retry can claim it at once. if let Some(token) = claim_paid_recovery(state.db.pool(), &existing_id).await? { - enqueue_persisted_paid_recovery(state, &existing_id, &owner, &namespace).await?; - sqlx::query( - "UPDATE remember_jobs SET status = 'uploaded', error_msg = NULL, updated_at = NOW() WHERE id = $1 AND recovery_claim_token = $2 AND blob_id IS NOT NULL AND status IN ('failed', 'uploaded')", - ) - .bind(&existing_id) - .bind(&token) - .execute(state.db.pool()) - .await - .map_err(|e| AppError::Internal(format!("Failed to publish paid recovery: {}", e)))?; + if let Err(e) = + enqueue_persisted_paid_recovery(state, &existing_id, &owner, &namespace).await + { + release_paid_recovery(state.db.pool(), &existing_id, &token, &e.to_string()) + .await; + return Err(e); + } + return accepted(existing_id, "uploaded".to_string()); } - accepted(existing_id, "uploaded".to_string()) + + // No claim: another request holds a live recovery, or the row + // moved on (e.g. to `done`). Report what is actually there + // rather than asserting `uploaded` on nobody's behalf. + let actual = read_job_status(state.db.pool(), &existing_id, &existing_status).await?; + accepted(existing_id, actual) } ExistingJobAction::ReturnAsIs => { tracing::info!( @@ -1128,13 +1125,37 @@ fn paid_recovery_operation( }) } +/// Re-read a job's status after losing a claim, falling back to what the +/// caller already saw if the row is gone. +async fn read_job_status( + pool: &sqlx::PgPool, + job_id: &str, + fallback: &str, +) -> Result { + let status: Option = + sqlx::query_scalar("SELECT status FROM remember_jobs WHERE id = $1") + .bind(job_id) + .fetch_optional(pool) + .await + .map_err(|e| AppError::Internal(format!("Failed to re-read job status: {}", e)))?; + Ok(status.unwrap_or_else(|| fallback.to_string())) +} + +/// Claim recovery of a paid blob and publish it as `uploaded` in one statement. +/// +/// A `failed` row is claimable at once, whatever its lease says: the recovery +/// that held the lease is over (the wallet job marked it failed, or enqueue +/// failed), so fencing it only turns the retry the caller was told to make into +/// a no-op. An `uploaded` row is fenced for `PREPARE_CLAIM_TTL_SECS`, since a +/// recovery may still be running for it; if the process died between claim and +/// enqueue, the lease expiring is what lets a later retry take it over. async fn claim_paid_recovery( pool: &sqlx::PgPool, job_id: &str, ) -> Result, AppError> { let token = uuid::Uuid::new_v4().to_string(); let claimed: Option = sqlx::query_scalar( - "UPDATE remember_jobs SET recovery_claimed_at = NOW(), recovery_claim_token = $3, updated_at = NOW() WHERE id = $1 AND status IN ('failed', 'uploaded') AND blob_id IS NOT NULL AND (recovery_claimed_at IS NULL OR recovery_claimed_at < NOW() - make_interval(secs => $2)) RETURNING recovery_claim_token", + "UPDATE remember_jobs SET status = 'uploaded', error_msg = NULL, recovery_claimed_at = NOW(), recovery_claim_token = $3, updated_at = NOW() WHERE id = $1 AND blob_id IS NOT NULL AND (status = 'failed' OR (status = 'uploaded' AND (recovery_claimed_at IS NULL OR recovery_claimed_at < NOW() - make_interval(secs => $2)))) RETURNING recovery_claim_token", ) .bind(job_id) .bind(PREPARE_CLAIM_TTL_SECS) @@ -1145,6 +1166,27 @@ async fn claim_paid_recovery( Ok(claimed) } +/// Undo a paid-recovery claim whose enqueue failed: back to `failed`, lease +/// cleared, so the next same-key POST reclaims immediately. Only the holder of +/// `token` can release, and only while the row still says `uploaded`. +async fn release_paid_recovery(pool: &sqlx::PgPool, job_id: &str, token: &str, msg: &str) { + let released = sqlx::query( + "UPDATE remember_jobs SET status = 'failed', error_msg = $3, recovery_claimed_at = NULL, recovery_claim_token = NULL, updated_at = NOW() WHERE id = $1 AND recovery_claim_token = $2 AND status = 'uploaded'", + ) + .bind(job_id) + .bind(token) + .bind(format!("paid recovery could not be queued: {}", msg)) + .execute(pool) + .await; + if let Err(e) = released { + tracing::warn!( + "failed to release paid recovery claim for job {}: {}", + job_id, + e + ); + } +} + async fn enqueue_persisted_paid_recovery( state: &AppState, job_id: &str, @@ -1633,13 +1675,13 @@ pub async fn remember_manual( #[cfg(test)] mod tests { use super::{ - batch_summary_inputs, build_bulk_status_results, claim_remember_preparation, - existing_job_action, find_remember_job_by_key, paid_recovery_operation, - redact_hex_addresses, request_fingerprint, sanitize_job_error_for_client, - should_spawn_after_reset, split_text_chunks, summarize_for_embedding, - validate_idempotency_key, ExistingJobAction, INFRA_JOB_ERROR_MESSAGE, - INFRA_JOB_RETRYING_MESSAGE, MAX_IDEMPOTENCY_KEY_BYTES, MAX_REMEMBER_TEXT_BYTES, - SUMMARIZE_BATCH_INPUT_BYTES, SUMMARIZE_CHUNK_BYTES, + batch_summary_inputs, build_bulk_status_results, claim_paid_recovery, + claim_remember_preparation, existing_job_action, find_remember_job_by_key, + paid_recovery_operation, read_job_status, redact_hex_addresses, release_paid_recovery, + request_fingerprint, sanitize_job_error_for_client, should_spawn_after_reset, + split_text_chunks, summarize_for_embedding, validate_idempotency_key, ExistingJobAction, + INFRA_JOB_ERROR_MESSAGE, INFRA_JOB_RETRYING_MESSAGE, MAX_IDEMPOTENCY_KEY_BYTES, + MAX_REMEMBER_TEXT_BYTES, SUMMARIZE_BATCH_INPUT_BYTES, SUMMARIZE_CHUNK_BYTES, }; use crate::jobs::WalletOperation; use crate::types::AppError; @@ -1649,32 +1691,113 @@ mod tests { // ── Idempotency key (GH #477) ──────────────────────────────── - #[tokio::test] - async fn paid_recovery_publish_never_regresses_done() { - let pool = idem_test_pool().await; + async fn insert_paid_row( + pool: &sqlx::PgPool, + status: &str, + claimed_secs_ago: Option, + ) -> String { let job_id = format!("remember-job-{}", uuid::Uuid::new_v4()); sqlx::query( - "INSERT INTO remember_jobs (id, owner, namespace, status, blob_id, recovery_claim_token) VALUES ($1, '0xowner', 'ns', 'done', 'blob-1', 'claim-1')", + "INSERT INTO remember_jobs (id, owner, namespace, status, blob_id, error_msg, recovery_claimed_at, recovery_claim_token) + VALUES ($1, '0xowner', 'ns', $2, 'blob-1', 'earlier failure', + CASE WHEN $3::BIGINT IS NULL THEN NULL ELSE NOW() - make_interval(secs => $3::BIGINT) END, + CASE WHEN $3::BIGINT IS NULL THEN NULL ELSE 'old-claim' END)", ) .bind(&job_id) - .execute(&pool) + .bind(status) + .bind(claimed_secs_ago) + .execute(pool) .await .unwrap(); - let published = sqlx::query( - "UPDATE remember_jobs SET status = 'uploaded', error_msg = NULL, updated_at = NOW() WHERE id = $1 AND recovery_claim_token = $2 AND blob_id IS NOT NULL AND status IN ('failed', 'uploaded')", + job_id + } + + async fn status_of( + pool: &sqlx::PgPool, + job_id: &str, + ) -> (String, Option, Option) { + sqlx::query_as( + "SELECT status, error_msg, recovery_claim_token FROM remember_jobs WHERE id = $1", ) - .bind(&job_id) - .bind("claim-1") - .execute(&pool) + .bind(job_id) + .fetch_one(pool) .await - .unwrap(); - assert_eq!(published.rows_affected(), 0); - let status: String = sqlx::query_scalar("SELECT status FROM remember_jobs WHERE id = $1") - .bind(&job_id) - .fetch_one(&pool) - .await - .unwrap(); - assert_eq!(status, "done"); + .unwrap() + } + + #[tokio::test] + async fn paid_recovery_claim_never_regresses_done() { + let pool = idem_test_pool().await; + let job_id = insert_paid_row(&pool, "done", None).await; + assert!(claim_paid_recovery(&pool, &job_id).await.unwrap().is_none()); + assert_eq!(status_of(&pool, &job_id).await.0, "done"); + // The route reports the row as it is, not `uploaded`. + assert_eq!( + read_job_status(&pool, &job_id, "failed").await.unwrap(), + "done" + ); + } + + /// A recovery that failed (wallet job or enqueue) leaves its lease behind. + /// That lease must not fence the retry: the failed recovery is over. + #[tokio::test] + async fn a_failed_paid_job_is_reclaimable_despite_a_fresh_lease() { + let pool = idem_test_pool().await; + let job_id = insert_paid_row(&pool, "failed", Some(1)).await; + let token = claim_paid_recovery(&pool, &job_id).await.unwrap(); + assert!( + token.is_some(), + "a failed paid job must be re-claimable at once" + ); + let (status, error_msg, stored) = status_of(&pool, &job_id).await; + assert_eq!(status, "uploaded", "the claim publishes the recovery"); + assert_eq!(error_msg, None); + assert_eq!(stored, token); + } + + /// Two same-key POSTs racing on one failed paid row enqueue one recovery. + #[tokio::test] + async fn a_live_paid_recovery_is_fenced() { + let pool = idem_test_pool().await; + let job_id = insert_paid_row(&pool, "failed", None).await; + assert!(claim_paid_recovery(&pool, &job_id).await.unwrap().is_some()); + assert!( + claim_paid_recovery(&pool, &job_id).await.unwrap().is_none(), + "the second request must see the recovery as taken", + ); + assert_eq!( + read_job_status(&pool, &job_id, "failed").await.unwrap(), + "uploaded" + ); + } + + /// A claim whose process died before enqueue is taken over once the lease runs out. + #[tokio::test] + async fn an_expired_paid_recovery_lease_is_reclaimable() { + let pool = idem_test_pool().await; + let job_id = insert_paid_row(&pool, "uploaded", Some(61)).await; + assert!(claim_paid_recovery(&pool, &job_id).await.unwrap().is_some()); + } + + #[tokio::test] + async fn a_released_paid_recovery_is_failed_and_reclaimable_at_once() { + let pool = idem_test_pool().await; + let job_id = insert_paid_row(&pool, "failed", None).await; + let token = claim_paid_recovery(&pool, &job_id).await.unwrap().unwrap(); + + // A stale holder cannot release someone else's claim. + release_paid_recovery(&pool, &job_id, "not-the-token", "boom").await; + assert_eq!(status_of(&pool, &job_id).await.0, "uploaded"); + + release_paid_recovery(&pool, &job_id, &token, "queue full").await; + let (status, error_msg, stored) = status_of(&pool, &job_id).await; + assert_eq!(status, "failed"); + assert_eq!( + error_msg.as_deref(), + Some("paid recovery could not be queued: queue full") + ); + assert_eq!(stored, None); + assert!(claim_paid_recovery(&pool, &job_id).await.unwrap().is_some()); } #[tokio::test] From f8a0e05188d61e176bec01fe8ee8006bd79d3829 Mon Sep 17 00:00:00 2001 From: Le Tien Phat <91601109+Niko1444@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:09:26 +0700 Subject: [PATCH 09/70] fix(sdk): tell a timed-out remember wait to keep waiting on the job id (WALM-670) The timeout message said "retry with the same idempotency key" and that remember_and_wait / rememberAndWait does this for you. Neither is reliably true. `remember` drops its key as soon as the POST is accepted, and an explicit idempotencyKey is never stored, so following the advice minted a new key and a second paid write while the first job was still running. A wait only times out on a job that is still pending, running or uploaded (a failed job raises), so the advice is now to call waitForRememberJob / wait_for_remember_job on the job id again, and to pass an explicit key again on any retry. Python also stores the redacted server_error on the exception attribute, not just in the message. --- packages/python-sdk-memwal/memwal/client.py | 14 +++++++--- .../python-sdk-memwal/tests/test_client.py | 26 ++++++++++++++++--- packages/sdk/src/memwal.ts | 8 ++++-- .../sdk/test/remember-idempotency.test.mjs | 3 +++ 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/packages/python-sdk-memwal/memwal/client.py b/packages/python-sdk-memwal/memwal/client.py index 79a946080..1b527b533 100644 --- a/packages/python-sdk-memwal/memwal/client.py +++ b/packages/python-sdk-memwal/memwal/client.py @@ -1703,12 +1703,18 @@ def __init__( # "no status read got through (last poll: HTTP 429)" already names the rate limit. named = last_status is None and last_refusal == 429 rate_limited = "; wait hit a rate limit (429)" if saw_rate_limit and not named else "" + # Redacted once, for both the message and the attribute. + server_error = _redact_internal_urls(server_error) if server_error else None if last_status is not None: - suffix = f" ({_redact_internal_urls(server_error)})" if server_error else "" + # The wait only times out on a job still pending/running/uploaded + # (a failed job raises), so the advice is to keep waiting on it. + suffix = f" ({server_error})" if server_error else "" detail = ( - f"last status: {last_status}{suffix}. Retry with the same idempotency key " - "to keep waiting on this job; remember_and_wait does this for you " - "on the same client instance." + f"last status: {last_status}{suffix}. The job is still in progress: " + f"call wait_for_remember_job({job_id!r}) to keep waiting on it. Sending " + "the text again under a new idempotency key stores it twice; a key you " + "passed as idempotency_key is not remembered by the client, so pass it " + "again on any retry." ) else: detail = ( diff --git a/packages/python-sdk-memwal/tests/test_client.py b/packages/python-sdk-memwal/tests/test_client.py index b35e6d8cb..9f0212ec9 100644 --- a/packages/python-sdk-memwal/tests/test_client.py +++ b/packages/python-sdk-memwal/tests/test_client.py @@ -580,9 +580,29 @@ async def running(*_args: Any, **_kwargs: Any) -> dict: assert "still retrying" in (exc.value.server_error or "") assert str(exc.value).startswith("remember job timed out after 300ms") assert "last status: running" in str(exc.value) - # Python keys are per client instance (uuid4), so only a retry on the - # same client reuses the job; a new client would mint a second write. - assert "on the same client instance" in str(exc.value) + # The job is still live, so the advice is to wait on its id, not to + # POST again: `remember` drops its key once accepted, so a re-send + # would mint a new key and a second paid write. + assert "call wait_for_remember_job('slow-job')" in str(exc.value) + assert "idempotency_key is not remembered" in str(exc.value) + + async def test_timeout_attribute_carries_the_redacted_server_error( + self, memwal_client: MemWal + ) -> None: + async def running(*_args: Any, **_kwargs: Any) -> dict: + return { + "job_id": "slow-job", + "status": "running", + "error": "sidecar http://localhost:9000/upload unreachable", + } + + memwal_client._signed_request = running # type: ignore[method-assign] + with pytest.raises(MemWalRememberJobTimeout) as exc: + await memwal_client.wait_for_remember_job( + "slow-job", poll_interval_ms=0, timeout_ms=300 + ) + assert exc.value.server_error == "sidecar [internal] unreachable" + assert "localhost" not in str(exc.value) async def test_every_poll_refused_reports_unknown_state( self, memwal_client: MemWal diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index b619479ea..cad80b03c 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -242,7 +242,9 @@ function requestTimeoutError(method: string, path: string, ms: number): Error { * Under upload congestion the relayer keeps a job `running` for tens of * minutes, far past a 60s wait, and says so in the job's `error`. Carrying the * last status the relayer reported is what lets a caller tell "still retrying, - * wait on the same key" from "we never got an answer" (GH #966). + * wait on the same job" from "we never got an answer" (GH #966). The wait + * only times out on a job that is still pending/running/uploaded (a `failed` + * job throws), so the advice is always to keep waiting on the job id. */ function rememberJobTimeoutError( jobId: string, @@ -256,7 +258,9 @@ function rememberJobTimeoutError( const rateLimited = sawRateLimit && (lastSeen || lastRefusal !== 429) ? "; wait hit a rate limit (429)" : ""; const detail = lastSeen ? `last status: ${lastSeen.status}${serverError ? ` (${serverError})` : ""}. ` + - `Retry with the same idempotency key to keep waiting on this job; rememberAndWait does this for you.` + `The job is still in progress: call waitForRememberJob("${jobId}") to keep waiting on it. ` + + `Sending the text again under a new idempotency key stores it twice; a key you passed ` + + `as idempotencyKey is not remembered by the client, so pass it again on any retry.` : `no status read got through (last poll: HTTP ${lastRefusal ?? 0}), so the job's state is unknown.`; const err = new Error(`remember job timed out after ${timeoutMs}ms (job_id=${jobId})${rateLimited}; ${detail}`); err.name = "MemWalRememberJobTimeout"; diff --git a/packages/sdk/test/remember-idempotency.test.mjs b/packages/sdk/test/remember-idempotency.test.mjs index 1461ee189..a52255613 100644 --- a/packages/sdk/test/remember-idempotency.test.mjs +++ b/packages/sdk/test/remember-idempotency.test.mjs @@ -108,6 +108,9 @@ test("a timed-out wait reports the job's last status and the relayer's message", assert.match(err.serverError, /still retrying/); assert.match(err.message, /^remember job timed out after 300ms/); assert.match(err.message, /last status: running/); + // Still live: wait on the job id rather than POSTing again. + assert.match(err.message, /call waitForRememberJob\("slow-job"\)/); + assert.match(err.message, /idempotencyKey is not remembered/); return true; }, ); From 619b83d19a34923a04493ff486e5b7a945b15e3c Mon Sep 17 00:00:00 2001 From: ducnmm <165614309+ducnmm@users.noreply.github.com> Date: Mon, 12 Oct 2026 12:11:29 +0700 Subject: [PATCH 10/70] fix(sdk): reject bad mock page sizes and honor recent recall (#1167) * fix(sdk): reject bad mock page sizes and honor recent recall MemWalMock.listNamespaces treated limit 0 as an endless page and a negative limit as slice's drop-the-tail. Bulk writes skipped the client's empty-batch check, idempotency keys were ignored, and sort recent stayed oldest-first. Positional recall(query, options, namespace) also dropped the namespace on both the mock and MemWal. Fixes #1144, #1145, #1146, #1147, #1124, #1038 * test(sdk): cover the mock limit, idempotency, and recent-recall edges The first tests matched the issue repros. These also lock the branches a tie-break-only sort, a 50-row window cap, a byte-length key, and a mutated options object would have left green. --- packages/sdk/CHANGELOG.md | 5 + packages/sdk/src/memwal.ts | 7 +- packages/sdk/src/mock.ts | 92 +++++- packages/sdk/test/mock.test.mjs | 280 ++++++++++++++++++ .../test/recall-positional-namespace.test.mjs | 81 +++++ 5 files changed, 457 insertions(+), 8 deletions(-) create mode 100644 packages/sdk/test/recall-positional-namespace.test.mjs diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index 4d9234d4f..cbeab09ef 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -4,6 +4,11 @@ ### Fixed +- `MemWalMock.listNamespaces` rejects a non-positive or non-integer `limit` with `limit must be positive`, matching the relayer. `limit: 0` used to return `has_more: true` and `next_cursor: null`, so the documented pagination loop never finished, and `limit: -1` silently dropped the last namespace. (#1144, #1145) +- `MemWalMock.rememberBulkAsync` rejects an empty or non-array batch with the same error as `MemWal`. (#1146) +- `MemWalMock` honors `idempotencyKey`. A retry of the same text and namespace returns the original job, a different payload throws, and `forget` / `clear` release the key so the text can be written again. (#1147) +- `MemWalMock.recall({ sort: "recent" })` orders a bounded semantic window (5× `limit`, at least `limit`, at most 50) by write time before truncating. Relevance order is unchanged. (#1124) +- Positional `recall(query, options, namespace)` uses that namespace when `options.namespace` is omitted, on both `MemWal` and `MemWalMock`. An explicit `options.namespace` still wins. (#1038) - `rememberBulkAndWait` / `waitForRememberJobs` no longer resolve a batch they could not read as if it were still uploading. The status endpoint counts against the delegate-key budget, and a 429 on every poll used to be retried silently until the wait ran out, leaving each item as `timeout` with "polling timed out", even when nothing had been stored. When no poll got through, the wait now makes one confirming read. If that is rate-limited too, it throws `MemWalRateLimited` (`status: 429`, `jobIds`, `retryAfterSeconds`). An item still unsettled at the deadline names its last known status, or says no read got through. (WALM-671, #967) ## 0.1.9 diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index 22c7365b0..81413a5ab 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -921,7 +921,12 @@ export class MemWal { } else if (typeof limitOrOptions === "number") { options = { limit: limitOrOptions, namespace }; } else { - options = limitOrOptions; + // Positional namespace fills an options object that omitted it. + // An explicit options.namespace still wins. (#1038) + options = { ...limitOrOptions }; + if (options.namespace === undefined && namespace !== undefined) { + options.namespace = namespace; + } } } const limit = options.topK ?? options.limit ?? 10; diff --git a/packages/sdk/src/mock.ts b/packages/sdk/src/mock.ts index c9c48791b..8bab8543f 100644 --- a/packages/sdk/src/mock.ts +++ b/packages/sdk/src/mock.ts @@ -89,6 +89,25 @@ function validateText(text: string, field = "text"): void { } } +const MAX_IDEMPOTENCY_KEY_BYTES = 255; + +function validateIdempotencyKey(key: string | undefined): void { + if (key === undefined) return; + if (key.length === 0) { + throw new Error("idempotency_key cannot be empty (omit it instead)"); + } + if (new TextEncoder().encode(key).length > MAX_IDEMPOTENCY_KEY_BYTES) { + throw new Error( + `idempotency_key exceeds maximum length of ${MAX_IDEMPOTENCY_KEY_BYTES} bytes`, + ); + } +} + +/** How many semantic candidates `sort: "recent"` reorders. Mirrors `RecallSort::candidate_limit`. */ +function recentCandidateLimit(limit: number): number { + return Math.max(Math.min(limit * 5, 50), limit); +} + /** * Deterministic, dependency-free in-memory implementation of the core MemWal API. * It never opens a socket, reads credentials, or contacts Sui/Walrus. Recall uses @@ -99,6 +118,8 @@ export class MemWalMock { private readonly namespace: string; private readonly memories: MockMemory[] = []; private readonly jobs = new Map(); + /** idempotency key → job id. Released when that job's memory is forgotten. */ + private readonly idempotency = new Map(); private sequence = 0; private constructor(config: MemWalMockConfig = {}) { @@ -117,14 +138,15 @@ export class MemWalMock { destroy(): void { this.memories.length = 0; this.jobs.clear(); + this.idempotency.clear(); } async rememberAsync( text: string, namespace?: string, - _options: { idempotencyKey?: string } = {} + options: { idempotencyKey?: string } = {} ): Promise { - const memory = this.store(text, namespace); + const memory = this.store(text, namespace, undefined, options.idempotencyKey); return { job_id: memory.jobId, status: "done" }; } @@ -171,6 +193,9 @@ export class MemWalMock { async rememberBulkAsync( items: RememberBulkItem[] ): Promise { + if (!Array.isArray(items) || items.length === 0) { + throw new Error("rememberBulkAsync: items must be a non-empty array"); + } const jobIds = items.map( (item) => this.store(item.text, item.namespace).jobId ); @@ -268,7 +293,12 @@ export class MemWalMock { } else if (typeof limitOrOptions === "number") { options = { limit: limitOrOptions, namespace }; } else { - options = limitOrOptions; + // Positional namespace fills an options object that omitted it. + // An explicit options.namespace still wins. (#1038) + options = { ...limitOrOptions }; + if (options.namespace === undefined && namespace !== undefined) { + options.namespace = namespace; + } } } validateText(query, "query"); @@ -278,7 +308,7 @@ export class MemWalMock { throw new Error("limit must be a non-negative integer"); } const queryTokens = tokenize(query); - const ranked = this.memories + const semantic = this.memories .filter((memory) => memory.namespace === resolvedNamespace) .map((memory) => ({ memory, @@ -293,7 +323,20 @@ export class MemWalMock { (left, right) => left.distance - right.distance || left.memory.sequence - right.memory.sequence - ) + ); + // `recent` reorders a bounded semantic window by write time, then + // truncates. Relevance keeps the cosine order. (#1124) + const ordered = + options.sort === "recent" + ? semantic + .slice(0, recentCandidateLimit(resolvedLimit)) + .sort( + (left, right) => + right.memory.sequence - left.memory.sequence || + left.distance - right.distance, + ) + : semantic; + const ranked = ordered .slice(0, resolvedLimit) .map(({ memory, distance: memoryDistance }) => ({ blob_id: memory.blobId, @@ -386,6 +429,15 @@ export class MemWalMock { } async listNamespaces(options: ListNamespacesOptions = {}): Promise { + // The relayer rejects limit < 1 with this message. slice(0, 0) would + // otherwise report has_more with a null cursor, and slice(0, -1) + // would drop the last namespace. (#1144, #1145) + if ( + options.limit !== undefined && + (!Number.isInteger(options.limit) || options.limit < 1) + ) { + throw new Error("limit must be positive"); + } const grouped = new Map(); for (const memory of this.memories) { const entry = grouped.get(memory.namespace) ?? { count: 0, bytes: 0, sequence: 0 }; @@ -464,6 +516,7 @@ export class MemWalMock { if (index < 0) return false; const [memory] = this.memories.splice(index, 1); this.jobs.delete(memory.jobId); + this.releaseIdempotency(memory.jobId); return true; } @@ -481,18 +534,36 @@ export class MemWalMock { this.memories.splice(index, 1); } } - for (const memory of removed) this.jobs.delete(memory.jobId); + for (const memory of removed) { + this.jobs.delete(memory.jobId); + this.releaseIdempotency(memory.jobId); + } return removed.length; } private store( text: string, namespace?: string, - blobId?: string + blobId?: string, + idempotencyKey?: string, ): MockMemory { validateText(text); + validateIdempotencyKey(idempotencyKey); const resolvedNamespace = namespace ?? this.namespace; if (!resolvedNamespace) throw new Error("namespace cannot be empty"); + if (idempotencyKey !== undefined) { + const existingId = this.idempotency.get(idempotencyKey); + const existing = existingId ? this.jobs.get(existingId) : undefined; + if (existing) { + if (existing.text !== text || existing.namespace !== resolvedNamespace) { + throw new Error( + "idempotency_key was already used for a request with different content", + ); + } + return existing; + } + if (existingId) this.idempotency.delete(idempotencyKey); + } this.sequence += 1; const suffix = this.sequence.toString().padStart(6, "0"); const memory: MockMemory = { @@ -505,9 +576,16 @@ export class MemWalMock { }; this.memories.push(memory); this.jobs.set(memory.jobId, memory); + if (idempotencyKey !== undefined) this.idempotency.set(idempotencyKey, memory.jobId); return memory; } + private releaseIdempotency(jobId: string): void { + for (const [key, id] of this.idempotency) { + if (id === jobId) this.idempotency.delete(key); + } + } + private toRememberResult(memory: MockMemory): RememberResult { return { id: memory.id, diff --git a/packages/sdk/test/mock.test.mjs b/packages/sdk/test/mock.test.mjs index 45d2d928f..a0c4c63d4 100644 --- a/packages/sdk/test/mock.test.mjs +++ b/packages/sdk/test/mock.test.mjs @@ -257,3 +257,283 @@ test("MemWalMock namespace walks defer new writes until the next poll", async () assert.deepEqual(poll.namespaces.map(ns => ns.name), ["bravo"]); assert.equal(poll.namespaces[0].memory_count, 2); }); + +test("MemWalMock.listNamespaces rejects a non-positive limit instead of paging forever", async () => { + const mock = MemWalMock.create(); + await mock.remember("User likes coffee", "ns-a"); + await mock.remember("User likes tea", "ns-b"); + await mock.remember("User likes water", "ns-c"); + + await assert.rejects( + () => mock.listNamespaces({ limit: 0 }), + /limit must be positive/, + ); + await assert.rejects( + () => mock.listNamespaces({ limit: -1 }), + /limit must be positive/, + ); + await assert.rejects( + () => mock.listNamespaces({ limit: 1.5 }), + /limit must be positive/, + ); + + const page = await mock.listNamespaces({ limit: 1 }); + assert.equal(page.namespaces.length, 1); + assert.equal(page.has_more, true); + assert.ok(page.next_cursor); + + // The documented walk still finishes, and a rejected limit did not drop a namespace. + let cursor; + let more = true; + const seen = []; + let iters = 0; + while (more && iters < 10) { + const next = await mock.listNamespaces({ cursor, limit: 1 }); + if (next.has_more) assert.ok(next.next_cursor); + seen.push(...next.namespaces.map((ns) => ns.name)); + cursor = next.next_cursor ?? undefined; + more = next.has_more; + iters += 1; + } + assert.equal(more, false); + assert.equal(iters, 3); + assert.deepEqual(seen, ["ns-a", "ns-b", "ns-c"]); +}); + +test("MemWalMock.rememberBulkAsync rejects an empty or non-array batch", async () => { + const mock = MemWalMock.create(); + const message = "rememberBulkAsync: items must be a non-empty array"; + await assert.rejects(() => mock.rememberBulkAsync([]), { + name: "Error", + message, + }); + await assert.rejects(() => mock.rememberBulkAsync("nope"), { + name: "Error", + message, + }); + await assert.rejects(() => mock.rememberBulk(null), { + name: "Error", + message, + }); + await assert.rejects(() => mock.rememberBulk([]), { + name: "Error", + message, + }); +}); + +test("MemWalMock collapses a repeated idempotency key onto the original job", async () => { + const mock = MemWalMock.create(); + const first = await mock.rememberAsync("dedupe me", "ns", { + idempotencyKey: "key-123", + }); + const second = await mock.rememberAsync("dedupe me", "ns", { + idempotencyKey: "key-123", + }); + assert.equal(first.job_id, second.job_id); + assert.equal( + (await mock.recall({ query: "dedupe", namespace: "ns", limit: 10 })).total, + 1, + ); + + await assert.rejects( + () => mock.rememberAsync("other text", "ns", { idempotencyKey: "key-123" }), + /different content/, + ); + assert.equal( + (await mock.recall({ query: "other", namespace: "ns", maxDistance: 0.5 })).total, + 0, + ); + await assert.rejects( + () => mock.rememberAsync("dedupe me", "other-ns", { idempotencyKey: "key-123" }), + /different content/, + ); + assert.equal( + (await mock.recall({ query: "dedupe", namespace: "other-ns", maxDistance: 0.5 })).total, + 0, + ); + + const viaRemember = await mock.remember("via remember", "ns", { + idempotencyKey: "remember-key", + }); + const viaRememberAgain = await mock.remember("via remember", "ns", { + idempotencyKey: "remember-key", + }); + assert.equal(viaRemember.job_id, viaRememberAgain.job_id); + + const maxKey = "k".repeat(255); + const atMax = await mock.rememberAsync("boundary", "ns", { idempotencyKey: maxKey }); + const atMaxAgain = await mock.rememberAsync("boundary", "ns", { idempotencyKey: maxKey }); + assert.equal(atMax.job_id, atMaxAgain.job_id); + // "é" is two UTF-8 bytes. 128 of them are 256 bytes even though the string is shorter than 255. + await assert.rejects( + () => mock.rememberAsync("accent", "ns", { idempotencyKey: "é".repeat(128) }), + /maximum length/, + ); + await mock.rememberAsync("accent", "ns", { idempotencyKey: "é".repeat(127) }); + await assert.rejects( + () => mock.rememberAsync("dedupe me", "ns", { idempotencyKey: "" }), + /cannot be empty/, + ); + await assert.rejects( + () => + mock.rememberAsync("dedupe me", "ns", { + idempotencyKey: "k".repeat(256), + }), + /maximum length/, + ); + + const written = await mock.rememberAndWait("again", "ns", { + idempotencyKey: "key-again", + }); + assert.equal(mock.forget(written.blob_id), true); + const rewritten = await mock.rememberAndWait("again", "ns", { + idempotencyKey: "key-again", + }); + assert.notEqual(rewritten.job_id, written.job_id); + const after = await mock.recall({ + query: "again", + namespace: "ns", + maxDistance: 0.5, + }); + assert.equal(after.total, 1); + assert.equal(after.results[0].blob_id, rewritten.blob_id); + + const cleared = MemWalMock.create(); + await cleared.rememberAsync("kept", "ns", { idempotencyKey: "clear-key" }); + assert.equal(cleared.clear("ns"), 1); + const afterClear = await cleared.rememberAsync("kept", "ns", { + idempotencyKey: "clear-key", + }); + const kept = await cleared.recall({ query: "kept", namespace: "ns", maxDistance: 0.5 }); + assert.equal(kept.total, 1); + assert.equal(kept.results[0].blob_id, `mock-blob-${afterClear.job_id.slice(-6)}`); +}); + +test("MemWalMock.recall sort recent returns the newest match and keeps relevance ties", async () => { + const mock = MemWalMock.create(); + await mock.rememberAndWait("project release version one"); + await mock.rememberAndWait("project release version two"); + + const recent = await mock.recall({ + query: "project release", + sort: "recent", + limit: 1, + }); + assert.equal(recent.results[0].text, "project release version two"); + + const relevance = await mock.recall({ + query: "project release", + sort: "relevance", + limit: 1, + }); + assert.equal(relevance.results[0].text, "project release version one"); + + const omitted = await mock.recall({ query: "project release", limit: 1 }); + assert.equal(omitted.results[0].text, "project release version one"); + + // A newer, worse match inside the window wins. A distance tie-break would keep the older one. + await mock.rememberAndWait("project notes"); + const recentWorse = await mock.recall({ + query: "project release", + sort: "recent", + limit: 2, + }); + assert.deepEqual( + recentWorse.results.map((memory) => memory.text), + ["project notes", "project release version two"], + ); + const relevanceWorse = await mock.recall({ + query: "project release", + sort: "relevance", + limit: 1, + }); + assert.equal(relevanceWorse.results[0].text, "project release version one"); +}); + +test("MemWalMock.recall sort recent stays inside the semantic candidate window", async () => { + const mock = MemWalMock.create(); + for (let i = 1; i <= 6; i++) { + await mock.rememberAndWait(`alpha fact ${i}`); + } + // Equal overlap, so semantic order is insertion order. limit 1 → window 5. + const recent = await mock.recall({ query: "alpha", sort: "recent", limit: 1 }); + assert.equal(recent.results[0].text, "alpha fact 5"); + + // topK wins over limit, so the window is 5, not 10 * 5. + const byTopK = await mock.recall({ + query: "alpha", + sort: "recent", + limit: 10, + topK: 1, + }); + assert.equal(byTopK.results[0].text, "alpha fact 5"); +}); + +test("MemWalMock.recall sort recent caps the window at 50 and never below limit", async () => { + const capped = MemWalMock.create(); + for (let i = 1; i <= 51; i++) { + await capped.rememberAndWait(`alpha fact ${i}`); + } + // limit 11 → window min(55, 50) = 50. Newest of those 50 ties comes first. + const atCap = await capped.recall({ query: "alpha", sort: "recent", limit: 11 }); + assert.equal(atCap.results[0].text, "alpha fact 50"); + assert.equal(atCap.results.length, 11); + assert.equal(atCap.results.at(-1).text, "alpha fact 40"); + + const uncapped = MemWalMock.create(); + for (let i = 1; i <= 60; i++) { + await uncapped.rememberAndWait(`alpha fact ${i}`); + } + // limit 60 is above the 50 ceiling, so the window grows and the newest row stays in it. + const wide = await uncapped.recall({ query: "alpha", sort: "recent", limit: 60 }); + assert.equal(wide.results[0].text, "alpha fact 60"); + assert.equal(wide.results.length, 60); + assert.equal(wide.results.at(-1).text, "alpha fact 1"); +}); + +test("MemWalMock.recall sort recent filters namespace and still applies the token budget", async () => { + const mock = MemWalMock.create(); + await mock.rememberAndWait("alpha one", "kept"); + await mock.rememberAndWait("alpha two", "kept"); + await mock.rememberAndWait("alpha elsewhere", "other"); + + const result = await mock.recall({ + query: "alpha", + namespace: "kept", + sort: "recent", + limit: 2, + maxTokens: 3, + }); + + assert.deepEqual( + result.results.map((memory) => memory.text), + ["alpha two"], + ); + assert.equal(result.meta.truncated, true); +}); + +test("MemWalMock.recall uses a positional namespace when the options object omits one", async () => { + const mock = MemWalMock.create({ namespace: "default" }); + await mock.rememberAndWait("profile fact", "profile"); + await mock.rememberAndWait("default fact", "default"); + + const positional = await mock.recall("fact", { limit: 5 }, "profile"); + assert.deepEqual( + positional.results.map((memory) => memory.text), + ["profile fact"], + ); + + const explicit = await mock.recall( + "fact", + { limit: 5, namespace: "default" }, + "profile", + ); + assert.deepEqual( + explicit.results.map((memory) => memory.text), + ["default fact"], + ); + + const options = { limit: 5 }; + await mock.recall("fact", options, "profile"); + assert.deepEqual(options, { limit: 5 }); +}); diff --git a/packages/sdk/test/recall-positional-namespace.test.mjs b/packages/sdk/test/recall-positional-namespace.test.mjs new file mode 100644 index 000000000..31b50dfc5 --- /dev/null +++ b/packages/sdk/test/recall-positional-namespace.test.mjs @@ -0,0 +1,81 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { MemWal } from "../dist/memwal.js"; + +// GH #1038: recall(query, options, namespace) type-checks, but the options +// object used to replace the whole call and drop the namespace argument. + +const originalFetch = globalThis.fetch; + +test.afterEach(() => { + globalThis.fetch = originalFetch; +}); + +function stubRecall() { + const sent = {}; + globalThis.fetch = async (url, init = {}) => { + const path = new URL(url).pathname; + if (path === "/version") { + return Response.json({ + apiVersion: "1.0.0", + relayerVersion: "1.0.0", + minSupportedSdk: { typescript: "0.0.4" }, + }); + } + if (path === "/api/config") { + return Response.json({ packageId: "0x1", network: "testnet" }); + } + if (path === "/api/recall" && init.method === "POST") { + sent.body = JSON.parse(init.body); + return Response.json({ results: [], total: 0 }); + } + throw new Error(`unexpected request ${path}`); + }; + return sent; +} + +function client() { + const memwal = MemWal.create({ + key: new Uint8Array(32).fill(1), + accountId: "0x1", + serverUrl: "https://relayer.example", + namespace: "default", + }); + memwal.buildSealSession = async () => "test-session"; + return memwal; +} + +test("recall(query, options, namespace) sends the positional namespace", async () => { + const sent = stubRecall(); + await client().recall("q", { limit: 5 }, "profile"); + assert.equal(sent.body.namespace, "profile"); + assert.equal(sent.body.limit, 5); +}); + +test("recall(query, options, namespace) keeps an explicit options.namespace", async () => { + const sent = stubRecall(); + await client().recall("q", { limit: 5, namespace: "from-options" }, "profile"); + assert.equal(sent.body.namespace, "from-options"); +}); + +test("recall(query, limit, namespace) still sends the positional namespace", async () => { + const sent = stubRecall(); + await client().recall("q", 5, "profile"); + assert.equal(sent.body.namespace, "profile"); + assert.equal(sent.body.limit, 5); +}); + +test("recall(query, options, namespace) does not mutate the options object", async () => { + const sent = stubRecall(); + const options = { maxTokens: 200 }; + await client().recall("q", options, "profile"); + assert.deepEqual(options, { maxTokens: 200 }); + assert.equal(sent.body.namespace, "profile"); +}); + +test("recall(query, options) without a namespace keeps the client default", async () => { + const sent = stubRecall(); + await client().recall("q", { limit: 5 }); + assert.equal(sent.body.namespace, "default"); +}); From 6a7058006eff507f15e64be63d01dbb98eee4d95 Mon Sep 17 00:00:00 2001 From: ducnmm <165614309+ducnmm@users.noreply.github.com> Date: Mon, 12 Oct 2026 12:14:36 +0700 Subject: [PATCH 11/70] fix(sdk): treat a remember options object as options and fix job status polls (#1168) * fix(sdk): treat a remember options object as options and fix job status polls remember(text, { idempotencyKey }) and rememberAndWait used to send that object as the namespace. Empty job ids no longer hit GET /api/remember/. Duplicate ids in waitForRememberJobs settle in every slot. Bulk status returns namespace, status polls skip the SEAL session, batches over 20 are rejected before signing, and a destroyed client throws instead of reading a zeroed key or probing /version. #1142 #1087 #1121 #1084 #1104 #1135 #1136 #1090 * fix(sdk): reject a destroyed key still in flight and merge remember options An in-flight getPublicKeyHex resolved to the zeroed cache, and a key derived after destroy was the well-known key of the wiped seed. A third options argument was also dropped when the namespace slot was an object. --- SKILL.md | 3 +- packages/sdk/CHANGELOG.md | 7 + packages/sdk/src/memwal.ts | 188 ++++++--- packages/sdk/src/mock.ts | 62 ++- packages/sdk/src/types.ts | 2 + .../sdk/test/remember-call-shape.test.mjs | 383 ++++++++++++++++++ services/server/src/routes/remember.rs | 23 +- services/server/src/types.rs | 3 + 8 files changed, 606 insertions(+), 65 deletions(-) create mode 100644 packages/sdk/test/remember-call-shape.test.mjs diff --git a/SKILL.md b/SKILL.md index 5d628ed84..750239d0d 100644 --- a/SKILL.md +++ b/SKILL.md @@ -150,7 +150,7 @@ const stored = await memwal.waitForRememberJob(accepted.job_id, { | Method | Description | Returns | |---|---|---| -| `remember(text, namespace?)` | Accept one memory job immediately | `{ job_id, status }` | +| `remember(text, namespaceOrOptions?, options?)` | Accept one memory job immediately. A plain object in the namespace slot is options (`{ idempotencyKey }`), not a namespace | `{ job_id, status }` | | `rememberAndWait(text, namespace?, opts?)` | Store one memory and wait for completion | `{ id, job_id, blob_id, owner, namespace }` | | `recall({ query, limit?, topK?, namespace?, maxDistance? })` *(preferred)* or `recall(query, limit?, namespace?)` | Semantic search for memories | `{ results: [{ blob_id, text, distance }], total }` | | `analyze(text, namespace?)` | Extract facts and accept one memory job per fact | `{ job_ids, facts, fact_count, status, owner }` | @@ -236,6 +236,7 @@ interface AnalyzeResult { interface RememberBulkStatusItem { job_id: string; status: "pending" | "running" | "uploaded" | "done" | "failed" | "not_found"; + namespace?: string; // present when the job row exists; omitted for not_found blob_id?: string; error?: string; } diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index cbeab09ef..d4c565047 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -9,6 +9,13 @@ - `MemWalMock` honors `idempotencyKey`. A retry of the same text and namespace returns the original job, a different payload throws, and `forget` / `clear` release the key so the text can be written again. (#1147) - `MemWalMock.recall({ sort: "recent" })` orders a bounded semantic window (5× `limit`, at least `limit`, at most 50) by write time before truncating. Relevance order is unchanged. (#1124) - Positional `recall(query, options, namespace)` uses that namespace when `options.namespace` is omitted, on both `MemWal` and `MemWalMock`. An explicit `options.namespace` still wins. (#1038) +- `remember(text, { idempotencyKey })` and `rememberAndWait(text, { timeoutMs }, { idempotencyKey })` treat a plain object in the namespace slot as options. A third options argument is merged, and its fields win. That call used to send the object as the namespace, or drop the third argument and mint a second blob. An explicit string namespace still wins. (#1142) +- `getRememberStatus("")` returns `not_found` without a request, and `waitForRememberJob("")` fails immediately with status 404. `GET /api/remember/` is not a route; the empty 404 body used to throw `SyntaxError`, and the wait treated that as transient until timeout. (#1087) +- `waitForRememberJobs` settles every slot when the same job id is listed more than once. `indexOf` plus a `Set` of ids left the later copies as `timeout`. (#1121, #1084) +- `rememberBulkAsync` rejects more than 20 items before signing, matching the relayer's `MAX_BULK_ITEMS`. (#1104) +- Remember-job status reads (`getRememberStatus`, `waitForRememberJob`, `getRememberBulkStatus`) no longer attach `x-seal-session`. Status is metadata and does not decrypt. (#1135) +- Bulk status includes each job's `namespace`. `waitForRememberJobs(jobIds)` uses that value when the caller did not pass one, instead of the client default. A caller-supplied namespace still wins. (#1136) +- `getPublicKeyHex()` and any request after `destroy()` throw, including a read already in flight. `getPublicKeyHex()` used to resolve to the zeroed key, or to the well-known key of the wiped seed, and `remember()` still sent `GET /version` before noticing the client was destroyed. (#1090) - `rememberBulkAndWait` / `waitForRememberJobs` no longer resolve a batch they could not read as if it were still uploading. The status endpoint counts against the delegate-key budget, and a 429 on every poll used to be retried silently until the wait ran out, leaving each item as `timeout` with "polling timed out", even when nothing had been stored. When no poll got through, the wait now makes one confirming read. If that is rate-limited too, it throws `MemWalRateLimited` (`status: 429`, `jobIds`, `retryAfterSeconds`). An item still unsettled at the deadline names its last known status, or says no read got through. (WALM-671, #967) ## 0.1.9 diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index 81413a5ab..738d11f9a 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -157,6 +157,40 @@ const DEFAULT_REQUEST_TIMEOUT_MS = 30_000; * own rather than the 30s default a stalled socket would hold it for. */ const CONFIRM_READ_TIMEOUT_MS = 5_000; +/** Matches the relayer's `MAX_BULK_ITEMS`. Reject before a signed request. */ +const MAX_BULK_ITEMS = 20; + +interface RememberCallOptions { + idempotencyKey?: string; + pollIntervalMs?: number; + timeoutMs?: number; +} + +/** + * `remember(text, { idempotencyKey })` is the natural call when the namespace + * stays at the client default. A plain object in the namespace slot is those + * options, not a namespace name. (#1142) + */ +function splitRememberArgs( + namespaceOrOptions?: string | RememberCallOptions, + options?: RememberCallOptions, +): { namespace?: string; options: RememberCallOptions } { + // The third argument is the dedicated options bag. When the namespace + // slot is also an object, keep both and let the bag win on a clash. + // `rememberAndWait(text, { timeoutMs }, { idempotencyKey })` is typed + // and used to drop the key, so a retry minted a second blob. (#1142) + if (typeof namespaceOrOptions === "object" && namespaceOrOptions !== null) { + return { namespace: undefined, options: { ...namespaceOrOptions, ...options } }; + } + return { namespace: namespaceOrOptions, options: options ?? {} }; +} + +function destroyedClientError(): Error { + return new Error( + "Walrus Memory client was destroyed; its keys are zeroed. Create a new one.", + ); +} + /** `POST /api/restore` bounds itself at 55s server-side and answers with an * error rather than going quiet, so the client must outlast that or it would * abandon a response already on its way. */ @@ -438,13 +472,14 @@ export class MemWal { */ async rememberAsync( text: string, - namespace?: string, - options: { idempotencyKey?: string } = {}, + namespaceOrOptions?: string | { idempotencyKey?: string }, + options?: { idempotencyKey?: string }, ): Promise { - const resolvedNamespace = namespace ?? this.namespace; + const split = splitRememberArgs(namespaceOrOptions, options); + const resolvedNamespace = split.namespace ?? this.namespace; const requestIdentity = `${resolvedNamespace}\0${text}`; - const generatedKey = options.idempotencyKey === undefined; - const idempotencyKey = options.idempotencyKey + const generatedKey = split.options.idempotencyKey === undefined; + const idempotencyKey = split.options.idempotencyKey ?? this.pendingRememberKeys.get(requestIdentity) ?? (await derivedIdempotencyKey(requestIdentity)); if (generatedKey) this.pendingRememberKeys.set(requestIdentity, idempotencyKey); @@ -469,11 +504,22 @@ export class MemWal { * resolves at the terminal state and hides progress. */ async getRememberStatus(jobId: string): Promise { + // `GET /api/remember/` is not a route. The relayer answers 404 with an + // empty body, and parsing that threw SyntaxError. (#1087) + if (typeof jobId !== "string" || jobId.length === 0) { + return { + job_id: jobId, + status: "not_found", + error: `Job ${jobId} not found`, + }; + } const status = await this.signedRequest( "GET", `/api/remember/${jobId}`, {}, [200, 404], + // Status is metadata. No SEAL decrypt, so no session. (#1135) + { includeDelegateKey: false }, ); if (!("status" in status)) { @@ -494,6 +540,12 @@ export class MemWal { jobId: string, opts: { pollIntervalMs?: number; timeoutMs?: number } = {}, ): Promise { + if (typeof jobId !== "string" || jobId.length === 0) { + throw Object.assign(new Error(`remember job not found: ${jobId}`), { + status: 404, + jobId, + }); + } const { pollIntervalMs = 1500, timeoutMs = 60_000 } = opts; const deadline = Date.now() + timeoutMs; let attempt = 0; @@ -524,7 +576,7 @@ export class MemWal { // and the client deadline keeps ONE stalled poll from // swallowing the entire budget, so the loop still gets to // retry. An expired poll surfaces as a transient 504. - { timeoutMs: this.pollDeadlineMs(deadline) }, + { timeoutMs: this.pollDeadlineMs(deadline), includeDelegateKey: false }, ); } catch (err) { const httpStatus = (err as { status?: number }).status ?? 0; @@ -576,19 +628,22 @@ export class MemWal { */ async rememberAndWait( text: string, - namespace?: string, - opts: { pollIntervalMs?: number; timeoutMs?: number; idempotencyKey?: string } = {}, + namespaceOrOptions?: string | RememberCallOptions, + opts?: RememberCallOptions, ): Promise { - const resolvedNamespace = namespace ?? this.namespace; + const split = splitRememberArgs(namespaceOrOptions, opts); + const resolvedNamespace = split.namespace ?? this.namespace; const requestIdentity = `${resolvedNamespace}\0${text}`; - const generatedKey = opts.idempotencyKey === undefined; - const idempotencyKey = opts.idempotencyKey + const generatedKey = split.options.idempotencyKey === undefined; + const idempotencyKey = split.options.idempotencyKey ?? this.pendingRememberKeys.get(requestIdentity) ?? (await derivedIdempotencyKey(requestIdentity)); if (generatedKey) this.pendingRememberKeys.set(requestIdentity, idempotencyKey); + // Pass the resolved string. rememberAsync would otherwise read the + // options object in this slot as a namespace. (#1142) const accepted = await this.rememberAsync(text, resolvedNamespace, { idempotencyKey }); - const completed = await this.waitForRememberJob(accepted.job_id, opts); + const completed = await this.waitForRememberJob(accepted.job_id, split.options); // Clear only after terminal success. A polling timeout/transport failure // keeps the key so retrying the high-level operation reuses the same job. if (generatedKey) this.pendingRememberKeys.delete(requestIdentity); @@ -601,15 +656,15 @@ export class MemWal { * The relayer continues embedding, encrypting, uploading, and indexing in the background. * Use rememberAndWait() when the caller needs the final blob_id before continuing. * - * @param text - The text to remember - * @param namespace - Optional namespace override + * A plain object in the namespace position is options, so + * `remember(text, { idempotencyKey })` keeps the client namespace. (#1142) */ async remember( text: string, - namespace?: string, - options: { idempotencyKey?: string } = {}, + namespaceOrOptions?: string | { idempotencyKey?: string }, + options?: { idempotencyKey?: string }, ): Promise { - return this.rememberAsync(text, namespace, options); + return this.rememberAsync(text, namespaceOrOptions, options); } /** @@ -637,6 +692,12 @@ export class MemWal { if (!Array.isArray(items) || items.length === 0) { throw new Error("rememberBulkAsync: items must be a non-empty array"); } + // The relayer rejects this after the request is signed. Say so first. (#1104) + if (items.length > MAX_BULK_ITEMS) { + throw new Error( + `rememberBulkAsync: items exceeds maximum of ${MAX_BULK_ITEMS} per bulk request`, + ); + } const normalised = items.map((item) => ({ text: item.text, @@ -668,7 +729,8 @@ export class MemWal { "/api/remember/bulk/status", { job_ids: jobIds }, [200], - { timeoutMs: opts.timeoutMs }, + // Status is metadata. No SEAL decrypt, so no session. (#1135) + { timeoutMs: opts.timeoutMs, includeDelegateKey: false }, ); } @@ -697,29 +759,43 @@ export class MemWal { let answered = false; const settle = (jobId: string, status: RememberBulkStatusItem) => { - const idx = jobIds.indexOf(jobId); lastSeen.set(jobId, status); - if (status.status === "done") { - results[idx] = { - id: jobId, - blob_id: status.blob_id ?? "", - status: "done", - namespace: namespaces[idx] ?? this.namespace, - }; - pending.delete(jobId); - } else if (status.status === "failed" || status.status === "not_found") { - results[idx] = { - id: jobId, - blob_id: "", - status: "failed", - namespace: namespaces[idx] ?? this.namespace, - error: - status.status === "not_found" - ? "job not found" - : redactInternalUrls(status.error ?? "unknown error"), - }; - pending.delete(jobId); + // The same id can occupy more than one slot. `indexOf` updated only + // the first, and `pending` is a Set, so the copies stayed `timeout`. + // (#1121, #1084) + const terminal = + status.status === "done" || + status.status === "failed" || + status.status === "not_found"; + for (let idx = 0; idx < jobIds.length; idx++) { + if (jobIds[idx] !== jobId) continue; + if (namespaces[idx] === undefined && status.namespace) { + results[idx].namespace = status.namespace; + } + if (!terminal) continue; + // Caller namespace, then the job's own, then the client default. (#1136) + const namespace = namespaces[idx] ?? status.namespace ?? this.namespace; + if (status.status === "done") { + results[idx] = { + id: jobId, + blob_id: status.blob_id ?? "", + status: "done", + namespace, + }; + } else { + results[idx] = { + id: jobId, + blob_id: "", + status: "failed", + namespace, + error: + status.status === "not_found" + ? "job not found" + : redactInternalUrls(status.error ?? "unknown error"), + }; + } } + if (terminal) pending.delete(jobId); }; while (pending.size > 0 && Date.now() < deadline) { @@ -790,8 +866,9 @@ export class MemWal { } } - for (const jobId of pending) { - const idx = jobIds.indexOf(jobId); + for (let idx = 0; idx < jobIds.length; idx++) { + const jobId = jobIds[idx]; + if (!pending.has(jobId)) continue; const seen = lastSeen.get(jobId); results[idx].error = seen ? `still ${seen.status} after ${timeoutMs}ms` + (seen.error ? `: ${redactInternalUrls(seen.error)}` : "") @@ -1304,6 +1381,10 @@ export class MemWal { */ async getPublicKeyHex(): Promise { const pk = await this.getPublicKey(); + // `await` yields even when the key was already cached. destroy() + // zero-fills that cache in place, so encode only after a fresh check + // or this resolves to 64 zero hex chars. (#1090) + if (this.destroyed) throw destroyedClientError(); return bytesToHex(pk); } @@ -1312,11 +1393,18 @@ export class MemWal { // ============================================================ private async getPublicKey(): Promise { + // destroy() zero-fills the cache in place, and an uncached key would + // otherwise derive the well-known key of the all-zero seed. Re-check + // after every await, and never cache a key derived once wiped. (#1090) + if (this.destroyed) throw destroyedClientError(); if (!this.publicKey) { const ed = await getEd(); - this.publicKey = await ed.getPublicKeyAsync(this.privateKey); + if (this.destroyed) throw destroyedClientError(); + const derived = await ed.getPublicKeyAsync(this.privateKey); + if (this.destroyed) throw destroyedClientError(); + this.publicKey = derived; } - return this.publicKey; + return this.publicKey.slice(); } private async ensureCompatibleRelayer(): Promise { @@ -1652,6 +1740,8 @@ export class MemWal { const options = Array.isArray(acceptedStatusesOrOptions) ? requestOptions : acceptedStatusesOrOptions; + // Before the version probe. A destroyed client must not open GET /version. (#1090) + if (this.destroyed) throw destroyedClientError(); await this.ensureCompatibleRelayer(); const ed = await getEd(); @@ -1670,14 +1760,16 @@ export class MemWal { const message = `${timestamp}.${method}.${path}.${bodySha256}.${nonce}.${this.accountId}`; const msgBytes = new TextEncoder().encode(message); - if (this.destroyed) { - throw new Error( - "Walrus Memory client was destroyed; its keys are zeroed. Create a new one.", - ); - } + // Again after the version await. A destroy that lands while that + // request is in flight must still refuse to sign. (#1090) + if (this.destroyed) throw destroyedClientError(); // Sign with Ed25519 const signature = await ed.signAsync(msgBytes, this.privateKey); + // destroy() during the sign, or during public-key derivation, must + // not reach fetch with the wiped seed's well-known key. (#1090) + if (this.destroyed) throw destroyedClientError(); const publicKey = await this.getPublicKey(); + if (this.destroyed) throw destroyedClientError(); // Make HTTP request const url = `${this.serverUrl}${path}`; diff --git a/packages/sdk/src/mock.ts b/packages/sdk/src/mock.ts index 8bab8543f..5f0168f3d 100644 --- a/packages/sdk/src/mock.ts +++ b/packages/sdk/src/mock.ts @@ -23,6 +23,27 @@ import type { } from "./types.js"; import { applyTokenBudget, estimateTokens } from "./tokens.js"; +/** Matches the relayer's `MAX_BULK_ITEMS`. */ +const MAX_BULK_ITEMS = 20; + +interface RememberCallOptions { + idempotencyKey?: string; + pollIntervalMs?: number; + timeoutMs?: number; +} + +/** A plain object in the namespace slot is options, not a namespace. + * A third options argument is merged, and its fields win. (#1142) */ +function splitRememberArgs( + namespaceOrOptions?: string | RememberCallOptions, + options?: RememberCallOptions, +): { namespace?: string; options: RememberCallOptions } { + if (typeof namespaceOrOptions === "object" && namespaceOrOptions !== null) { + return { namespace: undefined, options: { ...namespaceOrOptions, ...options } }; + } + return { namespace: namespaceOrOptions, options: options ?? {} }; +} + export interface MemWalMockSeed { text: string; namespace?: string; @@ -143,19 +164,25 @@ export class MemWalMock { async rememberAsync( text: string, - namespace?: string, - options: { idempotencyKey?: string } = {} + namespaceOrOptions?: string | { idempotencyKey?: string }, + options?: { idempotencyKey?: string }, ): Promise { - const memory = this.store(text, namespace, undefined, options.idempotencyKey); + const split = splitRememberArgs(namespaceOrOptions, options); + const memory = this.store( + text, + split.namespace, + undefined, + split.options.idempotencyKey, + ); return { job_id: memory.jobId, status: "done" }; } async remember( text: string, - namespace?: string, - options: { idempotencyKey?: string } = {} + namespaceOrOptions?: string | { idempotencyKey?: string }, + options?: { idempotencyKey?: string }, ): Promise { - return this.rememberAsync(text, namespace, options); + return this.rememberAsync(text, namespaceOrOptions, options); } async getRememberStatus(jobId: string): Promise { @@ -181,13 +208,14 @@ export class MemWalMock { async rememberAndWait( text: string, - namespace?: string, - opts: { pollIntervalMs?: number; timeoutMs?: number; idempotencyKey?: string } = {} + namespaceOrOptions?: string | RememberCallOptions, + opts?: RememberCallOptions, ): Promise { - const accepted = await this.rememberAsync(text, namespace, { - idempotencyKey: opts.idempotencyKey, + const split = splitRememberArgs(namespaceOrOptions, opts); + const accepted = await this.rememberAsync(text, split.namespace, { + idempotencyKey: split.options.idempotencyKey, }); - return this.waitForRememberJob(accepted.job_id, opts); + return this.waitForRememberJob(accepted.job_id, split.options); } async rememberBulkAsync( @@ -196,6 +224,11 @@ export class MemWalMock { if (!Array.isArray(items) || items.length === 0) { throw new Error("rememberBulkAsync: items must be a non-empty array"); } + if (items.length > MAX_BULK_ITEMS) { + throw new Error( + `rememberBulkAsync: items exceeds maximum of ${MAX_BULK_ITEMS} per bulk request`, + ); + } const jobIds = items.map( (item) => this.store(item.text, item.namespace).jobId ); @@ -215,7 +248,12 @@ export class MemWalMock { results: jobIds.map((jobId) => { const memory = this.jobs.get(jobId); return memory - ? { job_id: jobId, status: "done", blob_id: memory.blobId } + ? { + job_id: jobId, + status: "done", + blob_id: memory.blobId, + namespace: memory.namespace, + } : { job_id: jobId, status: "not_found" }; }), }; diff --git a/packages/sdk/src/types.ts b/packages/sdk/src/types.ts index e253c1090..219665952 100644 --- a/packages/sdk/src/types.ts +++ b/packages/sdk/src/types.ts @@ -229,6 +229,8 @@ export interface RememberBulkAcceptedResult { export interface RememberBulkStatusItem { job_id: string; status: "pending" | "running" | "uploaded" | "done" | "failed" | "not_found"; + /** Set when the relayer found the job. Absent for `not_found`. */ + namespace?: string; blob_id?: string; error?: string; } diff --git a/packages/sdk/test/remember-call-shape.test.mjs b/packages/sdk/test/remember-call-shape.test.mjs new file mode 100644 index 000000000..7c37809d2 --- /dev/null +++ b/packages/sdk/test/remember-call-shape.test.mjs @@ -0,0 +1,383 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { MemWal, MemWalMock } from "../dist/index.js"; + +const originalFetch = globalThis.fetch; + +test.afterEach(() => { + globalThis.fetch = originalFetch; +}); + +function client() { + return MemWal.create({ + key: new Uint8Array(32).fill(1), + accountId: "0x1", + serverUrl: "https://relayer.example", + }); +} + +function versionBody() { + return Response.json({ + apiVersion: "1.0.0", + relayerVersion: "1.0.0", + minSupportedSdk: { typescript: "0.0.4" }, + }); +} + +test("remember(text, options) keeps the client namespace", async () => { + let posted; + let seal = false; + globalThis.fetch = async (url, init = {}) => { + const path = new URL(url).pathname; + if (path === "/version") return versionBody(); + if (path === "/api/remember" && init.method === "POST") { + posted = JSON.parse(init.body); + seal = init.headers["x-seal-session"] !== undefined; + return Response.json({ job_id: "job-1", status: "pending" }, { status: 202 }); + } + throw new Error(`unexpected request ${path}`); + }; + + const memwal = client(); + memwal.buildSealSession = async () => "test-session"; + const accepted = await memwal.remember("hello", { idempotencyKey: "key-1" }); + + assert.equal(accepted.job_id, "job-1"); + assert.equal(posted.namespace, "default"); + assert.equal(posted.idempotency_key, "key-1"); + assert.equal(posted.text, "hello"); + assert.equal(seal, true); +}); + +test("remember(text, namespace, options) still uses the string namespace", async () => { + let posted; + globalThis.fetch = async (url, init = {}) => { + const path = new URL(url).pathname; + if (path === "/version") return versionBody(); + if (path === "/api/remember" && init.method === "POST") { + posted = JSON.parse(init.body); + return Response.json({ job_id: "job-1", status: "pending" }, { status: 202 }); + } + throw new Error(`unexpected request ${path}`); + }; + + const memwal = client(); + memwal.buildSealSession = async () => "test-session"; + await memwal.remember("hello", "profile", { idempotencyKey: "key-1" }); + + assert.equal(posted.namespace, "profile"); + assert.equal(posted.idempotency_key, "key-1"); +}); + +test("rememberAndWait(text, options) does not send the options object as a namespace", async () => { + let posted; + globalThis.fetch = async (url, init = {}) => { + const path = new URL(url).pathname; + if (path === "/version") return versionBody(); + if (path === "/api/remember" && init.method === "POST") { + posted = JSON.parse(init.body); + return Response.json({ job_id: "job-1", status: "pending" }, { status: 202 }); + } + if (path === "/api/remember/job-1") { + return Response.json({ + job_id: "job-1", + status: "done", + blob_id: "blob-1", + owner: "0xabc", + namespace: "default", + }); + } + throw new Error(`unexpected request ${path}`); + }; + + const memwal = client(); + memwal.buildSealSession = async () => "test-session"; + const stored = await memwal.rememberAndWait("hello", { + timeoutMs: 5_000, + idempotencyKey: "key-1", + }); + + assert.equal(posted.namespace, "default"); + assert.equal(posted.idempotency_key, "key-1"); + assert.equal(stored.blob_id, "blob-1"); + assert.equal(stored.namespace, "default"); +}); + +test("rememberAndWait merges a third options bag and lets it win", async () => { + let posted; + globalThis.fetch = async (url, init = {}) => { + const path = new URL(url).pathname; + if (path === "/version") return versionBody(); + if (path === "/api/remember" && init.method === "POST") { + posted = JSON.parse(init.body); + return Response.json({ job_id: "job-1", status: "pending" }, { status: 202 }); + } + if (path === "/api/remember/job-1") { + return Response.json({ + job_id: "job-1", + status: "pending", + namespace: "default", + }); + } + throw new Error(`unexpected request ${path}`); + }; + + const memwal = client(); + memwal.buildSealSession = async () => "test-session"; + const started = Date.now(); + await assert.rejects( + memwal.rememberAndWait( + "hello", + { timeoutMs: 30, idempotencyKey: "from-slot" }, + { idempotencyKey: "stable" }, + ), + /timed out after 30ms/, + ); + + assert.ok(Date.now() - started < 5_000); + assert.equal(posted.namespace, "default"); + assert.equal(posted.idempotency_key, "stable"); +}); + +test("an empty job id never requests GET /api/remember/", async () => { + let fetches = 0; + globalThis.fetch = async () => { + fetches += 1; + throw new Error("empty job id must not be requested"); + }; + + const memwal = client(); + const status = await memwal.getRememberStatus(""); + assert.equal(status.status, "not_found"); + assert.equal(status.job_id, ""); + + const started = Date.now(); + await assert.rejects(memwal.waitForRememberJob(""), (err) => { + assert.equal(err.status, 404); + assert.match(err.message, /remember job not found/); + return true; + }); + assert.ok(Date.now() - started < 1_000); + assert.equal(fetches, 0); +}); + +test("status reads do not build a SEAL session", async () => { + const paths = []; + globalThis.fetch = async (url, init = {}) => { + const path = new URL(url).pathname; + paths.push(path); + assert.equal(init.headers?.["x-seal-session"], undefined); + if (path === "/version") return versionBody(); + if (path === "/api/remember/job-1") { + return Response.json({ + job_id: "job-1", + status: "done", + blob_id: "blob-1", + owner: "0xabc", + namespace: "profile", + }); + } + if (path === "/api/remember/bulk/status") { + return Response.json({ + results: [{ job_id: "job-1", status: "done", blob_id: "blob-1", namespace: "profile" }], + }); + } + throw new Error(`unexpected request ${path}`); + }; + + const memwal = client(); + memwal.buildSealSession = async () => { + throw new Error("status must not build a seal session"); + }; + + const one = await memwal.getRememberStatus("job-1"); + assert.equal(one.namespace, "profile"); + const bulk = await memwal.getRememberBulkStatus(["job-1"]); + assert.equal(bulk.results[0].namespace, "profile"); + const waited = await memwal.waitForRememberJob("job-1", { pollIntervalMs: 1, timeoutMs: 5_000 }); + assert.equal(waited.namespace, "profile"); + assert.deepEqual(paths.filter((path) => path !== "/version"), [ + "/api/remember/job-1", + "/api/remember/bulk/status", + "/api/remember/job-1", + ]); +}); + +test("duplicate job ids settle in every slot, using the server namespace", async () => { + globalThis.fetch = async (url, init = {}) => { + const path = new URL(url).pathname; + if (path === "/version") return versionBody(); + if (path === "/api/remember/bulk/status") { + const body = JSON.parse(init.body); + assert.deepEqual(body.job_ids, ["job-a", "job-a"]); + return Response.json({ + results: [ + { job_id: "job-a", status: "done", blob_id: "blob-a", namespace: "profile" }, + ], + }); + } + throw new Error(`unexpected request ${path}`); + }; + + const memwal = client(); + memwal.buildSealSession = async () => { + throw new Error("status must not build a seal session"); + }; + const batch = await memwal.waitForRememberJobs(["job-a", "job-a"], [], { + pollIntervalMs: 1, + timeoutMs: 5_000, + }); + + assert.equal(batch.succeeded, 2); + assert.equal(batch.failed, 0); + assert.equal(batch.results[0].status, "done"); + assert.equal(batch.results[1].status, "done"); + assert.equal(batch.results[0].namespace, "profile"); + assert.equal(batch.results[1].namespace, "profile"); + assert.equal(batch.results[0].blob_id, "blob-a"); +}); + +test("duplicate ids that stay pending time out in every slot", async () => { + globalThis.fetch = async (url) => { + const path = new URL(url).pathname; + if (path === "/version") return versionBody(); + if (path === "/api/remember/bulk/status") { + return Response.json({ + results: [{ job_id: "job-a", status: "pending", namespace: "profile" }], + }); + } + throw new Error(`unexpected request ${path}`); + }; + + const batch = await client().waitForRememberJobs(["job-a", "job-a"], [], { + pollIntervalMs: 1, + timeoutMs: 30, + }); + + assert.equal(batch.succeeded, 0); + assert.equal(batch.results[0].status, "timeout"); + assert.equal(batch.results[1].status, "timeout"); + assert.match(batch.results[0].error, /still pending after 30ms/); + assert.match(batch.results[1].error, /still pending after 30ms/); + assert.equal(batch.results[0].namespace, "profile"); + assert.equal(batch.results[1].namespace, "profile"); +}); + +test("a caller namespace wins over the one on the status row", async () => { + globalThis.fetch = async (url) => { + const path = new URL(url).pathname; + if (path === "/version") return versionBody(); + if (path === "/api/remember/bulk/status") { + return Response.json({ + results: [ + { job_id: "job-a", status: "done", blob_id: "blob-a", namespace: "profile" }, + { job_id: "job-b", status: "not_found" }, + ], + }); + } + throw new Error(`unexpected request ${path}`); + }; + + const memwal = client(); + const batch = await memwal.waitForRememberJobs(["job-a", "job-b"], ["caller"], { + pollIntervalMs: 1, + timeoutMs: 5_000, + }); + + assert.equal(batch.results[0].namespace, "caller"); + assert.equal(batch.results[0].status, "done"); + assert.equal(batch.results[1].status, "failed"); + assert.equal(batch.results[1].error, "job not found"); + assert.equal(batch.results[1].namespace, "default"); +}); + +test("rememberBulkAsync rejects more than 20 items before signing", async () => { + let fetches = 0; + globalThis.fetch = async () => { + fetches += 1; + throw new Error("over-limit bulk must not be requested"); + }; + + const items = Array.from({ length: 21 }, (_, i) => ({ text: `fact ${i}` })); + await assert.rejects(client().rememberBulkAsync(items), /maximum of 20/); + assert.equal(fetches, 0); +}); + +test("destroy() rejects key reads and requests without calling the relayer", async () => { + let fetches = 0; + globalThis.fetch = async () => { + fetches += 1; + throw new Error("a destroyed client must not call the relayer"); + }; + + const cached = client(); + const before = await cached.getPublicKeyHex(); + assert.match(before, /^[0-9a-f]{64}$/); + cached.destroy(); + await assert.rejects(cached.getPublicKeyHex(), /destroyed/); + await assert.rejects(cached.remember("hi"), /destroyed/); + + const fresh = client(); + fresh.destroy(); + await assert.rejects(fresh.getPublicKeyHex(), /destroyed/); + await assert.rejects(fresh.remember("hi"), /destroyed/); + assert.equal(fetches, 0); +}); + +test("destroy() during an in-flight getPublicKeyHex rejects", async () => { + const cached = client(); + const before = await cached.getPublicKeyHex(); + assert.match(before, /^[0-9a-f]{64}$/); + assert.notEqual(before, "0".repeat(64)); + + const pendingCached = cached.getPublicKeyHex(); + cached.destroy(); + await assert.rejects(pendingCached, /destroyed/); + + const fresh = client(); + const pendingFresh = fresh.getPublicKeyHex(); + fresh.destroy(); + await assert.rejects(pendingFresh, /destroyed/); +}); + +test("destroy() during an in-flight remember does not send the remember", async () => { + let releaseVersion; + const versionGate = new Promise((resolve) => { + releaseVersion = resolve; + }); + const requested = []; + globalThis.fetch = async (url) => { + const path = new URL(url).pathname; + requested.push(path); + if (path === "/version") { + await versionGate; + return versionBody(); + } + throw new Error(`a wiped client still sent ${path}`); + }; + + const memwal = client(); + memwal.buildSealSession = async () => "test-session"; + const pending = memwal.remember("hi"); + await new Promise((resolve) => setImmediate(resolve)); + memwal.destroy(); + releaseVersion(); + + await assert.rejects(pending, /destroyed/); + assert.deepEqual(requested, ["/version"]); +}); + +test("MemWalMock.remember(text, options) stores under the client namespace", async () => { + const mock = MemWalMock.create(); + const accepted = await mock.remember("hello", { idempotencyKey: "k" }); + const status = await mock.getRememberStatus(accepted.job_id); + assert.equal(status.namespace, "default"); + assert.equal(typeof status.namespace, "string"); + + const stored = await mock.rememberAndWait("hello again", { timeoutMs: 10, idempotencyKey: "k2" }); + assert.equal(stored.namespace, "default"); + + const items = Array.from({ length: 21 }, (_, i) => ({ text: `fact ${i}` })); + await assert.rejects(mock.rememberBulkAsync(items), /maximum of 20/); +}); diff --git a/services/server/src/routes/remember.rs b/services/server/src/routes/remember.rs index 874a60ce7..bd2b4af57 100644 --- a/services/server/src/routes/remember.rs +++ b/services/server/src/routes/remember.rs @@ -1446,14 +1446,21 @@ pub async fn remember_bulk( )) } -type BulkStatusRow = (String, String, String, Option, Option); +type BulkStatusRow = ( + String, + String, + String, + Option, + Option, + String, +); fn build_bulk_status_results( job_ids: Vec, rows: Vec, ) -> Vec { let mut by_id = std::collections::HashMap::with_capacity(rows.len()); - for (id, _owner_db, status, blob_id, error_msg) in rows { + for (id, _owner_db, status, blob_id, error_msg, namespace) in rows { by_id.insert( id.clone(), RememberBulkStatusItem { @@ -1461,6 +1468,7 @@ fn build_bulk_status_results( error: sanitize_job_error_for_client(&status, error_msg), status, blob_id, + namespace: Some(namespace), }, ); } @@ -1475,6 +1483,7 @@ fn build_bulk_status_results( status: "not_found".to_string(), blob_id: None, error: None, + namespace: None, }); results.push(item); } @@ -1484,7 +1493,7 @@ fn build_bulk_status_results( /// POST /api/remember/bulk/status — poll multiple job statuses at once /// -/// Returns `{ results: [{ job_id, status, blob_id?, error? }] }` preserving the +/// Returns `{ results: [{ job_id, status, namespace?, blob_id?, error? }] }` preserving the /// same order as `job_ids[]` in the request. All jobs must belong to the /// authenticated owner. pub async fn remember_bulk_status( @@ -1504,7 +1513,7 @@ pub async fn remember_bulk_status( let rows: Vec = sqlx::query_as( - "SELECT id, owner, status, blob_id, error_msg FROM remember_jobs WHERE id = ANY($1) AND owner = $2", + "SELECT id, owner, status, blob_id, error_msg, namespace FROM remember_jobs WHERE id = ANY($1) AND owner = $2", ) .bind(&body.job_ids) .bind(&auth.owner) @@ -2266,6 +2275,7 @@ mod tests { "done".to_string(), Some("blob-1".to_string()), None, + "profile".to_string(), ), ( "job-2".to_string(), @@ -2273,6 +2283,7 @@ mod tests { "failed".to_string(), None, Some("boom".to_string()), + "notes".to_string(), ), ], ); @@ -2281,13 +2292,17 @@ mod tests { assert_eq!(results[0].job_id, "job-2"); assert_eq!(results[0].status, "failed"); assert_eq!(results[0].error.as_deref(), Some("boom")); + assert_eq!(results[0].namespace.as_deref(), Some("notes")); assert_eq!(results[1].job_id, "missing"); assert_eq!(results[1].status, "not_found"); + assert_eq!(results[1].namespace, None); assert_eq!(results[2].job_id, "job-1"); assert_eq!(results[2].status, "done"); assert_eq!(results[2].blob_id.as_deref(), Some("blob-1")); + assert_eq!(results[2].namespace.as_deref(), Some("profile")); assert_eq!(results[3].job_id, "job-2"); assert_eq!(results[3].status, "failed"); + assert_eq!(results[3].namespace.as_deref(), Some("notes")); } // ── Text size limit ────────────────────────────────────────── diff --git a/services/server/src/types.rs b/services/server/src/types.rs index 52b5a9ee0..9a93ed671 100644 --- a/services/server/src/types.rs +++ b/services/server/src/types.rs @@ -1874,6 +1874,9 @@ pub struct RememberBulkStatusRequest { pub struct RememberBulkStatusItem { pub job_id: String, pub status: String, + /// Set when the job row exists. Omitted for `not_found`, which has no row. + #[serde(skip_serializing_if = "Option::is_none")] + pub namespace: Option, #[serde(skip_serializing_if = "Option::is_none")] pub blob_id: Option, #[serde(skip_serializing_if = "Option::is_none")] From 6626a7108292fe995548138fb0e9eaba05494889 Mon Sep 17 00:00:00 2001 From: ducnmm <165614309+ducnmm@users.noreply.github.com> Date: Mon, 12 Oct 2026 12:15:50 +0700 Subject: [PATCH 12/70] fix: reject blank text, treat [::1] as local, and pin weekday dates (#1134) * fix: reject blank text, treat [::1] as local, and pin weekday dates Whitespace-only remember, analyze, and embed text is rejected before a job is created (#1129). The TypeScript client treats bracketed ::1 as loopback (#1128). Fact extraction copies a server-built weekday calendar (#1131). AUTH_REJECTED names a pasted wallet address without changing the 401 (#1132). * refactor: one weekday table, inline the loopback host check The English weekday names lived in a match and again in the calendar loop. One Sunday-first array covers both, indexed by num_days_from_sunday. The IPv6 bracket strip stays inside normalizeServerUrl. --- SKILL.md | 2 +- docs/troubleshooting/overview.md | 1 + packages/python-sdk-memwal/CHANGELOG.md | 2 + packages/python-sdk-memwal/memwal/client.py | 20 ++- .../tests/test_auth_rejected_message.py | 1 + .../tests/test_blank_text.py | 22 +++ packages/sdk/CHANGELOG.md | 3 + packages/sdk/src/manual.ts | 3 +- packages/sdk/src/memwal.ts | 7 + packages/sdk/src/utils.ts | 21 ++- .../sdk/test/normalize-server-url.test.mjs | 46 +++++++ .../sdk/test/sanitize-server-error.test.mjs | 2 +- services/server/src/routes/admin.rs | 2 +- services/server/src/routes/analyze.rs | 2 +- services/server/src/routes/remember.rs | 4 +- services/server/src/services/extractor.rs | 128 ++++++++++++++---- .../server/src/services/prompts/extract.txt | 16 ++- services/server/src/types.rs | 20 +++ 18 files changed, 262 insertions(+), 40 deletions(-) create mode 100644 packages/python-sdk-memwal/tests/test_blank_text.py create mode 100644 packages/sdk/test/normalize-server-url.test.mjs diff --git a/SKILL.md b/SKILL.md index 750239d0d..501778720 100644 --- a/SKILL.md +++ b/SKILL.md @@ -575,7 +575,7 @@ Lifecycle hooks run automatically: | `health()` returns error | Check relayer URL is correct and reachable | | `recall()` returns empty | Verify namespace matches what was used in `remember()` | | `recall()` returns unrelated filler | Recall is top-K without a default relevance threshold; filter by `distance`, for example `distance < 0.8`, and calibrate the cutoff against your data | -| `401 Unauthorized` | Usually wrong `MEMWAL_PRIVATE_KEY`, key not registered on the account, account ID mismatch, or staging/mainnet mismatch. Check `.env.local` and dashboard credentials | +| `401 Unauthorized` | Usually wrong `MEMWAL_PRIVATE_KEY`, key not registered on the account, account ID mismatch (a wallet address is not a MemWal account ID), or staging/mainnet mismatch. Check `.env.local` and dashboard credentials | | SDK import errors | Run `pnpm add @mysten-incubation/memwal` — check Node.js ≥ 18 | | Manual client errors | Install peer deps: `@mysten/sui @mysten/seal @mysten/walrus` | | Direct Sui reads fail or examples look stale | Prefer `SuiGrpcClient` from `@mysten/sui/grpc`; JSON-RPC snippets using `SuiClient` / `getFullnodeUrl` may be stale | diff --git a/docs/troubleshooting/overview.md b/docs/troubleshooting/overview.md index cbd0be152..314a349aa 100644 --- a/docs/troubleshooting/overview.md +++ b/docs/troubleshooting/overview.md @@ -44,6 +44,7 @@ This section covers credential and authorization errors. | --- | --- | --- | | The delegate key is not registered onchain yet, or its registration transaction is not indexed | Confirm the key appears under your account in the dashboard, and that any `addDelegateKey` call succeeded. | Register the public key onchain through the dashboard or `addDelegateKey`, then wait a few seconds for indexing and retry. | | The account ID does not match the account that owns the key | Compare the `accountId` in your configuration against the account that lists this delegate key in the dashboard. A stale value in `localStorage` is a common source. | Use the account ID that you generated and that owns this delegate key, and do not reuse an account ID copied from another project. | +| `accountId` is a Sui wallet address | A wallet address and a MemWal account ID are both `0x` plus 64 hex characters. The relayer returns this same 401 when the id is not a MemWal account, so the error text names the case instead of distinguishing it from a bad key. | Copy the account ID from the Walrus Memory dashboard. Do not paste the connected wallet address. | | The client points at a different network than the one where the account lives | Confirm your `serverUrl` or environment preset matches where you created the account. Production is `https://relayer.memory.walrus.xyz` and staging is `https://relayer-staging.memory.walrus.xyz`. | Point the client at the relayer for the correct network, or recreate the account and key in the environment you intend to use. | | The system clock is outside the 5-minute signing window | Confirm the machine clock is accurate and synchronized using the Network Time Protocol (NTP). | Synchronize the clock and retry. | | The delegate key is revoked, or the account is deactivated | Confirm the key is present and the account is active in the dashboard. | Re-add the delegate key, or reactivate the account, which only the owner can do. | diff --git a/packages/python-sdk-memwal/CHANGELOG.md b/packages/python-sdk-memwal/CHANGELOG.md index 333e550a3..14c0d91e0 100644 --- a/packages/python-sdk-memwal/CHANGELOG.md +++ b/packages/python-sdk-memwal/CHANGELOG.md @@ -4,6 +4,8 @@ ### Fixed +- `remember`, `remember_bulk`, `analyze`, and `embed` reject whitespace-only text before the request. A blank body used to be accepted and then sit until the caller timed out. (#1129) +- A 401 `AUTH_REJECTED` now says a wallet address is not a MemWal account ID. (#1132) - `wait_for_remember_jobs` / `remember_bulk_and_wait` no longer return a batch they could not read as if it were still uploading. When every status poll is rate-limited, the wait makes one confirming read and raises `MemWalRateLimited` (`status` 429, `job_ids`, `retry_after`) if that is refused too. An item still unsettled at the deadline names its last known status, or says no read got through. (WALM-671, #967) ## 0.1.12 diff --git a/packages/python-sdk-memwal/memwal/client.py b/packages/python-sdk-memwal/memwal/client.py index b5d0e09d0..2c7836902 100644 --- a/packages/python-sdk-memwal/memwal/client.py +++ b/packages/python-sdk-memwal/memwal/client.py @@ -96,7 +96,8 @@ SEAL_SESSION_SAFETY_MARGIN_MS = 30_000 AUTH_REJECTED_MESSAGE = ( "401 from relayer: typically wrong private key, key not registered on this " - "account, account ID mismatch, or staging/mainnet mismatch. Check .env.local " + "account, account ID mismatch (a wallet address is not a MemWal account ID), " + "or staging/mainnet mismatch. Check .env.local " "and dashboard credentials. Full troubleshooting: " "https://docs.wal.app/walrus-memory/troubleshooting/overview#401-auth_rejected-errors" ) @@ -121,6 +122,18 @@ def _server_url_for_log(parsed: ParseResult) -> str: return f"{parsed.scheme}://{host}" +def _reject_blank_text(text: object, message: str = "Text cannot be empty") -> None: + """Reject empty and whitespace-only text before a request is sent. + + The relayer does the same (#1129). Failing here means an older relayer + cannot accept the job and leave the caller polling until timeout. + Non-blank text is not trimmed. + """ + + if not isinstance(text, str) or not text.strip(): + raise ValueError(message) + + def normalize_server_url(url: str) -> str: """Strip a trailing slash and warn on plaintext HTTP to a remote host. @@ -414,6 +427,7 @@ async def remember( :class:`RememberAcceptedResult` with ``job_id`` and initial status (``"pending"``). """ + _reject_blank_text(text) resolved_namespace = namespace or self._namespace request_identity = f"{resolved_namespace}\0{text}" generated_key = idempotency_key is None @@ -569,6 +583,8 @@ async def remember_bulk_async( if not items: raise ValueError("remember_bulk_async: items must be a non-empty array") + for index, item in enumerate(items): + _reject_blank_text(item.text, f"items[{index}].text cannot be empty") payload_items: List[Dict[str, Any]] = [ { @@ -898,6 +914,7 @@ async def analyze( :class:`AnalyzeResult` with extracted ``facts`` + per-fact ``job_ids`` for downstream polling. """ + _reject_blank_text(text) body: Dict[str, Any] = { "text": text, "namespace": namespace or self._namespace, @@ -970,6 +987,7 @@ async def embed(self, text: str) -> EmbedResult: :meth:`remember` (server handles embed + encrypt + upload). """ + _reject_blank_text(text, "text cannot be empty") data = await self._signed_request( "POST", "/api/embed", diff --git a/packages/python-sdk-memwal/tests/test_auth_rejected_message.py b/packages/python-sdk-memwal/tests/test_auth_rejected_message.py index d514a092d..87588f443 100644 --- a/packages/python-sdk-memwal/tests/test_auth_rejected_message.py +++ b/packages/python-sdk-memwal/tests/test_auth_rejected_message.py @@ -12,6 +12,7 @@ def test_auth_rejected_message_points_to_troubleshooting_guide() -> None: assert "docs.wal.app/walrus-memory/troubleshooting/overview" in AUTH_REJECTED_MESSAGE + assert "wallet address" in AUTH_REJECTED_MESSAGE def test_auth_503_is_retryable_not_a_credential_failure() -> None: diff --git a/packages/python-sdk-memwal/tests/test_blank_text.py b/packages/python-sdk-memwal/tests/test_blank_text.py new file mode 100644 index 000000000..e0e175023 --- /dev/null +++ b/packages/python-sdk-memwal/tests/test_blank_text.py @@ -0,0 +1,22 @@ +"""Whitespace-only text is rejected before a request is sent (#1129).""" + +from __future__ import annotations + +import pytest + +from memwal.client import _reject_blank_text + + +@pytest.mark.parametrize("text", ["", " ", "\t", "\n", " \t\n "]) +def test_blank_text_rejected(text: str) -> None: + with pytest.raises(ValueError, match="Text cannot be empty"): + _reject_blank_text(text) + + +def test_padded_text_is_not_trimmed() -> None: + _reject_blank_text(" hello ") + + +def test_bulk_item_message_names_the_index() -> None: + with pytest.raises(ValueError, match=r"items\[0\].text cannot be empty"): + _reject_blank_text(" \t", "items[0].text cannot be empty") diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index d4c565047..5ec940b26 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -16,6 +16,9 @@ - Remember-job status reads (`getRememberStatus`, `waitForRememberJob`, `getRememberBulkStatus`) no longer attach `x-seal-session`. Status is metadata and does not decrypt. (#1135) - Bulk status includes each job's `namespace`. `waitForRememberJobs(jobIds)` uses that value when the caller did not pass one, instead of the client default. A caller-supplied namespace still wins. (#1136) - `getPublicKeyHex()` and any request after `destroy()` throw, including a read already in flight. `getPublicKeyHex()` used to resolve to the zeroed key, or to the well-known key of the wiped seed, and `remember()` still sent `GET /version` before noticing the client was destroyed. (#1090) +- `normalizeServerUrl` no longer warns about plaintext HTTP for IPv6 loopback. `URL.hostname` keeps the brackets on `[::1]`, so the `::1` exemption never matched `http://[::1]:8000`. (#1128) +- `remember`, `rememberBulk`, `analyze`, and `embed` reject whitespace-only text immediately, with the same messages the relayer returns. A blank body used to be accepted and then sit until the client timed out. (#1129) +- A 401 `AUTH_REJECTED` now says a wallet address is not a MemWal account ID. The relayer still returns one 401 for every credential failure, so the message names the mix-up instead of revealing which check failed. (#1132) - `rememberBulkAndWait` / `waitForRememberJobs` no longer resolve a batch they could not read as if it were still uploading. The status endpoint counts against the delegate-key budget, and a 429 on every poll used to be retried silently until the wait ran out, leaving each item as `timeout` with "polling timed out", even when nothing had been stored. When no poll got through, the wait now makes one confirming read. If that is rate-limited too, it throws `MemWalRateLimited` (`status: 429`, `jobIds`, `retryAfterSeconds`). An item still unsettled at the deadline names its last known status, or says no read got through. (WALM-671, #967) ## 0.1.9 diff --git a/packages/sdk/src/manual.ts b/packages/sdk/src/manual.ts index deb77d41c..72257f17b 100644 --- a/packages/sdk/src/manual.ts +++ b/packages/sdk/src/manual.ts @@ -46,6 +46,7 @@ import { normalizePrivateKey, u64ToLeHex, normalizeServerUrl, + assertNonBlankText, sanitizeServerError, clockDriftErrorFromResponse, scoringWeightsToWire, @@ -351,7 +352,7 @@ export class MemWalManual { * 3. Send {encrypted_data, vector} to server — server handles Walrus upload relay */ async rememberManual(text: string, namespace?: string): Promise { - if (!text) throw new Error("Text cannot be empty"); + assertNonBlankText(text); const ns = namespace ?? this.namespace; diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index 738d11f9a..1a5a5b6f7 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -64,6 +64,7 @@ import { bytesToHex, normalizePrivateKey, normalizeServerUrl, + assertNonBlankText, sanitizeServerError, redactInternalUrls, clockDriftErrorFromResponse, @@ -475,6 +476,7 @@ export class MemWal { namespaceOrOptions?: string | { idempotencyKey?: string }, options?: { idempotencyKey?: string }, ): Promise { + assertNonBlankText(text); const split = splitRememberArgs(namespaceOrOptions, options); const resolvedNamespace = split.namespace ?? this.namespace; const requestIdentity = `${resolvedNamespace}\0${text}`; @@ -698,6 +700,9 @@ export class MemWal { `rememberBulkAsync: items exceeds maximum of ${MAX_BULK_ITEMS} per bulk request`, ); } + items.forEach((item, i) => { + assertNonBlankText(item?.text, `items[${i}].text cannot be empty`); + }); const normalised = items.map((item) => ({ text: item.text, @@ -1151,6 +1156,7 @@ export class MemWal { * @returns EmbedResult with vector */ async embed(text: string): Promise { + assertNonBlankText(text, "text cannot be empty"); return this.signedRequest( "POST", "/api/embed", @@ -1178,6 +1184,7 @@ export class MemWal { text: string, namespaceOrOptions?: string | AnalyzeOptions, ): Promise { + assertNonBlankText(text); const options = normalizeAnalyzeOptions(namespaceOrOptions); const body: Record = { text, diff --git a/packages/sdk/src/utils.ts b/packages/sdk/src/utils.ts index cbeb3cbea..6634f1cc6 100644 --- a/packages/sdk/src/utils.ts +++ b/packages/sdk/src/utils.ts @@ -261,13 +261,16 @@ export function normalizePrivateKey(key: string): string { * non-localhost host (plaintext HTTP on the open internet exposes * signed requests and any server-side secrets to passive interception). * - Localhost / 127.0.0.1 / ::1 are exempt from the warning (common in dev). + * `URL.hostname` keeps the brackets on an IPv6 address (`[::1]`), so the + * comparison strips them. Otherwise `http://[::1]` warns as if it were remote. * - Does NOT throw — explicit user-supplied `http://` is honored. */ export function normalizeServerUrl(url: string): string { const trimmed = url.replace(/\/$/, ""); try { const parsed = new URL(trimmed); - const host = parsed.hostname.toLowerCase(); + let host = parsed.hostname.toLowerCase(); + if (host.startsWith("[") && host.endsWith("]")) host = host.slice(1, -1); const isLocal = host === "localhost" || host === "127.0.0.1" || @@ -287,6 +290,20 @@ export function normalizeServerUrl(url: string): string { return trimmed; } +/** + * Reject empty and whitespace-only text before a request is sent. + * + * The relayer does the same. A blank body used to be accepted and then + * sit until the client timed out (#1129). Non-blank text is not trimmed. + */ +export function assertNonBlankText(text: string, message = "Text cannot be empty"): void { + if (typeof text !== "string" || text.trim().length === 0) { + const err = new Error(message) as Error & { status?: number }; + err.status = 400; + throw err; + } +} + // ============================================================ // Error Sanitization (LOW-26) // ============================================================ @@ -317,7 +334,7 @@ export function sanitizeServerError( return { message: "401 from relayer: typically wrong private key, key not registered on this account, " + - "account ID mismatch, or staging/mainnet mismatch. Check .env.local and dashboard credentials. " + + "account ID mismatch (a wallet address is not a MemWal account ID), or staging/mainnet mismatch. Check .env.local and dashboard credentials. " + "Full troubleshooting: https://docs.wal.app/walrus-memory/troubleshooting/overview#401-auth_rejected-errors", raw: rawBody, serverCode: "AUTH_REJECTED", diff --git a/packages/sdk/test/normalize-server-url.test.mjs b/packages/sdk/test/normalize-server-url.test.mjs new file mode 100644 index 000000000..6f0f9de16 --- /dev/null +++ b/packages/sdk/test/normalize-server-url.test.mjs @@ -0,0 +1,46 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { assertNonBlankText, normalizeServerUrl } from "../dist/utils.js"; + +function warningsFrom(run) { + const warnings = []; + const original = console.warn; + console.warn = (...args) => warnings.push(args.join(" ")); + try { + const value = run(); + return { value, warnings }; + } finally { + console.warn = original; + } +} + +test("ipv6 loopback does not warn about plaintext HTTP", () => { + for (const url of ["http://[::1]:8000", "http://[::1]", "http://[0:0:0:0:0:0:0:1]:8000"]) { + const { value, warnings } = warningsFrom(() => normalizeServerUrl(url)); + assert.equal(value, url.replace(/\/$/, "")); + assert.deepEqual(warnings, [], url); + } +}); + +test("localhost and 127.0.0.1 stay exempt, a remote http host warns", () => { + for (const url of ["http://localhost:8000", "http://127.0.0.1:8000", "http://foo.localhost"]) { + const { warnings } = warningsFrom(() => normalizeServerUrl(url)); + assert.deepEqual(warnings, [], url); + } + const { value, warnings } = warningsFrom(() => normalizeServerUrl("http://relayer.example.com/")); + assert.equal(value, "http://relayer.example.com"); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /plaintext HTTP/); +}); + +test("whitespace-only text is rejected and surrounding spaces on a real fact are kept", () => { + for (const text of ["", " ", "\t", "\n", " \t\n "]) { + assert.throws(() => assertNonBlankText(text), { message: "Text cannot be empty", status: 400 }); + } + assert.doesNotThrow(() => assertNonBlankText(" hello ")); + assert.throws( + () => assertNonBlankText(" ", "items[0].text cannot be empty"), + { message: "items[0].text cannot be empty", status: 400 }, + ); +}); diff --git a/packages/sdk/test/sanitize-server-error.test.mjs b/packages/sdk/test/sanitize-server-error.test.mjs index 341d058f3..612a03813 100644 --- a/packages/sdk/test/sanitize-server-error.test.mjs +++ b/packages/sdk/test/sanitize-server-error.test.mjs @@ -5,7 +5,7 @@ import { sanitizeServerError } from "../dist/utils.js"; const AUTH_REJECTED = "401 from relayer: typically wrong private key, key not registered on this account, " + - "account ID mismatch, or staging/mainnet mismatch. Check .env.local and dashboard credentials. " + + "account ID mismatch (a wallet address is not a MemWal account ID), or staging/mainnet mismatch. Check .env.local and dashboard credentials. " + "Full troubleshooting: https://docs.wal.app/walrus-memory/troubleshooting/overview#401-auth_rejected-errors"; test("empty-body 401 uses AUTH_REJECTED troubleshooting instead of memwal_login", () => { diff --git a/services/server/src/routes/admin.rs b/services/server/src/routes/admin.rs index 9a708f04a..b98b7bcb6 100644 --- a/services/server/src/routes/admin.rs +++ b/services/server/src/routes/admin.rs @@ -468,7 +468,7 @@ pub async fn embed( Extension(_auth): Extension, Json(body): Json, ) -> Result, AppError> { - if body.text.is_empty() { + if text_is_blank(&body.text) { return Err(AppError::BadRequest("text cannot be empty".into())); } if body.text.len() > MAX_EMBED_TEXT_BYTES { diff --git a/services/server/src/routes/analyze.rs b/services/server/src/routes/analyze.rs index 1556ce1e9..10fe4aef3 100644 --- a/services/server/src/routes/analyze.rs +++ b/services/server/src/routes/analyze.rs @@ -157,7 +157,7 @@ pub async fn analyze( Json(body): Json, ) -> Result<(StatusCode, Json), AppError> { reject_if_writes_paused(state.config.writes_paused)?; - if body.text.is_empty() { + if text_is_blank(&body.text) { return Err(AppError::BadRequest("Text cannot be empty".into())); } // Reject oversize plaintext before spending an LLM call. diff --git a/services/server/src/routes/remember.rs b/services/server/src/routes/remember.rs index bd2b4af57..f04ef7b6e 100644 --- a/services/server/src/routes/remember.rs +++ b/services/server/src/routes/remember.rs @@ -758,7 +758,7 @@ pub async fn remember( Json(body): Json, ) -> Result<(StatusCode, Json), AppError> { reject_if_writes_paused(state.config.writes_paused)?; - if body.text.is_empty() { + if text_is_blank(&body.text) { return Err(AppError::BadRequest("Text cannot be empty".into())); } // Reject oversize plaintext before spending embed + encrypt compute. @@ -1315,7 +1315,7 @@ pub async fn remember_bulk( ))); } for (i, item) in body.items.iter().enumerate() { - if item.text.is_empty() { + if text_is_blank(&item.text) { return Err(AppError::BadRequest(format!( "items[{}].text cannot be empty", i diff --git a/services/server/src/services/extractor.rs b/services/server/src/services/extractor.rs index a0783fbe1..f9ec3e6eb 100644 --- a/services/server/src/services/extractor.rs +++ b/services/server/src/services/extractor.rs @@ -200,7 +200,13 @@ const FACT_EXTRACTION_PROMPT: &str = include_str!("prompts/extract.txt"); /// stays cacheable (same pattern as the `` block). /// Callers without an `occurred_at` pass `None`; the impl does NOT /// fall back to `now()`. -pub const FACT_EXTRACTION_PROMPT_VERSION: &str = "extract.v6"; +/// +/// v7: the `` user message now carries a server-computed +/// weekday calendar (`last` / `this` / `next` for each weekday). The +/// prompt tells the model to copy those dates. Leaving the arithmetic +/// to the model stamped the wrong day, and the weekday name disagreed +/// with the ISO date (GH #1131). +pub const FACT_EXTRACTION_PROMPT_VERSION: &str = "extract.v7"; /// Map a bucket name from the extractor LLM to a numeric importance score. /// Unknown / missing buckets default to `IMPORTANCE_STANDARD` so a noisy @@ -498,23 +504,60 @@ fn render_related_memories_block(memories: &[&str]) -> String { out } -/// Render a `` block for the temporal-anchor -/// user message. Format is a single self-closing XML-style tag (no body -/// content) carrying the RFC 3339 UTC timestamp. +/// Sunday-first so `Weekday::num_days_from_sunday` indexes it. +const WEEKDAYS: [&str; 7] = [ + "Sunday", + "Monday", + "Tuesday", + "Wednesday", + "Thursday", + "Friday", + "Saturday", +]; + +/// Render the temporal-anchor user message: the `` tag plus a +/// weekday calendar the model copies instead of calculating. /// -/// The prompt (see `prompts/extract.txt`, v6 onwards) instructs the LLM -/// to use this timestamp as the absolute anchor for resolving in-turn -/// relative-time references ("last Friday", "yesterday") into absolute -/// dates that end up *inside the extracted fact text* — so the date -/// flows into both the SEAL-encrypted blob and the embedding vector. +/// The prompt (see `prompts/extract.txt`, v7) tells the LLM to copy +/// `last` / `this` / `next` dates from this block. "this DAY" is that +/// weekday on or after `occurred_at` (today, when today is that +/// weekday). Asking the model to do the arithmetic stamped the wrong +/// day, and the weekday name disagreed with the ISO date (GH #1131). /// -/// No prompt-injection escaping is needed here: the timestamp value is -/// a server-controlled RFC 3339 serialisation, not user-supplied text. -/// `chrono::DateTime::to_rfc3339` produces deterministic output -/// from the standard library — `2023-05-25T17:50:00+00:00` — with no -/// embeddable control characters. +/// No prompt-injection escaping is needed: every value is a +/// server-controlled date, not user-supplied text. Weekday names are +/// English literals so a non-English process locale cannot rename them. fn render_occurred_at_block(occurred_at: chrono::DateTime) -> String { - format!("", occurred_at.to_rfc3339()) + use chrono::{Datelike, Duration}; + + let today = occurred_at.date_naive(); + let mut out = format!( + "\n", + occurred_at.to_rfc3339(), + WEEKDAYS[today.weekday().num_days_from_sunday() as usize], + ); + out.push_str( + "Weekday calendar — copy these dates exactly; do not calculate your own. \ +A bare weekday (\"by Friday\", \"on Sunday\") means the \"this\" date. \ +\"next DAY\" is seven days after \"this\". \"last DAY\" is seven days before \"this\".\n", + ); + let mut weekday = chrono::Weekday::Sun; + for name in WEEKDAYS { + let ahead = i64::from(weekday.days_since(today.weekday())); + let this_day = today + Duration::days(ahead); + let last_day = this_day - Duration::days(7); + let next_day = this_day + Duration::days(7); + let today_mark = if ahead == 0 { " (today)" } else { "" }; + out.push_str(&format!( + "{name}: last {last}; this {this}{today_mark}; next {next}\n", + last = last_day.format("%Y-%m-%d"), + this = this_day.format("%Y-%m-%d"), + today_mark = today_mark, + next = next_day.format("%Y-%m-%d"), + )); + weekday = weekday.succ(); + } + out } /// Parsed shape of OpenRouter's "200 OK wrapping an upstream @@ -1103,19 +1146,23 @@ mod tests { ); // The version const must track the prompt: if the prompt changes, // the version should not silently stay behind. - assert_eq!(FACT_EXTRACTION_PROMPT_VERSION, "extract.v6"); + assert_eq!(FACT_EXTRACTION_PROMPT_VERSION, "extract.v7"); } #[test] fn extract_v6_prompt_contains_temporal_anchor_section() { - // the extract.v6 prompt must instruct the LLM about the - // `` tag. Pin three load-bearing + // the extract.v7 prompt must instruct the LLM about the + // `` tag. Pin the load-bearing // pieces of the temporal-anchor section so a future prompt edit // can't silently remove them. let prompt = FACT_EXTRACTION_PROMPT; assert!( - prompt.contains("``"), - "v6 must document the context tag the LLM should expect" + prompt.contains("``"), + "v7 must document the context tag the LLM should expect" + ); + assert!( + prompt.contains("Copy dates from the calendar"), + "v7 must tell the model to copy the weekday calendar instead of calculating" ); assert!( prompt.contains("temporal anchor"), @@ -1149,10 +1196,17 @@ mod tests { .unwrap() .with_timezone(&chrono::Utc); let block = super::render_occurred_at_block(ts); - assert_eq!( - block, "", - "tag shape must match what the prompt's v6 examples reference" + assert!( + block.starts_with( + "" + ), + "tag shape must keep the RFC 3339 timestamp and name the weekday: {block}" + ); + // 2023-05-25 is a Thursday. "last Friday" in the prompt example is 19 May. + assert!( + block.contains("Thursday: last 2023-05-18; this 2023-05-25 (today); next 2023-06-01") ); + assert!(block.contains("Friday: last 2023-05-19; this 2023-05-26; next 2023-06-02")); } #[test] @@ -1164,10 +1218,30 @@ mod tests { .unwrap() .with_timezone(&chrono::Utc); let block = super::render_occurred_at_block(ts_naive); - // -07:00 + 7h = +00:00 → "2023-05-26T00:50:00.123+00:00" - assert_eq!( - block, "", - "non-UTC inputs must be normalised to UTC in the rendered tag" + // -07:00 + 7h = +00:00 → "2023-05-26T00:50:00.123+00:00", a Friday. + assert!( + block.starts_with( + "" + ), + "non-UTC inputs must be normalised to UTC in the rendered tag: {block}" + ); + assert!(block.contains("Friday: last 2023-05-19; this 2023-05-26 (today); next 2023-06-02")); + } + + #[test] + fn weekday_calendar_uses_the_next_occurrence_on_or_after_today() { + // GH #1131: occurred_at is Wednesday 2026-10-07. "by Sunday" is + // 2026-10-11 and "by Friday" is 2026-10-09. The model was + // inventing both, so those dates have to be in the calendar. + let ts = chrono::DateTime::parse_from_rfc3339("2026-10-07T10:00:00Z") + .unwrap() + .with_timezone(&chrono::Utc); + let block = super::render_occurred_at_block(ts); + assert!(block.contains("weekday=\"Wednesday\"")); + assert!(block.contains("Sunday: last 2026-10-04; this 2026-10-11; next 2026-10-18")); + assert!(block.contains("Friday: last 2026-10-02; this 2026-10-09; next 2026-10-16")); + assert!( + block.contains("Wednesday: last 2026-09-30; this 2026-10-07 (today); next 2026-10-14") ); } diff --git a/services/server/src/services/prompts/extract.txt b/services/server/src/services/prompts/extract.txt index 29862d41e..417cbf928 100644 --- a/services/server/src/services/prompts/extract.txt +++ b/services/server/src/services/prompts/extract.txt @@ -10,10 +10,10 @@ You may receive a `` block alongside the input. It lists exist - Do NOT edit, merge, or supersede existing memories — only emit what is new in the input - If no `` block is present, or it is empty, extract as usual without context -You may also receive a `` tag alongside the input. The `occurred_at` value is the absolute timestamp (RFC 3339) of when this conversation turn took place. Use it as the **temporal anchor** for facts extracted from this input: +You may also receive a `` tag, followed by a weekday calendar, alongside the input. The `occurred_at` value is the absolute timestamp (RFC 3339) of when this conversation turn took place. Use it as the **temporal anchor** for facts extracted from this input: - **Default to attaching a date** when a fact describes a current event, recent action, state change happening now, or near-future plan. Use the format `Weekday, D Month YYYY (YYYY-MM-DD)` — for example `Friday, 19 May 2023 (2023-05-19)` — embedding both the natural-language form and the ISO date. -- **Resolve specific relative time references against `occurred_at` before writing the fact.** "Yesterday" + occurred_at=2023-05-25 → `Wednesday, 24 May 2023 (2023-05-24)`. "Last Friday" → use occurred_at's weekday to compute the prior Friday. "Two weeks ago" → subtract 14 days. "This morning" → the same date as occurred_at. +- **Resolve specific relative time references against `occurred_at` and the weekday calendar before writing the fact.** Copy dates from the calendar. Do not calculate them. "Yesterday" + occurred_at=2023-05-25 → `Wednesday, 24 May 2023 (2023-05-24)`. "Last Friday" copies the calendar's **last** Friday. "Two weeks ago" → subtract 14 days. "This morning" → the same date as occurred_at. A bare weekday ("by Friday", "on Sunday", "this Friday") copies the calendar's **this** date. "Next Friday" copies the **next** date. The weekday name and the `(YYYY-MM-DD)` in the fact must be that same calendar cell. If they would disagree, copy the calendar instead of keeping either invented half. - **Keep vague references honest.** If a relative reference is too imprecise to resolve to a specific date (e.g. "a few years ago", "recently", "sometime in March", "next week"), do NOT fabricate a specific date. Either preserve the user's vague phrasing as-is, or — when the conversation timestamp lets you narrow it without inventing — anchor to a month using the format `around late May 2023 (2023-05)`. - **CRITICAL: if a fact is recounting an event from the past that has its own original date or relative reference (e.g. "I went to Paris in 2019", "I broke my arm when I was twelve"), keep that original date — do NOT replace it with `occurred_at`.** The `occurred_at` anchor is only for events happening *in* or *around* this conversation turn, not for arbitrary recounted memories. - **When you are uncertain whether a fact is time-anchored or stable, prefer leaving the date off.** A timeless fact is recoverable; a wrongly-stamped fact pollutes retrieval. Stable traits (allergies, names, locations, preferences, ongoing roles) should never carry a date. @@ -89,7 +89,17 @@ Input: "User: I went hiking last Friday with my brother." Output: standard User went hiking on Friday, 19 May 2023 (2023-05-19) with their brother -(Note: "last Friday" was resolved against occurred_at=2023-05-25 (a Thursday), so the prior Friday is 19 May 2023. The absolute date in both natural-language and ISO form is attached to the fact.) +(Note: "last Friday" is copied from the weekday calendar's **last** Friday for occurred_at=2023-05-25 (a Thursday), which is 19 May 2023. The absolute date in both natural-language and ISO form is attached to the fact. Do not calculate a different Friday.) + +Example with `` — a bare weekday copies the calendar's "this" date: + +Friday: last 2026-10-02; this 2026-10-09; next 2026-10-16 +Sunday: last 2026-10-04; this 2026-10-11; next 2026-10-18 +Input: "User: I want to ship my first Sui contract by Sunday." +Output: +standard User wants to ship their first Sui contract by Sunday, 11 October 2026 (2026-10-11) + +(Note: "by Sunday" names a weekday and does not say "last" or "next", so the date is the calendar's **this** Sunday, 2026-10-11. The weekday name and the ISO date are that same cell.) Example with `` — recounted old event keeps its own date: diff --git a/services/server/src/types.rs b/services/server/src/types.rs index 9a93ed671..b510f2d80 100644 --- a/services/server/src/types.rs +++ b/services/server/src/types.rs @@ -1932,6 +1932,15 @@ fn default_namespace() -> String { "default".to_string() } +/// Empty or whitespace-only text. Callers reject it before creating a job. +/// +/// `str::is_empty` lets `" "`, `"\t"`, and `"\n"` through. Those used to be +/// accepted and then sit until the client timed out (GH #1129). A non-blank +/// fact is not trimmed: surrounding spaces stay part of the stored text. +pub fn text_is_blank(text: &str) -> bool { + text.trim().is_empty() +} + /// Shared namespace validation for every request that carries a namespace. /// /// API-compatibility note: this rejects the empty string, and the read paths @@ -4136,6 +4145,17 @@ mod tests { assert!(validate_namespace("has\ttab").is_ok()); } + #[test] + fn blank_text_is_empty_or_whitespace_only() { + assert!(text_is_blank("")); + assert!(text_is_blank(" ")); + assert!(text_is_blank("\t")); + assert!(text_is_blank("\n")); + assert!(text_is_blank(" \t\n ")); + // Surrounding whitespace on a real fact is kept by the caller. + assert!(!text_is_blank(" hello ")); + } + // ── HealthResponse.prompt_versions wire shape ──────────────── #[test] From a6fd85225e131cbfd340f662c81bfa598ecce785 Mon Sep 17 00:00:00 2001 From: HoangDucBach Date: Tue, 8 Sep 2026 14:58:21 +0700 Subject: [PATCH 13/70] fix(server): return each blob once in recall (WALM-594) search_similar kept every vector_entries row, so a remember retry or restore could surface the same Walrus blob twice with identical scores. --- services/server/src/routes/recall.rs | 32 +++---- services/server/src/storage/db.rs | 128 ++++++++++++++++++++++++++- 2 files changed, 137 insertions(+), 23 deletions(-) diff --git a/services/server/src/routes/recall.rs b/services/server/src/routes/recall.rs index 6bbef7752..2c725111e 100644 --- a/services/server/src/routes/recall.rs +++ b/services/server/src/routes/recall.rs @@ -85,18 +85,12 @@ async fn generate_recall_embedding_cached( /// `text` (the ranker never reads text; manual recall never decrypts). /// /// The ranked output is mapped back to the original `SearchHit`s **by -/// original index, not by `blob_id`**. `blob_id` is not unique -/// (`vector_entries` has no UNIQUE constraint on it and `search_similar` -/// does not `SELECT DISTINCT`, so `restore` can produce — and a query can -/// return — multiple hits with the same blob_id). Keying the round-trip on -/// blob_id would collapse those duplicates, silently dropping hits and -/// reordering them — re-introducing the very manual-vs-non-manual -/// divergence this fix removes (the non-manual paths keep duplicates 1:1). -/// So we stash each hit's input index in the throwaway `HydratedMemory`'s -/// `blob_id` slot — the ranker treats that field as opaque carry-through -/// (it scores only distance/recency/importance) — and reorder the original -/// `Vec` by the ranked index sequence. Indices are unique, so no -/// hit is dropped and `results.len() == hits.len()` always. +/// original index, not by `blob_id`**. `search_similar` already collapses +/// duplicate blob_ids (WALM-594), but this remap still must not key on +/// `blob_id`: the ranker only reads distance/recency/importance, and we +/// stash each hit's input index in the throwaway `HydratedMemory`'s +/// `blob_id` slot so a duplicate-bearing input cannot silently drop or +/// reorder hits. Indices are unique, so `results.len() == hits.len()`. /// /// At default weights `rank()` short-circuits, so the input (cosine) order /// is returned unchanged. @@ -107,7 +101,7 @@ fn rank_search_hits( now: chrono::DateTime, ) -> Vec { // Carry the original index through the ranker via the (opaque-to-ranker) - // blob_id field, so we can reorder duplicate-blob_id hits unambiguously. + // blob_id field so reorder is unambiguous even if two hits share a blob_id. let hydrated: Vec = hits .iter() .enumerate() @@ -650,13 +644,11 @@ Available: 10708877"; assert_eq!(ranked[0].created_at, t_now() - chrono::Duration::days(7)); } - /// Regression guard (deep-review blocker): `blob_id` is NOT - /// unique — `search_similar` can return multiple hits with the same - /// blob_id. A blob_id-keyed round-trip would collapse them, silently - /// dropping hits and reordering — re-introducing the manual-vs-non-manual - /// divergence this fix removes (the non-manual paths keep duplicates 1:1). - /// The index-based reorder must keep every hit. Tested at BOTH default - /// (short-circuit) and active weights. + /// Ranker remap is by input index, not `blob_id`. `search_similar` + /// collapses duplicate blob_ids (WALM-594); this still pins that a + /// blob_id-keyed round-trip would drop/reorder if duplicates ever + /// reached the ranker. Tested at BOTH default (short-circuit) and + /// active weights. #[test] fn manual_ranking_keeps_duplicate_blob_ids() { use crate::services::extractor::{IMPORTANCE_TRIVIAL, IMPORTANCE_VITAL}; diff --git a/services/server/src/storage/db.rs b/services/server/src/storage/db.rs index 4a6ae1e8d..fb8bec3e6 100644 --- a/services/server/src/storage/db.rs +++ b/services/server/src/storage/db.rs @@ -3,11 +3,23 @@ use std::sync::Arc; use pgvector::Vector; use sqlx::postgres::PgPoolOptions; use sqlx::{PgConnection, PgPool, Postgres, Transaction}; +use std::collections::HashSet; use std::sync::atomic::AtomicBool; use crate::alerts::AlertManager; use crate::types::{AppError, SearchHit}; +/// Keep the first hit for each `blob_id` (closest: callers already order by +/// cosine distance). `vector_entries` has no UNIQUE on blob_id, so a +/// remember retry or restore can insert a second row for the same Walrus +/// blob; returning both spends `limit` on one memory (WALM-594 / GH #694). +pub(crate) fn unique_hits_by_blob_id(hits: Vec) -> Vec { + let mut seen = HashSet::with_capacity(hits.len()); + hits.into_iter() + .filter(|hit| seen.insert(hit.blob_id.clone())) + .collect() +} + /// Tombstone retention for both the read-API `must_resync` clock and the /// background sweep. Keep a single constant so the two cannot drift. pub const TOMBSTONE_RETENTION: chrono::Duration = chrono::Duration::days(30); @@ -819,6 +831,116 @@ mod tests { ); } + fn hit(blob_id: &str, distance: f64) -> crate::types::SearchHit { + crate::types::SearchHit { + blob_id: blob_id.to_string(), + distance, + created_at: chrono::Utc::now(), + importance: 0.5, + } + } + + #[test] + fn unique_hits_by_blob_id_keeps_first_and_distinct() { + let unique = super::unique_hits_by_blob_id(vec![ + hit("dup", 0.10), + hit("dup", 0.10), + hit("other", 0.20), + hit("dup", 0.30), + ]); + let ids: Vec<&str> = unique.iter().map(|h| h.blob_id.as_str()).collect(); + assert_eq!(ids, vec!["dup", "other"]); + assert_eq!(unique[0].distance, 0.10); + } + + #[test] + fn unique_hits_by_blob_id_empty() { + assert!(super::unique_hits_by_blob_id(vec![]).is_empty()); + } + + #[tokio::test] + async fn search_similar_returns_each_blob_id_once() { + let Some(db) = test_db().await else { + eprintln!("skipping DB integration test: DATABASE_URL is not configured"); + return; + }; + let suffix = uuid::Uuid::new_v4(); + let owner = format!("0xrecall-dedupe-owner-{suffix}"); + let namespace = format!("ns-{suffix}"); + let blob_id = format!("dup-blob-{suffix}"); + let other_blob = format!("other-blob-{suffix}"); + let query = vec![1.0_f32; 1536]; + let near = vec![1.0_f32; 1536]; + let far = { + let mut v = vec![0.0_f32; 1536]; + v[0] = 1.0; + v + }; + + db.insert_vector( + &format!("row-a-{suffix}"), + &owner, + &namespace, + &blob_id, + &near, + 1, + 0.5, + None, + None, + None, + ) + .await + .unwrap(); + db.insert_vector( + &format!("row-b-{suffix}"), + &owner, + &namespace, + &blob_id, + &near, + 1, + 0.5, + None, + None, + None, + ) + .await + .unwrap(); + db.insert_vector( + &format!("row-c-{suffix}"), + &owner, + &namespace, + &other_blob, + &far, + 1, + 0.5, + None, + None, + None, + ) + .await + .unwrap(); + + let hits = db + .search_similar(&query, &owner, &namespace, 10) + .await + .unwrap(); + sqlx::query("DELETE FROM vector_entries WHERE owner = $1") + .bind(&owner) + .execute(db.pool()) + .await + .unwrap(); + + let ids: Vec<&str> = hits.iter().map(|h| h.blob_id.as_str()).collect(); + assert_eq!( + ids.iter().filter(|id| **id == blob_id).count(), + 1, + "the remembered blob must appear once, not once per index row" + ); + assert!(ids.contains(&blob_id.as_str())); + assert!(ids.contains(&other_blob.as_str())); + assert_eq!(ids.len(), 2); + } + #[tokio::test] async fn insert_vector_persists_agent_and_package_id() { let Some(db) = test_db().await else { @@ -2802,8 +2924,8 @@ impl VectorDb { Ok(row.and_then(|(plaintext,)| plaintext)) } - /// Search for similar vectors using pgvector cosine distance (<=>) - /// Returns blob_id and distance for each match + /// Search for similar vectors using pgvector cosine distance (<=>). + /// Each `blob_id` appears at most once (closest row wins). pub async fn search_similar( &self, query_vector: &[f32], @@ -2856,7 +2978,7 @@ impl VectorDb { }) .collect(); - Ok(results) + Ok(unique_hits_by_blob_id(results)) } /// Get all blob_ids for a given owner + namespace (used by restore flow) From 0eae8f2e06236cd0bbac7d22533ba4fe5ea0725b Mon Sep 17 00:00:00 2001 From: HoangDucBach Date: Wed, 9 Sep 2026 08:31:29 +0700 Subject: [PATCH 14/70] fix(server): refill recall limit after collapsing duplicate blobs Duplicate index rows were eating the ANN LIMIT window. Over-fetch by the dropped count, then unique and truncate. Ranker docs no longer narrate WALM-594. --- services/server/src/routes/recall.rs | 17 ++-- services/server/src/storage/db.rs | 126 +++++++++++++++++++++++++-- 2 files changed, 126 insertions(+), 17 deletions(-) diff --git a/services/server/src/routes/recall.rs b/services/server/src/routes/recall.rs index 2c725111e..46f3a2b76 100644 --- a/services/server/src/routes/recall.rs +++ b/services/server/src/routes/recall.rs @@ -85,12 +85,10 @@ async fn generate_recall_embedding_cached( /// `text` (the ranker never reads text; manual recall never decrypts). /// /// The ranked output is mapped back to the original `SearchHit`s **by -/// original index, not by `blob_id`**. `search_similar` already collapses -/// duplicate blob_ids (WALM-594), but this remap still must not key on -/// `blob_id`: the ranker only reads distance/recency/importance, and we -/// stash each hit's input index in the throwaway `HydratedMemory`'s -/// `blob_id` slot so a duplicate-bearing input cannot silently drop or -/// reorder hits. Indices are unique, so `results.len() == hits.len()`. +/// original index, not by `blob_id`**. The ranker treats `blob_id` as +/// opaque carry-through (it scores only distance/recency/importance), so +/// we stash each hit's input index in that slot. Indices are unique, so a +/// blob_id-keyed round-trip cannot drop hits and `results.len() == hits.len()`. /// /// At default weights `rank()` short-circuits, so the input (cosine) order /// is returned unchanged. @@ -644,11 +642,8 @@ Available: 10708877"; assert_eq!(ranked[0].created_at, t_now() - chrono::Duration::days(7)); } - /// Ranker remap is by input index, not `blob_id`. `search_similar` - /// collapses duplicate blob_ids (WALM-594); this still pins that a - /// blob_id-keyed round-trip would drop/reorder if duplicates ever - /// reached the ranker. Tested at BOTH default (short-circuit) and - /// active weights. + /// Remap is by input index, not blob_id, so two hits sharing a blob_id + /// are both kept. Tested at default (short-circuit) and active weights. #[test] fn manual_ranking_keeps_duplicate_blob_ids() { use crate::services::extractor::{IMPORTANCE_TRIVIAL, IMPORTANCE_VITAL}; diff --git a/services/server/src/storage/db.rs b/services/server/src/storage/db.rs index fb8bec3e6..b7dd1701a 100644 --- a/services/server/src/storage/db.rs +++ b/services/server/src/storage/db.rs @@ -941,6 +941,84 @@ mod tests { assert_eq!(ids.len(), 2); } + #[tokio::test] + async fn search_similar_fills_limit_when_duplicate_rows_crowd_the_window() { + let Some(db) = test_db().await else { + eprintln!("skipping DB integration test: DATABASE_URL is not configured"); + return; + }; + let suffix = uuid::Uuid::new_v4(); + let owner = format!("0xrecall-dedupe-limit-{suffix}"); + let namespace = format!("ns-{suffix}"); + let blob_a = format!("blob-a-{suffix}"); + let blob_b = format!("blob-b-{suffix}"); + let query = vec![1.0_f32; 1536]; + let near = vec![1.0_f32; 1536]; + let far = { + let mut v = vec![0.0_f32; 1536]; + v[0] = 1.0; + v + }; + + db.insert_vector( + &format!("row-a1-{suffix}"), + &owner, + &namespace, + &blob_a, + &near, + 1, + 0.5, + None, + None, + None, + ) + .await + .unwrap(); + db.insert_vector( + &format!("row-a2-{suffix}"), + &owner, + &namespace, + &blob_a, + &near, + 1, + 0.5, + None, + None, + None, + ) + .await + .unwrap(); + db.insert_vector( + &format!("row-b-{suffix}"), + &owner, + &namespace, + &blob_b, + &far, + 1, + 0.5, + None, + None, + None, + ) + .await + .unwrap(); + + let hits = db + .search_similar(&query, &owner, &namespace, 2) + .await + .unwrap(); + sqlx::query("DELETE FROM vector_entries WHERE owner = $1") + .bind(&owner) + .execute(db.pool()) + .await + .unwrap(); + + let ids: Vec<&str> = hits.iter().map(|h| h.blob_id.as_str()).collect(); + assert_eq!(ids.len(), 2, "limit=2 must refill after collapsing blob A"); + assert!(ids.contains(&blob_a.as_str())); + assert!(ids.contains(&blob_b.as_str())); + } + #[tokio::test] async fn insert_vector_persists_agent_and_package_id() { let Some(db) = test_db().await else { @@ -2925,13 +3003,51 @@ impl VectorDb { } /// Search for similar vectors using pgvector cosine distance (<=>). - /// Each `blob_id` appears at most once (closest row wins). + /// Each `blob_id` appears at most once (closest row wins). Duplicate + /// index rows are over-fetched so `limit` still fills with distinct blobs. pub async fn search_similar( &self, query_vector: &[f32], owner: &str, namespace: &str, limit: usize, + ) -> Result, AppError> { + if limit == 0 { + return Ok(vec![]); + } + + // First-wins unique after ANN `LIMIT` is HNSW-friendly, but duplicate + // index rows consume that window. Widen by the number dropped until + // `limit` unique blobs are filled or the namespace is exhausted. + let mut fetch_limit = limit; + loop { + let rows = self + .search_similar_rows(query_vector, owner, namespace, fetch_limit) + .await?; + let fetched = rows.len(); + let mut unique = unique_hits_by_blob_id(rows); + if unique.len() >= limit { + unique.truncate(limit); + return Ok(unique); + } + if fetched < fetch_limit { + return Ok(unique); + } + let dropped = fetched - unique.len(); + let next = limit.saturating_add(dropped); + if next <= fetch_limit { + return Ok(unique); + } + fetch_limit = next; + } + } + + async fn search_similar_rows( + &self, + query_vector: &[f32], + owner: &str, + namespace: &str, + fetch_limit: usize, ) -> Result, AppError> { let embedding = Vector::from(query_vector.to_vec()); @@ -2957,7 +3073,7 @@ impl VectorDb { .bind(embedding) .bind(owner) .bind(namespace) - .bind(limit as i64) + .bind(fetch_limit as i64) .fetch_all(&self.pool) .await .map_err(|e| AppError::Internal(format!("Failed to search vectors: {}", e))); @@ -2968,7 +3084,7 @@ impl VectorDb { ); let rows = result?; - let results = rows + Ok(rows .into_iter() .map(|(blob_id, distance, created_at, importance)| SearchHit { blob_id, @@ -2976,9 +3092,7 @@ impl VectorDb { created_at, importance, }) - .collect(); - - Ok(unique_hits_by_blob_id(results)) + .collect()) } /// Get all blob_ids for a given owner + namespace (used by restore flow) From 73b13e666d8493afb1c86b81fc988c67d7817ec7 Mon Sep 17 00:00:00 2001 From: HoangDucBach Date: Wed, 9 Sep 2026 09:01:40 +0700 Subject: [PATCH 15/70] fix(server): grow recall over-fetch by duplicate factor, cap at 4x limit + dropped only added the unique shortfall, so a 2x index crawled many HNSW round-trips. Widen by dropped rows in the current window and stop at limit * 4. --- services/server/src/storage/db.rs | 116 ++++++++++++++++++++++++++++-- 1 file changed, 111 insertions(+), 5 deletions(-) diff --git a/services/server/src/storage/db.rs b/services/server/src/storage/db.rs index b7dd1701a..fa87be1e6 100644 --- a/services/server/src/storage/db.rs +++ b/services/server/src/storage/db.rs @@ -20,6 +20,31 @@ pub(crate) fn unique_hits_by_blob_id(hits: Vec) -> Vec { .collect() } +/// ANN over-fetch ceiling: `fetch_limit <= limit * this`. Stops a +/// single heavily duplicated nearest blob from walking the whole +/// namespace one unique-shortfall at a time. +const SEARCH_SIMILAR_OVERFETCH_FACTOR: usize = 4; + +/// Next ANN `LIMIT` after collapsing `dropped` duplicate rows from a +/// full page. Grows by the observed duplicate count in the current +/// window (not `limit + dropped`, which only adds the unique shortfall). +/// `None` means stop: no dups, no growth, or the cap is hit. +pub(crate) fn next_search_fetch_limit( + limit: usize, + fetch_limit: usize, + dropped: usize, +) -> Option { + if dropped == 0 { + return None; + } + let cap = limit.saturating_mul(SEARCH_SIMILAR_OVERFETCH_FACTOR).max(limit); + if fetch_limit >= cap { + return None; + } + let next = fetch_limit.saturating_add(dropped).min(cap); + (next > fetch_limit).then_some(next) +} + /// Tombstone retention for both the read-API `must_resync` clock and the /// background sweep. Keep a single constant so the two cannot drift. pub const TOMBSTONE_RETENTION: chrono::Duration = chrono::Duration::days(30); @@ -858,6 +883,27 @@ mod tests { assert!(super::unique_hits_by_blob_id(vec![]).is_empty()); } + #[test] + fn next_search_fetch_limit_grows_by_dropped_not_unique_shortfall() { + // limit=2, two copies of A: first page unique=1, dropped=1 → LIMIT 3. + assert_eq!(super::next_search_fetch_limit(2, 2, 1), Some(3)); + // Three copies of A: LIMIT 3 is still all A (dropped=2) → LIMIT 5, + // not stop after the first widen. + assert_eq!(super::next_search_fetch_limit(2, 3, 2), Some(5)); + // Cap at limit * 4. + assert_eq!(super::next_search_fetch_limit(2, 5, 4), Some(8)); + assert_eq!(super::next_search_fetch_limit(2, 8, 7), None); + } + + #[test] + fn next_search_fetch_limit_uniform_2x_at_http_cap() { + // limit=100, half the page duplicate: grow by dropped (50), not + // by unique shortfall alone. 100 → 150, then 150 → 225. + assert_eq!(super::next_search_fetch_limit(100, 100, 50), Some(150)); + assert_eq!(super::next_search_fetch_limit(100, 150, 75), Some(225)); + assert_eq!(super::next_search_fetch_limit(100, 400, 50), None); + } + #[tokio::test] async fn search_similar_returns_each_blob_id_once() { let Some(db) = test_db().await else { @@ -1019,6 +1065,67 @@ mod tests { assert!(ids.contains(&blob_b.as_str())); } + #[tokio::test] + async fn search_similar_widens_again_when_three_copies_crowd_limit_two() { + let Some(db) = test_db().await else { + eprintln!("skipping DB integration test: DATABASE_URL is not configured"); + return; + }; + let suffix = uuid::Uuid::new_v4(); + let owner = format!("0xrecall-dedupe-triple-{suffix}"); + let namespace = format!("ns-{suffix}"); + let blob_a = format!("blob-a-{suffix}"); + let blob_b = format!("blob-b-{suffix}"); + let query = vec![1.0_f32; 1536]; + let near = vec![1.0_f32; 1536]; + let far = { + let mut v = vec![0.0_f32; 1536]; + v[0] = 1.0; + v + }; + + for (row, blob, embedding) in [ + ("a1", blob_a.as_str(), &near), + ("a2", blob_a.as_str(), &near), + ("a3", blob_a.as_str(), &near), + ("b", blob_b.as_str(), &far), + ] { + db.insert_vector( + &format!("row-{row}-{suffix}"), + &owner, + &namespace, + blob, + embedding, + 1, + 0.5, + None, + None, + None, + ) + .await + .unwrap(); + } + + let hits = db + .search_similar(&query, &owner, &namespace, 2) + .await + .unwrap(); + sqlx::query("DELETE FROM vector_entries WHERE owner = $1") + .bind(&owner) + .execute(db.pool()) + .await + .unwrap(); + + let ids: Vec<&str> = hits.iter().map(|h| h.blob_id.as_str()).collect(); + assert_eq!( + ids.len(), + 2, + "first widen (LIMIT 3) is still all A; second widen must include B" + ); + assert!(ids.contains(&blob_a.as_str())); + assert!(ids.contains(&blob_b.as_str())); + } + #[tokio::test] async fn insert_vector_persists_agent_and_package_id() { let Some(db) = test_db().await else { @@ -3017,8 +3124,8 @@ impl VectorDb { } // First-wins unique after ANN `LIMIT` is HNSW-friendly, but duplicate - // index rows consume that window. Widen by the number dropped until - // `limit` unique blobs are filled or the namespace is exhausted. + // index rows consume that window. Widen by the dropped count in the + // current window (duplicate factor), capped at `limit * 4`. let mut fetch_limit = limit; loop { let rows = self @@ -3034,10 +3141,9 @@ impl VectorDb { return Ok(unique); } let dropped = fetched - unique.len(); - let next = limit.saturating_add(dropped); - if next <= fetch_limit { + let Some(next) = next_search_fetch_limit(limit, fetch_limit, dropped) else { return Ok(unique); - } + }; fetch_limit = next; } } From d9aa05303398d82b131c0f3f2432cb374a4ff8da Mon Sep 17 00:00:00 2001 From: HoangDucBach Date: Tue, 8 Sep 2026 09:16:36 +0700 Subject: [PATCH 16/70] fix(sdk): throw when positional recall gets a namespace string (WALM-161) recall(query, namespace) was silently treated as options and ran against the default namespace. Fail loudly instead of dropping the intended scope. --- docs/sdk/api-reference.md | 2 +- docs/sdk/changelog.mdx | 1 + packages/sdk/CHANGELOG.md | 1 + packages/sdk/src/manual.ts | 9 ++- packages/sdk/src/memwal.ts | 38 +++++------- packages/sdk/src/mock.ts | 36 +++++------- packages/sdk/src/recall-args.ts | 58 +++++++++++++++++++ .../sdk/test/manual-seal-identity.test.mjs | 8 +++ packages/sdk/test/mock.test.mjs | 28 +++++++++ packages/sdk/test/recall-args.test.mjs | 31 ++++++++++ 10 files changed, 165 insertions(+), 47 deletions(-) create mode 100644 packages/sdk/src/recall-args.ts create mode 100644 packages/sdk/test/recall-args.test.mjs diff --git a/docs/sdk/api-reference.md b/docs/sdk/api-reference.md index ec931610e..7fa288a94 100644 --- a/docs/sdk/api-reference.md +++ b/docs/sdk/api-reference.md @@ -112,7 +112,7 @@ Search for memories matching a natural language query, scoped to `owner + namesp - Preferred form: `recall({ query, limit?, topK?, namespace?, maxDistance?, sort?, scoringWeights? })` - `limit` defaults to `10`; `topK` is an alias and wins when both are set -- Legacy positional forms still work: `recall(query)`, `recall(query, limit)`, `recall(query, limit, namespace)`, and `recall(query, options)` +- Legacy positional forms still work: `recall(query)`, `recall(query, limit)`, `recall(query, limit, namespace)`, and `recall(query, options)`. `recall(query, namespace)` throws `TypeError` — namespace is not a valid second argument. - `maxDistance` filters weak matches client-side by dropping results where `distance >= maxDistance` - `sort` picks the ordering: `"relevance"` (default) or `"recent"` for newest-among-matches - `scoringWeights` blends recency and importance into the ranking (see [Ordering](#ordering) below) diff --git a/docs/sdk/changelog.mdx b/docs/sdk/changelog.mdx index a304afec1..b41d8dbce 100644 --- a/docs/sdk/changelog.mdx +++ b/docs/sdk/changelog.mdx @@ -77,6 +77,7 @@ This release adds write-time on recall results, lets callers sort by recency or ### Fixed - HTTP 503 from the relayer is reported as a retryable upstream outage, not a sign-in failure. Empty-body 401s still point at `memwal_login`. +- Positional `recall(query, namespace)` now throws `TypeError` instead of silently ignoring the namespace. ## 0.1.5 diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index 5ec940b26..99d504a3b 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -63,6 +63,7 @@ ### Fixed - HTTP 503 with `x-auth-error: AUTH_UPSTREAM_UNAVAILABLE` is reported as a retryable credential-verification outage, not a sign-in failure. Other 503s keep the generic sanitized body. Empty-body 401s still point at `memwal_login`. +- Positional `recall(query, namespace)` now throws `TypeError` instead of silently ignoring the namespace. ## 0.1.5 diff --git a/packages/sdk/src/manual.ts b/packages/sdk/src/manual.ts index 72257f17b..eee83861d 100644 --- a/packages/sdk/src/manual.ts +++ b/packages/sdk/src/manual.ts @@ -52,6 +52,7 @@ import { scoringWeightsToWire, } from "./utils.js"; import { assertCompatibleRelayer, compatibilityErrorFromStatus } from "./compatibility.js"; +import { resolveLimitOrOptions } from "./recall-args.js"; // ============================================================ // Constants @@ -380,8 +381,8 @@ export class MemWalManual { * 3. Download blobs from Walrus * 4. SEAL decrypt each blob */ + async recallManual(query: string, options: MemWalManualRecallOptions): Promise; async recallManual(query: string, limit?: number, namespace?: string): Promise; - async recallManual(query: string, options?: MemWalManualRecallOptions): Promise; async recallManual( query: string, limitOrOptions: number | MemWalManualRecallOptions = 10, @@ -389,7 +390,11 @@ export class MemWalManual { ): Promise { if (!query.trim()) throw new Error("Query cannot be empty"); - const options = typeof limitOrOptions === "number" ? { limit: limitOrOptions, namespace } : limitOrOptions; + const options = resolveLimitOrOptions( + "recallManual", + limitOrOptions, + namespace, + ); const limit = options.limit ?? 10; const ns = options.namespace ?? this.namespace; diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index 1a5a5b6f7..f52587779 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -76,6 +76,7 @@ import { } from "./compatibility.js"; import { applyTokenBudget, estimateTokens } from "./tokens.js"; import { pollingDelayMs } from "./polling-delay.js"; +import { resolveRecallCall } from "./recall-args.js"; // ============================================================ // Ed25519 Signing (lazy-loaded) @@ -934,6 +935,8 @@ export class MemWal { * - `recall(query, limit, namespace)` * - `recall(query, { limit, namespace, maxDistance, topK })` * + * `recall(query, namespace)` is invalid and throws `TypeError`. + * * `topK` and `limit` are aliases; if both are set, `topK` wins. * * @returns RecallResult with decrypted text results @@ -974,6 +977,13 @@ export class MemWal { } async recall(params: RecallParams): Promise; + /** + * @deprecated Positional `recall(query, limit, namespace)` is easy to + * misread as `recall(query, namespace)`. Prefer the object form + * `recall({ query, limit, namespace })`. Positional will be removed in a + * future major version of the SDK. + */ + async recall(query: string, options: RecallOptions): Promise; /** * @deprecated Positional `recall(query, limit, namespace)` is easy to * misread as `recall(query, namespace)`. Prefer the object form @@ -982,7 +992,7 @@ export class MemWal { */ async recall( query: string, - limitOrOptions?: number | RecallOptions, + limit?: number, namespace?: string, ): Promise; async recall( @@ -990,27 +1000,11 @@ export class MemWal { limitOrOptions: number | RecallOptions | undefined = 10, namespace?: string, ): Promise { - let query: string; - let options: RecallOptions; - if (typeof queryOrParams === "object") { - const { query: q, ...rest } = queryOrParams; - query = q; - options = rest; - } else { - query = queryOrParams; - if (limitOrOptions == null) { - options = { limit: 10, namespace }; - } else if (typeof limitOrOptions === "number") { - options = { limit: limitOrOptions, namespace }; - } else { - // Positional namespace fills an options object that omitted it. - // An explicit options.namespace still wins. (#1038) - options = { ...limitOrOptions }; - if (options.namespace === undefined && namespace !== undefined) { - options.namespace = namespace; - } - } - } + const { query, options } = resolveRecallCall( + queryOrParams, + limitOrOptions, + namespace, + ); const limit = options.topK ?? options.limit ?? 10; const resolvedNamespace = options.namespace ?? this.namespace; diff --git a/packages/sdk/src/mock.ts b/packages/sdk/src/mock.ts index 5f0168f3d..ae1c9d43d 100644 --- a/packages/sdk/src/mock.ts +++ b/packages/sdk/src/mock.ts @@ -22,6 +22,7 @@ import type { ListNamespacesOptions, } from "./types.js"; import { applyTokenBudget, estimateTokens } from "./tokens.js"; +import { resolveRecallCall } from "./recall-args.js"; /** Matches the relayer's `MAX_BULK_ITEMS`. */ const MAX_BULK_ITEMS = 20; @@ -308,9 +309,16 @@ export class MemWalMock { } async recall(params: RecallParams): Promise; + /** + * @deprecated Prefer `recall({ query, limit, namespace })`. + */ + async recall(query: string, options: RecallOptions): Promise; + /** + * @deprecated Prefer `recall({ query, limit, namespace })`. + */ async recall( query: string, - limitOrOptions?: number | RecallOptions, + limit?: number, namespace?: string ): Promise; async recall( @@ -318,27 +326,11 @@ export class MemWalMock { limitOrOptions: number | RecallOptions | undefined = 10, namespace?: string ): Promise { - let query: string; - let options: RecallOptions; - if (typeof queryOrParams === "object") { - const { query: objectQuery, ...rest } = queryOrParams; - query = objectQuery; - options = rest; - } else { - query = queryOrParams; - if (limitOrOptions == null) { - options = { limit: 10, namespace }; - } else if (typeof limitOrOptions === "number") { - options = { limit: limitOrOptions, namespace }; - } else { - // Positional namespace fills an options object that omitted it. - // An explicit options.namespace still wins. (#1038) - options = { ...limitOrOptions }; - if (options.namespace === undefined && namespace !== undefined) { - options.namespace = namespace; - } - } - } + const { query, options } = resolveRecallCall( + queryOrParams, + limitOrOptions, + namespace + ); validateText(query, "query"); const resolvedNamespace = options.namespace ?? this.namespace; const resolvedLimit = options.topK ?? options.limit ?? 10; diff --git a/packages/sdk/src/recall-args.ts b/packages/sdk/src/recall-args.ts new file mode 100644 index 000000000..00fc77eef --- /dev/null +++ b/packages/sdk/src/recall-args.ts @@ -0,0 +1,58 @@ +import type { RecallOptions, RecallParams } from "./types.js"; + +/** + * Parse `limit` / options for deprecated positional recall-style calls. + * A string second argument is the `recall(query, namespace)` footgun (#293). + */ +export function resolveLimitOrOptions( + method: string, + limitOrOptions: number | T | undefined | null, + namespace?: string, + defaultLimit = 10, +): T { + if (limitOrOptions == null) { + return { limit: defaultLimit, namespace } as T; + } + if (typeof limitOrOptions === "number") { + return { limit: limitOrOptions, namespace } as T; + } + if (typeof limitOrOptions === "object" && !Array.isArray(limitOrOptions)) { + // Positional namespace fills an options object that omitted it. + // An explicit options.namespace still wins, and the caller's object + // is never mutated. (#1038) + if (limitOrOptions.namespace === undefined && namespace !== undefined) { + return { ...limitOrOptions, namespace } as T; + } + return limitOrOptions; + } + throw new TypeError( + `${method}() second argument must be a number (limit) or an options object, not a string. ` + + `Namespace goes in ${method}({ query, namespace }) or as the third argument: ` + + `${method}(query, limit, namespace).`, + ); +} + +/** Normalize object-style and positional `recall()` arguments. */ +export function resolveRecallCall( + queryOrParams: string | RecallParams, + limitOrOptions?: number | RecallOptions | null, + namespace?: string, +): { query: string; options: RecallOptions } { + if ( + queryOrParams !== null && + typeof queryOrParams === "object" && + !Array.isArray(queryOrParams) + ) { + const { query, ...rest } = queryOrParams; + return { query, options: rest }; + } + if (typeof queryOrParams !== "string") { + throw new TypeError( + "recall() first argument must be a query string or a RecallParams object.", + ); + } + return { + query: queryOrParams, + options: resolveLimitOrOptions("recall", limitOrOptions, namespace), + }; +} diff --git a/packages/sdk/test/manual-seal-identity.test.mjs b/packages/sdk/test/manual-seal-identity.test.mjs index cb6f8e781..5c7181114 100644 --- a/packages/sdk/test/manual-seal-identity.test.mjs +++ b/packages/sdk/test/manual-seal-identity.test.mjs @@ -231,3 +231,11 @@ test("manual recall returns valid memories when another ciphertext package is fo assert.equal(errors.length, 1); assert.match(errors[0], /Skipping ciphertext foreign: packageId does not match/); }); + +test("manual recall rejects a string second argument", async () => { + const { manual } = manualWithAccountResponse({}); + await assert.rejects(() => manual.recallManual("query", "profile"), { + name: "TypeError", + message: /recallManual/, + }); +}); diff --git a/packages/sdk/test/mock.test.mjs b/packages/sdk/test/mock.test.mjs index a0c4c63d4..194051cf0 100644 --- a/packages/sdk/test/mock.test.mjs +++ b/packages/sdk/test/mock.test.mjs @@ -85,6 +85,34 @@ test("MemWalMock matches production recall overloads and topK precedence", async assert.equal(objectStyle.results.length, 2); }); +test("MemWalMock throws when positional recall gets a namespace string as the second arg", async () => { + const mock = MemWalMock.create(); + await mock.rememberAndWait("food allergies", "profile"); + + await assert.rejects(() => mock.recall("food allergies", "profile"), { + name: "TypeError", + message: /not a string/, + }); + + const positional = await mock.recall("food allergies", 5, "profile"); + const optionsStyle = await mock.recall("food allergies", { + namespace: "profile", + }); + const objectStyle = await mock.recall({ + query: "food allergies", + namespace: "profile", + }); + const defaultNs = await mock.recall({ query: "food allergies" }); + + assert.deepEqual( + positional.results.map((memory) => memory.text), + ["food allergies"] + ); + assert.equal(optionsStyle.total, 1); + assert.equal(objectStyle.total, 1); + assert.equal(defaultNs.total, 0); +}); + test("MemWalMock matches production token-budget behavior", async () => { const mock = MemWalMock.create({ initialMemories: [ diff --git a/packages/sdk/test/recall-args.test.mjs b/packages/sdk/test/recall-args.test.mjs new file mode 100644 index 000000000..a7973dd1e --- /dev/null +++ b/packages/sdk/test/recall-args.test.mjs @@ -0,0 +1,31 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { resolveLimitOrOptions, resolveRecallCall } from "../dist/recall-args.js"; + +test("resolveRecallCall maps object and positional forms", () => { + assert.deepEqual(resolveRecallCall({ query: "food", namespace: "profile", limit: 3 }), { + query: "food", + options: { namespace: "profile", limit: 3 }, + }); + assert.deepEqual(resolveRecallCall("food", 5, "profile"), { + query: "food", + options: { limit: 5, namespace: "profile" }, + }); + assert.deepEqual(resolveRecallCall("food", { namespace: "profile" }), { + query: "food", + options: { namespace: "profile" }, + }); +}); + +test("resolveRecallCall rejects a namespace string as the second argument", () => { + assert.throws(() => resolveRecallCall("food allergies", "profile"), { + name: "TypeError", + message: /recall\(\{ query, namespace \}\)/, + }); + assert.throws(() => resolveLimitOrOptions("recallManual", "profile"), { + name: "TypeError", + message: /recallManual/, + }); + assert.throws(() => resolveRecallCall("food", ["profile"]), TypeError); +}); From 3ea2895d144e893124e66f3d5c3695ecdd5c5579 Mon Sep 17 00:00:00 2001 From: HoangDucBach Date: Wed, 9 Sep 2026 14:09:14 +0700 Subject: [PATCH 17/70] fix(sdk): point recall TypeError at a call shape both APIs share recallManual has no object-first overload, so hinting method({ query, namespace }) sent callers to query.trim() failure. --- docs/sdk/api-reference.md | 5 ++- packages/sdk/src/manual.ts | 3 ++ packages/sdk/src/memwal.ts | 42 +++++++++---------- packages/sdk/src/recall-args.ts | 19 +++++---- .../sdk/test/manual-seal-identity.test.mjs | 4 +- packages/sdk/test/recall-args.test.mjs | 9 +++- 6 files changed, 49 insertions(+), 33 deletions(-) diff --git a/docs/sdk/api-reference.md b/docs/sdk/api-reference.md index 7fa288a94..1048c696e 100644 --- a/docs/sdk/api-reference.md +++ b/docs/sdk/api-reference.md @@ -112,7 +112,7 @@ Search for memories matching a natural language query, scoped to `owner + namesp - Preferred form: `recall({ query, limit?, topK?, namespace?, maxDistance?, sort?, scoringWeights? })` - `limit` defaults to `10`; `topK` is an alias and wins when both are set -- Legacy positional forms still work: `recall(query)`, `recall(query, limit)`, `recall(query, limit, namespace)`, and `recall(query, options)`. `recall(query, namespace)` throws `TypeError` — namespace is not a valid second argument. +- Legacy positional forms still work: `recall(query)`, `recall(query, limit)`, `recall(query, limit, namespace)`, and `recall(query, options)`. `recall(query, namespace)` throws `TypeError`; namespace is not a valid second argument. - `maxDistance` filters weak matches client-side by dropping results where `distance >= maxDistance` - `sort` picks the ordering: `"relevance"` (default) or `"recent"` for newest-among-matches - `scoringWeights` blends recency and importance into the ranking (see [Ordering](#ordering) below) @@ -352,6 +352,9 @@ Embed locally, SEAL encrypt locally, send encrypted payload + vector to relayer Embed locally, search via relayer, download from Walrus, SEAL decrypt locally. Returns decrypted text results. +- `recallManual(query, { limit?, namespace?, scoringWeights? })` is valid +- `recallManual(query, namespace)` throws `TypeError`; namespace is not a valid second argument + ### `restore(namespace, limit?): Promise` Same as `MemWal.restore()` — delegates to the relayer. diff --git a/packages/sdk/src/manual.ts b/packages/sdk/src/manual.ts index eee83861d..d7eb715d6 100644 --- a/packages/sdk/src/manual.ts +++ b/packages/sdk/src/manual.ts @@ -380,6 +380,9 @@ export class MemWalManual { * 2. Search server for matching vectors * 3. Download blobs from Walrus * 4. SEAL decrypt each blob + * + * `recallManual(query, { limit?, namespace? })` is valid. + * `recallManual(query, namespace)` throws `TypeError`. */ async recallManual(query: string, options: MemWalManualRecallOptions): Promise; async recallManual(query: string, limit?: number, namespace?: string): Promise; diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index f52587779..accbd4afb 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -919,6 +919,25 @@ export class MemWal { return this.waitForRememberJobs(accepted.job_ids, namespaces, opts); } + /** + * Estimate the token cost of a string using the SDK's default + * character-based approximation (~chars/4, code-point aware). Useful to weigh + * a recall payload before injecting it into a model context: + * + * ```ts + * const hits = await memwal.recall({ query, limit: 8 }); + * const joined = hits.results.map((h) => h.text).join("\n---\n"); + * if (memwal.countTokens(joined) > 2048) { /* re-query with maxTokens *\/ } + * ``` + * + * This is an estimate, not an exact tokenizer count — see `estimateTokens` + * for accuracy caveats. For exact counts, pass your own counter to + * `recall({ maxTokens, countTokens })`. + */ + countTokens(text: string): number { + return estimateTokens(text); + } + /** * Recall memories similar to a query — server handles: * verify → embed query → search → Walrus download → decrypt → return plaintext. @@ -957,36 +976,17 @@ export class MemWal { * const result2 = await memwal.recall("food allergies", 5, "profile"); * ``` */ - /** - * Estimate the token cost of a string using the SDK's default - * character-based approximation (~chars/4, code-point aware). Useful to weigh - * a recall payload before injecting it into a model context: - * - * ```ts - * const hits = await memwal.recall({ query, limit: 8 }); - * const joined = hits.results.map((h) => h.text).join("\n---\n"); - * if (memwal.countTokens(joined) > 2048) { /* re-query with maxTokens *\/ } - * ``` - * - * This is an estimate, not an exact tokenizer count — see `estimateTokens` - * for accuracy caveats. For exact counts, pass your own counter to - * `recall({ maxTokens, countTokens })`. - */ - countTokens(text: string): number { - return estimateTokens(text); - } - async recall(params: RecallParams): Promise; /** * @deprecated Positional `recall(query, limit, namespace)` is easy to - * misread as `recall(query, namespace)`. Prefer the object form + * misread as `recall(query, namespace)`, which throws `TypeError`. Prefer * `recall({ query, limit, namespace })`. Positional will be removed in a * future major version of the SDK. */ async recall(query: string, options: RecallOptions): Promise; /** * @deprecated Positional `recall(query, limit, namespace)` is easy to - * misread as `recall(query, namespace)`. Prefer the object form + * misread as `recall(query, namespace)`, which throws `TypeError`. Prefer * `recall({ query, limit, namespace })`. Positional will be removed in a * future major version of the SDK. */ diff --git a/packages/sdk/src/recall-args.ts b/packages/sdk/src/recall-args.ts index 00fc77eef..3518270d1 100644 --- a/packages/sdk/src/recall-args.ts +++ b/packages/sdk/src/recall-args.ts @@ -1,9 +1,6 @@ import type { RecallOptions, RecallParams } from "./types.js"; -/** - * Parse `limit` / options for deprecated positional recall-style calls. - * A string second argument is the `recall(query, namespace)` footgun (#293). - */ +/** String second argument is a namespace, not options. */ export function resolveLimitOrOptions( method: string, limitOrOptions: number | T | undefined | null, @@ -25,11 +22,17 @@ export function resolveLimitOrOptions { const { manual } = manualWithAccountResponse({}); await assert.rejects(() => manual.recallManual("query", "profile"), { name: "TypeError", - message: /recallManual/, + message: + "recallManual() second argument must be a number (limit) or an options object, not a string. " + + "Namespace belongs in an options object (recallManual(query, { namespace })) or as the third argument (recallManual(query, limit, namespace)).", }); }); diff --git a/packages/sdk/test/recall-args.test.mjs b/packages/sdk/test/recall-args.test.mjs index a7973dd1e..7d537800b 100644 --- a/packages/sdk/test/recall-args.test.mjs +++ b/packages/sdk/test/recall-args.test.mjs @@ -21,11 +21,16 @@ test("resolveRecallCall maps object and positional forms", () => { test("resolveRecallCall rejects a namespace string as the second argument", () => { assert.throws(() => resolveRecallCall("food allergies", "profile"), { name: "TypeError", - message: /recall\(\{ query, namespace \}\)/, + message: + "recall() second argument must be a number (limit) or an options object, not a string. " + + "Namespace belongs in an options object (recall(query, { namespace })) or as the third argument (recall(query, limit, namespace)). " + + "Preferred form: recall({ query, namespace }).", }); assert.throws(() => resolveLimitOrOptions("recallManual", "profile"), { name: "TypeError", - message: /recallManual/, + message: + "recallManual() second argument must be a number (limit) or an options object, not a string. " + + "Namespace belongs in an options object (recallManual(query, { namespace })) or as the third argument (recallManual(query, limit, namespace)).", }); assert.throws(() => resolveRecallCall("food", ["profile"]), TypeError); }); From 1c8c15f3f2ea1d569efc29ba4c125af1b48b94c4 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Mon, 12 Oct 2026 15:55:07 +0700 Subject: [PATCH 18/70] fix(sdk): add default condition to exports map and place types first (Fixes #1143, #1040, WALM-721) - Declare "default" fallback condition across all exported subpaths (".", "./account", "./manual", "./ai") so bundlers (Next.js/Turbopack) and Node 22 require(esm) resolve cleanly instead of throwing ERR_PACKAGE_PATH_NOT_EXPORTED - Place "types" condition first for TypeScript nodenext/bundler resolution - Add exports-map regression tests asserting types-first ordering and default fallback --- packages/sdk/package.json | 12 ++++-- packages/sdk/test/exports-map.test.mjs | 57 ++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 packages/sdk/test/exports-map.test.mjs diff --git a/packages/sdk/package.json b/packages/sdk/package.json index e2a6472bb..35ccac9df 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -10,20 +10,24 @@ "types": "./dist/index.d.ts", "exports": { ".": { + "types": "./dist/index.d.ts", "import": "./dist/index.js", - "types": "./dist/index.d.ts" + "default": "./dist/index.js" }, "./account": { + "types": "./dist/account-entry.d.ts", "import": "./dist/account-entry.js", - "types": "./dist/account-entry.d.ts" + "default": "./dist/account-entry.js" }, "./manual": { + "types": "./dist/manual-entry.d.ts", "import": "./dist/manual-entry.js", - "types": "./dist/manual-entry.d.ts" + "default": "./dist/manual-entry.js" }, "./ai": { + "types": "./dist/ai/index.d.ts", "import": "./dist/ai/index.js", - "types": "./dist/ai/index.d.ts" + "default": "./dist/ai/index.js" } }, "scripts": { diff --git a/packages/sdk/test/exports-map.test.mjs b/packages/sdk/test/exports-map.test.mjs new file mode 100644 index 000000000..7a1f4a1e3 --- /dev/null +++ b/packages/sdk/test/exports-map.test.mjs @@ -0,0 +1,57 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const pkgJsonPath = path.join(__dirname, "..", "package.json"); +const pkg = JSON.parse(fs.readFileSync(pkgJsonPath, "utf8")); + +test("exports map declares types first, import, and default fallback (Fixes #1143, WALM-721)", () => { + assert.ok(pkg.exports, "package.json must define exports map"); + + const expectedSubpaths = [".", "./account", "./manual", "./ai"]; + for (const subpath of expectedSubpaths) { + const entry = pkg.exports[subpath]; + assert.ok(entry, `exports["${subpath}"] must be defined`); + + const keys = Object.keys(entry); + assert.equal( + keys[0], + "types", + `exports["${subpath}"] must specify "types" condition first for TypeScript nodenext resolution`, + ); + + assert.ok( + entry.import, + `exports["${subpath}"] must specify "import" condition for ESM imports`, + ); + + assert.ok( + entry.default, + `exports["${subpath}"] must specify "default" condition for bundlers (Next.js/Turbopack) and Node 22 require(esm)`, + ); + + assert.equal( + entry.default, + entry.import, + `exports["${subpath}"].default must match exports["${subpath}"].import`, + ); + } +}); + +test("all exported entry points resolve via ESM import.meta.resolve", async () => { + // Verify that the entrypoints resolve + const rootExport = await import.meta.resolve("@mysten-incubation/memwal", import.meta.url); + assert.ok(rootExport.endsWith("/dist/index.js")); + + const accountExport = await import.meta.resolve("@mysten-incubation/memwal/account", import.meta.url); + assert.ok(accountExport.endsWith("/dist/account-entry.js")); + + const manualExport = await import.meta.resolve("@mysten-incubation/memwal/manual", import.meta.url); + assert.ok(manualExport.endsWith("/dist/manual-entry.js")); + + const aiExport = await import.meta.resolve("@mysten-incubation/memwal/ai", import.meta.url); + assert.ok(aiExport.endsWith("/dist/ai/index.js")); +}); From 3c978cf4d24d35c930ab38e11275e9df100fd7b1 Mon Sep 17 00:00:00 2001 From: ducnmm <165614309+ducnmm@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:59:45 +0700 Subject: [PATCH 19/70] fix(mcp): bind /connect/mcp consent to the local bridge (#368) (#370) * fix(mcp): re-check the bridge before signing connect consent (#368) POST /preflight on load already rejects a phishing link. Two pieces of that fix were still missing: a wallet that connects after the consent screen is showing starts add_delegate_key without a click, and the page does not ask the bridge again immediately before the signature. Gate auto-continue on an explicit click, refuse to sign if the bridge no longer confirms the request, and warn when the relayer is not the default. * test(app): a connected wallet does not sign connect consent on its own The consent screen must stay up until the user clicks, even when a wallet is already connected. --- apps/app/src/pages/ConnectMcp.test.tsx | 16 ++++ apps/app/src/pages/ConnectMcp.tsx | 110 ++++++++++++++++++++----- 2 files changed, 104 insertions(+), 22 deletions(-) diff --git a/apps/app/src/pages/ConnectMcp.test.tsx b/apps/app/src/pages/ConnectMcp.test.tsx index 294631260..42aefa8f5 100644 --- a/apps/app/src/pages/ConnectMcp.test.tsx +++ b/apps/app/src/pages/ConnectMcp.test.tsx @@ -77,6 +77,22 @@ beforeEach(() => { }) describe('MCP sign-in hand-off', () => { + it('does not sign until the user approves, even when a wallet is already connected', async () => { + stubListener(async () => new Response('{}', { status: 200 })) + const query = `?port=${PORT}&publicKey=${PUBLIC_KEY}&relayer=${encodeURIComponent(RELAYER)}&connectState=${STATE}` + render( + + + , + ) + + const approve = await screen.findByRole('button', { name: /approve in wallet/i }) + expect(mocks.signAndExecute).not.toHaveBeenCalled() + await userEvent.click(approve) + expect(await screen.findByText(/MCP client connected/i)).toBeInTheDocument() + expect(mocks.signAndExecute).toHaveBeenCalledTimes(1) + }) + it('confirms the connection when the listener accepts the callback', async () => { stubListener(async () => new Response('{}', { status: 200 })) await signIn() diff --git a/apps/app/src/pages/ConnectMcp.tsx b/apps/app/src/pages/ConnectMcp.tsx index f1db1a695..4210d0814 100644 --- a/apps/app/src/pages/ConnectMcp.tsx +++ b/apps/app/src/pages/ConnectMcp.tsx @@ -14,9 +14,10 @@ * 1. Verify the exact state/public-key tuple against the localhost MCP * bridge. A copied/emailed URL has no matching bridge and fails closed. * 2. Render consent screen — show requested permissions + the full delegate - * public key/address returned by the verified bridge. - * 3. User clicks "Connect Sui Wallet" → standard dApp Kit wallet popup. - * 4. Build + sign `add_delegate_key(account, registry, publicKey, label, clock)` + * public key/address returned by the verified bridge. A wallet that is + * already connected does not skip this screen. + * 3. User clicks through consent → standard dApp Kit wallet popup if needed. + * 4. Re-check the local bridge, then sign `add_delegate_key` * via useSponsoredTransaction (matches SetupWizard pattern). * 5. POST result {accountId, walletAddress, packageId, txDigest, label} * to http://localhost:/callback — the MCP package's listener. @@ -136,6 +137,34 @@ function isVerifiedBridge( ) } +function sameRelayer(a: string, b: string): boolean { + const norm = (value: string) => value.trim().replace(/\/+$/, '').toLowerCase() + return norm(a) === norm(b) +} + +/** Ask the local bridge again. Network and abort errors propagate to the caller. */ +async function confirmLocalBridge( + port: string, + state: string, + publicKey: string, + relayer: string, + signal?: AbortSignal, +): Promise { + const response = await fetch(`http://127.0.0.1:${port}/preflight`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ state, publicKey, relayer }), + signal, + }) + const body: unknown = await response.json().catch(() => null) + if (!response.ok || !isVerifiedBridge(body, publicKey, relayer)) return null + return { + publicKey: body.publicKey, + label: body.label, + relayer: body.relayer, + } +} + export default function ConnectMcp() { const [params] = useSearchParams() const currentAccount = useCurrentAccount() @@ -171,6 +200,9 @@ export default function ConnectMcp() { const [verifiedBridge, setVerifiedBridge] = useState(null) const [preflightAttempt, setPreflightAttempt] = useState(0) const invalidRequestTrackedRef = useRef(false) + // Set only after the user clicks consent. A wallet that hydrates on its + // own must not start add_delegate_key (#368). + const initiatedRef = useRef(false) // Validate query string up-front. const paramsValid = useMemo(() => { @@ -208,21 +240,17 @@ export default function ConnectMcp() { void (async () => { try { - const response = await fetch(`http://127.0.0.1:${port}/preflight`, { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ state, publicKey, relayer }), - signal: controller.signal, - }) - const body: unknown = await response.json().catch(() => null) - if (!response.ok || !isVerifiedBridge(body, publicKey, relayer)) { + const confirmed = await confirmLocalBridge( + port, + state, + publicKey, + relayer, + controller.signal, + ) + if (!confirmed) { throw new Error('The local MCP bridge did not verify this connection request.') } - setVerifiedBridge({ - publicKey: body.publicKey, - label: body.label, - relayer: body.relayer, - }) + setVerifiedBridge(confirmed) setStep('consent') } catch (error) { if (controller.signal.aborted) return @@ -262,6 +290,7 @@ export default function ConnectMcp() { ) const handleConnect = useCallback(async () => { + initiatedRef.current = true if (!paramsValid || !verifiedBridge) { trackEvent('mcp_connect_failed', { error_type: 'invalid_request' }) setErrorMsg('This request was not verified by the local MCP client.') @@ -275,6 +304,28 @@ export default function ConnectMcp() { } trackEvent('mcp_connect_start', { wallet_connected: true }) + let bridge: VerifiedBridge + try { + const confirmed = await confirmLocalBridge(port, state, publicKey, relayer) + if (!confirmed) { + throw new Error('The local MCP bridge did not verify this connection request.') + } + bridge = confirmed + } catch (error) { + if (error instanceof DOMException && error.name === 'AbortError') return + trackEvent('mcp_connect_failed', { error_type: 'bridge_preflight_failed' }) + setVerifiedBridge(null) + setErrorMsg( + error instanceof TypeError + ? 'Nothing answered on your computer. Sign-in links stop working after five minutes, so most often the link was simply opened too late.' + : error instanceof Error + ? error.message + : 'Could not verify the local MCP bridge.', + ) + setStep('error') + return + } + setVerifiedBridge(bridge) setStep('signing') try { // Resolve the user's Walrus Memory account object. @@ -292,9 +343,9 @@ export default function ConnectMcp() { arguments: [ tx.object(accountId), tx.object(config.memwalRegistryId), - tx.pure('vector', hexToBytes(verifiedBridge.publicKey)), + tx.pure('vector', hexToBytes(bridge.publicKey)), // v1_new derives the Sui address on-chain — no address arg. - tx.pure('string', verifiedBridge.label), + tx.pure('string', bridge.label), tx.object('0x6'), ], }) @@ -330,7 +381,7 @@ export default function ConnectMcp() { walletAddress: currentAccount.address, packageId: config.memwalPackageId, txDigest: result.digest, - label: verifiedBridge.label, + label: bridge.label, state, } setCallbackPayload(payload) @@ -352,6 +403,9 @@ export default function ConnectMcp() { suiClient, signAndExecute, verifiedBridge, + port, + publicKey, + relayer, state, postCallback, ]) @@ -376,11 +430,14 @@ export default function ConnectMcp() { ) }, [devMock, paramsValid, port, publicKey, delegateAddress, relayer, state]) - // If the wallet popup completes after we asked it to open, auto-proceed. + // If the wallet popup completes after the user clicked consent, continue. + // A wallet that was already connected, or that hydrates on its own, must + // not skip the disclosure (#368). useEffect(() => { if (devMock) return - if (!walletPickerOpen && currentAccount && step === 'consent') { - // user picked a wallet — kick off the connect flow. + // A wallet that was already connected, or that hydrates on its own, + // must not skip the disclosure (#368). + if (initiatedRef.current && !walletPickerOpen && currentAccount && step === 'consent') { void handleConnect() } // we only want this to fire on wallet→connected transition. @@ -405,6 +462,7 @@ export default function ConnectMcp() { publicKey={'11'.repeat(32)} delegateAddress="0x7c30e2a1b4c89f01d2e3a4b5c6d7e8f90123456789abcdef0123456789abcd" relayer={config.memwalServerUrl} + relayerIsDefault wallet={currentAccount?.address ?? null} onConnect={() => undefined} /> @@ -441,6 +499,7 @@ export default function ConnectMcp() { publicKey={verifiedBridge.publicKey} delegateAddress={delegateAddress} relayer={verifiedBridge.relayer} + relayerIsDefault={sameRelayer(verifiedBridge.relayer, config.memwalServerUrl)} wallet={currentAccount?.address ?? null} onConnect={handleConnect} /> @@ -527,6 +586,7 @@ function ConsentCard({ publicKey, delegateAddress, relayer, + relayerIsDefault, wallet, onConnect, }: { @@ -534,6 +594,7 @@ function ConsentCard({ publicKey: string delegateAddress: string relayer: string + relayerIsDefault: boolean wallet: string | null onConnect: () => void }) { @@ -564,6 +625,11 @@ function ConsentCard({
Relayer {relayer} + {!relayerIsDefault && ( + + Non-default relayer. Memories would be served through this host. Continue only if you chose it. + + )}
Delegate public key From fb00610261f99c73ef048aeddb5457adb730a670 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Mon, 12 Oct 2026 16:06:43 +0700 Subject: [PATCH 20/70] fix(sdk): retain lowest-distance hits under high-relevance-only token budget (Fixes #1103, WALM-756) - In applyTokenBudget, select candidates by ascending distance under high-relevance-only strategy rather than array order - Preserve the caller's requested order (e.g. sort: recent) for the retained lowest-distance hits - Break distance ties stably by original index - Update TruncationStrategy documentation and recall client commentary - Add regression test reproducing the two-hit recency order bug and multi-hit order preservation --- packages/sdk/src/memwal.ts | 10 ++-- packages/sdk/src/tokens.ts | 40 +++++++++----- .../recall-token-budget-relevance.test.mjs | 53 +++++++++++++++++++ 3 files changed, 87 insertions(+), 16 deletions(-) create mode 100644 packages/sdk/test/recall-token-budget-relevance.test.mjs diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index 8954dde77..9dbd36a04 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -1041,10 +1041,12 @@ export class MemWal { }; } - // Client-side token budgeting: trim the (already distance-sorted) - // payload to fit maxTokens per the chosen strategy, and attach the - // token estimate + truncated flag. Omitting maxTokens leaves the - // result byte-identical to the pre-budget behavior (no meta). + // Client-side token budgeting: trim the payload to fit maxTokens per + // the chosen strategy, and attach the token estimate + truncated flag. + // Under high-relevance-only, lowest-distance hits are retained and + // returned in the caller's requested order (WALM-756). + // Omitting maxTokens leaves the result byte-identical to the pre-budget + // behavior (no meta). if (typeof options.maxTokens === "number") { const { results: budgeted, meta } = applyTokenBudget( processed.results, diff --git a/packages/sdk/src/tokens.ts b/packages/sdk/src/tokens.ts index 9de4b77e6..abff7fe69 100644 --- a/packages/sdk/src/tokens.ts +++ b/packages/sdk/src/tokens.ts @@ -108,10 +108,11 @@ function payloadTokens(results: RecallMemory[], countTokens: TokenCounter): numb * claim it truncated. * * Strategies: - * - `high-relevance-only`: keep WHOLE hits from the front while they fit; the - * first hit that would overflow (and everything after it) is dropped. A - * single leading hit larger than the whole budget is itself dropped. No hit - * is ever partially shown. Deterministic. + * - `high-relevance-only`: keep WHOLE hits in order of relevance (lowest distance + * first) while they fit within the budget; the first hit that would overflow + * (and everything after it) is dropped. Retained hits are returned in the + * caller's requested order. A single most-relevant hit larger than the whole + * budget is itself dropped. No hit is ever partially shown. Deterministic. * - `drop-tail`: preserve order, keep whole hits from the front, then TRUNCATE * the single boundary hit that straddles the budget so the payload fills to * (about) `maxTokens` — modelling "cut the end of the concatenated payload". @@ -188,15 +189,30 @@ export function applyTokenBudget( return { results: kept, meta: { tokenEstimate: running, truncated: true } }; } - // high-relevance-only: keep whole hits from the front while the running total - // stays within budget; stop at the first hit that would overflow. - const kept: RecallMemory[] = []; + // high-relevance-only: select whole hits in order of highest relevance (lowest + // distance) while they fit in the budget, then preserve the caller's original + // returned order for the retained set (WALM-756). + const indexed = results.map((m, index) => ({ + memory: m, + index, + cost: countTokens(m.text), + })); + + const byRelevance = [...indexed].sort((a, b) => { + const distA = a.memory.distance ?? 0; + const distB = b.memory.distance ?? 0; + if (distA !== distB) return distA - distB; + return a.index - b.index; + }); + + const selectedIndices = new Set(); let running = 0; - for (const m of results) { - const cost = countTokens(m.text); - if (running + cost > maxTokens) break; - kept.push(m); - running += cost; + for (const item of byRelevance) { + if (running + item.cost > maxTokens) break; + selectedIndices.add(item.index); + running += item.cost; } + + const kept = results.filter((_, idx) => selectedIndices.has(idx)); return { results: kept, meta: { tokenEstimate: running, truncated: true } }; } diff --git a/packages/sdk/test/recall-token-budget-relevance.test.mjs b/packages/sdk/test/recall-token-budget-relevance.test.mjs new file mode 100644 index 000000000..0962ae293 --- /dev/null +++ b/packages/sdk/test/recall-token-budget-relevance.test.mjs @@ -0,0 +1,53 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { applyTokenBudget } from "../dist/tokens.js"; + +test("WALM-756 / GH #1103 repro: recall token budget under high-relevance-only keeps lowest-distance hits", () => { + // What recall({ query, sort: "recent", maxTokens }) receives from the relayer: newest first. + const recentOrder = [ + { blob_id: "newest", text: "User mentioned the weather is nice today.", distance: 0.62 }, + { blob_id: "older", text: "User is allergic to peanuts.", distance: 0.08 }, + ]; + // With maxTokens=12, each fact is ~10-11 tokens, so only 1 fact can fit. + const result = applyTokenBudget(recentOrder, 12, "high-relevance-only"); + assert.deepEqual( + result.results.map((r) => r.blob_id), + ["older"], + "Expected lowest-distance hit ('older', distance: 0.08) to be kept", + ); + assert.equal(result.meta.truncated, true); +}); + +test("WALM-756 / GH #1103: retained hits preserve caller's requested order", () => { + // 4 hits in recency order (newest to oldest): + // hit1: distance 0.40, ~4 tokens + // hit2: distance 0.10, ~4 tokens + // hit3: distance 0.50, ~4 tokens + // hit4: distance 0.15, ~4 tokens + const recentOrder = [ + { blob_id: "h1", text: "fact 1 text.", distance: 0.40 }, + { blob_id: "h2", text: "fact 2 text.", distance: 0.10 }, + { blob_id: "h3", text: "fact 3 text.", distance: 0.50 }, + { blob_id: "h4", text: "fact 4 text.", distance: 0.15 }, + ]; + // Budget fits 2 hits (~8 tokens): + // By distance: h2 (0.10), h4 (0.15), h1 (0.40), h3 (0.50) + // Lowest distance that fit: h2 and h4. + // In caller's recency order: [h2, h4]. + const result = applyTokenBudget(recentOrder, 8, "high-relevance-only"); + assert.deepEqual( + result.results.map((r) => r.blob_id), + ["h2", "h4"], + ); + assert.equal(result.meta.truncated, true); +}); + +test("WALM-756 / GH #1103: equal distance breaks ties by original index", () => { + const hits = [ + { blob_id: "tie-a", text: "text a.", distance: 0.1 }, + { blob_id: "tie-b", text: "text b.", distance: 0.1 }, + ]; + // Budget fits only 1: tie-a should win + const result = applyTokenBudget(hits, 3, "high-relevance-only"); + assert.deepEqual(result.results.map((r) => r.blob_id), ["tie-a"]); +}); From 988b8c78969b478deed6599e5eeccc5b96a763b0 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Mon, 12 Oct 2026 18:03:03 +0700 Subject: [PATCH 21/70] fix(sdk): clamp polling delay to deadline and ensure boundary poll check (#1081) - In waitForRememberJob and waitForRememberJobs, clamp the polling sleep delay to Math.max(0, deadline - Date.now()) so the client never sleeps past the wait budget. - Refactor the polling loop so that if the remaining budget expires during sleep, a boundary poll is performed rather than exiting immediately without a status check. - Update pollDeadlineMs to supply a usable minimum timeout (1000ms, clamped to requestTimeoutMs) at or past the boundary so the final check does not immediately abort with a 1ms timeout. - Add regression tests verifying that a job settling before the deadline resolves instead of timing out, and that wait loops do not sleep past the deadline. --- packages/sdk/src/memwal.ts | 45 +++++- .../sdk/test/wait-deadline-clamp.test.mjs | 133 ++++++++++++++++++ 2 files changed, 172 insertions(+), 6 deletions(-) create mode 100644 packages/sdk/test/wait-deadline-clamp.test.mjs diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index 594eb3ce8..4bcca284f 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -590,11 +590,17 @@ export class MemWal { let lastSeen: RememberJobStatus | undefined; let lastRefusal: number | undefined; - while (Date.now() < deadline) { + while (true) { // A retry-after the server just gave us wins over our own curve; // the backoff resumes from where it was on the next normal poll. - await sleep(retryAfterMs > 0 ? retryAfterMs : pollingDelayMs(pollIntervalMs, attempt++)); + // Clamped to remaining budget so we never sleep past deadline (#1081). + const remainingBeforeSleep = Math.max(0, deadline - Date.now()); + const requestedDelay = retryAfterMs > 0 ? retryAfterMs : pollingDelayMs(pollIntervalMs, attempt++); retryAfterMs = 0; + const delay = Math.min(requestedDelay, remainingBeforeSleep); + if (delay > 0) { + await sleep(delay); + } let status: RememberStatusResponse; @@ -621,6 +627,9 @@ export class MemWal { if (isTransientPollingStatus(httpStatus)) { lastRefusal = httpStatus; if (httpStatus === 429) sawRateLimit = true; + if (Date.now() >= deadline) { + break; + } retryAfterMs = retryAfterDelayMs(err, deadline); continue; } @@ -652,6 +661,10 @@ export class MemWal { { status: 500, jobId }, ); } + + if (Date.now() >= deadline) { + break; + } } throw rememberJobTimeoutError(jobId, timeoutMs, lastSeen, lastRefusal, sawRateLimit); @@ -835,11 +848,17 @@ export class MemWal { if (terminal) pending.delete(jobId); }; - while (pending.size > 0 && Date.now() < deadline) { + while (pending.size > 0) { // A retry-after the server just gave us wins over our own curve; // the backoff resumes from where it was on the next normal poll. - await sleep(retryAfterMs > 0 ? retryAfterMs : pollingDelayMs(pollIntervalMs, attempt++)); + // Clamped to remaining budget so we never sleep past deadline (#1081). + const remainingBeforeSleep = Math.max(0, deadline - Date.now()); + const requestedDelay = retryAfterMs > 0 ? retryAfterMs : pollingDelayMs(pollIntervalMs, attempt++); retryAfterMs = 0; + const delay = Math.min(requestedDelay, remainingBeforeSleep); + if (delay > 0) { + await sleep(delay); + } const pendingIds = jobIds.filter((jobId) => pending.has(jobId)); if (pendingIds.length === 0) { @@ -858,6 +877,9 @@ export class MemWal { if (isTransientPollingStatus(httpStatus)) { lastRefusal = httpStatus; if (httpStatus === 429) sawRateLimit = true; + if (Date.now() >= deadline) { + break; + } retryAfterMs = retryAfterDelayMs(err, deadline); continue; } @@ -883,6 +905,10 @@ export class MemWal { settle(jobId, status); } + + if (pending.size === 0 || Date.now() >= deadline) { + break; + } } // Nothing settled and nothing was even seen: the polls may all have been @@ -1784,9 +1810,16 @@ export class MemWal { * one stalled socket there hangs every method on the client, not just one. */ /** Deadline for one status poll: the client deadline, clamped so it can - * never outlive the wait budget it belongs to. */ + * never outlive the wait budget it belongs to, but given a usable minimum + * at or past the deadline so a boundary check does not fail with an + * artificial 1ms AbortError (#1081). */ private pollDeadlineMs(deadline: number): number { - return Math.max(1, Math.min(this.requestTimeoutMs, deadline - Date.now())); + const remaining = deadline - Date.now(); + const minPollTimeout = Math.min(this.requestTimeoutMs, 1_000); + if (remaining <= 0) { + return minPollTimeout; + } + return Math.min(this.requestTimeoutMs, Math.max(remaining, minPollTimeout)); } private async fetchWithDeadline( diff --git a/packages/sdk/test/wait-deadline-clamp.test.mjs b/packages/sdk/test/wait-deadline-clamp.test.mjs new file mode 100644 index 000000000..9c994429e --- /dev/null +++ b/packages/sdk/test/wait-deadline-clamp.test.mjs @@ -0,0 +1,133 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { MemWal } from "../dist/memwal.js"; + +const originalFetch = globalThis.fetch; + +test.afterEach(() => { + globalThis.fetch = originalFetch; +}); + +function client() { + const memwal = MemWal.create({ + key: new Uint8Array(32).fill(1), + accountId: "0x1", + serverUrl: "https://relayer.example", + }); + memwal.buildSealSession = async () => "test-session"; + return memwal; +} + +function stub(pathHandler) { + globalThis.fetch = async (url, init = {}) => { + const path = new URL(url).pathname; + if (path === "/version") { + return Response.json({ + apiVersion: "1.0.0", + relayerVersion: "1.0.0", + minSupportedSdk: { typescript: "0.0.4" }, + }); + } + if (path === "/api/config") { + return Response.json({ packageId: "0x1", network: "testnet" }); + } + return pathHandler(path, init); + }; +} + +test("waitForRememberJob clamps sleep to deadline and resolves when job finishes before budget (#1081)", async () => { + const start = Date.now(); + const doneAfterMs = 250; + let pollCount = 0; + + stub((path) => { + if (path === "/api/remember/job-1") { + pollCount++; + const elapsed = Date.now() - start; + if (elapsed >= doneAfterMs) { + return Response.json({ + job_id: "job-1", + status: "done", + blob_id: "blob-1081", + owner: "0x1", + namespace: "default", + }); + } + return Response.json({ + job_id: "job-1", + status: "running", + }); + } + throw new Error(`unexpected request ${path}`); + }); + + // pollIntervalMs is larger than timeoutMs, so unclamped backoff would sleep past the deadline + const result = await client().waitForRememberJob("job-1", { + pollIntervalMs: 800, + timeoutMs: 400, + }); + + assert.equal(result.blob_id, "blob-1081"); + assert.ok(pollCount >= 2, "must poll at t=0 and at/near boundary"); + const totalElapsed = Date.now() - start; + assert.ok(totalElapsed < 1500, `wait overran budget: ${totalElapsed}ms`); +}); + +test("waitForRememberJob does not sleep past deadline on perpetual running job (#1081)", async () => { + const start = Date.now(); + + stub((path) => { + if (path === "/api/remember/job-running") { + return Response.json({ + job_id: "job-running", + status: "running", + }); + } + throw new Error(`unexpected request ${path}`); + }); + + // pollIntervalMs (2000ms) is much larger than timeoutMs (300ms) + await assert.rejects( + client().waitForRememberJob("job-running", { + pollIntervalMs: 2000, + timeoutMs: 300, + }), + /remember job timed out after 300ms \(job_id=job-running\)/, + ); + + const totalElapsed = Date.now() - start; + assert.ok(totalElapsed < 1200, `wait slept past deadline: ${totalElapsed}ms`); +}); + +test("waitForRememberJobs bulk clamps sleep to deadline and resolves job finishing before budget (#1081)", async () => { + const start = Date.now(); + const doneAfterMs = 250; + + stub((path) => { + if (path === "/api/remember/bulk/status") { + const elapsed = Date.now() - start; + const status = elapsed >= doneAfterMs ? "done" : "running"; + return Response.json({ + results: [ + { + job_id: "job-bulk-1", + status, + blob_id: status === "done" ? "blob-bulk-1" : undefined, + }, + ], + }); + } + throw new Error(`unexpected request ${path}`); + }); + + const settled = await client().waitForRememberJobs(["job-bulk-1"], ["default"], { + pollIntervalMs: 800, + timeoutMs: 400, + }); + + assert.equal(settled.results[0].status, "done"); + assert.equal(settled.results[0].blob_id, "blob-bulk-1"); + const totalElapsed = Date.now() - start; + assert.ok(totalElapsed < 1500, `bulk wait overran budget: ${totalElapsed}ms`); +}); From 0df5033032b364fb3c8f4bb6d78426621cc6a179 Mon Sep 17 00:00:00 2001 From: Harry Phan Date: Mon, 12 Oct 2026 15:28:03 +0700 Subject: [PATCH 22/70] fix(server,sdk): scope remember idempotency keys per namespace (WALM-781) - Scope unique partial index on remember_jobs to (owner, namespace, idempotency_key) - Add migration 025 to drop legacy uq_remember_jobs_owner_idempotency_key index - Update conflict handling and idempotency lookups in server routes/remember.rs and routes/analyze.rs - Scope TypeScript MemWalMock idempotency map to namespace + key and document in JSDoc - Add tests covering cross-namespace key re-use and intra-namespace deduplication Fixes WALM-781, #1170 --- packages/sdk/src/memwal.ts | 7 + packages/sdk/src/mock.ts | 9 +- packages/sdk/test/mock.test.mjs | 24 +++- .../013_remember_write_idempotency_index.sql | 4 +- .../025_drop_owner_idempotency_key_index.sql | 7 + services/server/src/routes/analyze.rs | 10 +- services/server/src/routes/remember.rs | 127 ++++++++++++++---- services/server/src/storage/db.rs | 8 +- 8 files changed, 155 insertions(+), 41 deletions(-) create mode 100644 services/server/migrations/025_drop_owner_idempotency_key_index.sql diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index 7a85b1fe6..074c63a68 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -165,6 +165,13 @@ const CONFIRM_READ_TIMEOUT_MS = 5_000; const MAX_BULK_ITEMS = 20; interface RememberCallOptions { + /** + * Optional idempotency key to prevent duplicate writes on retries. + * Idempotency keys are scoped per (owner, namespace), so identical + * keys across different namespaces will not collide. Retrying within + * the same namespace with the same key dedupes, while reusing a key + * with different text in the same namespace produces HTTP 409 Conflict. + */ idempotencyKey?: string; pollIntervalMs?: number; timeoutMs?: number; diff --git a/packages/sdk/src/mock.ts b/packages/sdk/src/mock.ts index ae1c9d43d..6eaf266dd 100644 --- a/packages/sdk/src/mock.ts +++ b/packages/sdk/src/mock.ts @@ -582,7 +582,8 @@ export class MemWalMock { const resolvedNamespace = namespace ?? this.namespace; if (!resolvedNamespace) throw new Error("namespace cannot be empty"); if (idempotencyKey !== undefined) { - const existingId = this.idempotency.get(idempotencyKey); + const mapKey = `${resolvedNamespace}\0${idempotencyKey}`; + const existingId = this.idempotency.get(mapKey); const existing = existingId ? this.jobs.get(existingId) : undefined; if (existing) { if (existing.text !== text || existing.namespace !== resolvedNamespace) { @@ -592,7 +593,7 @@ export class MemWalMock { } return existing; } - if (existingId) this.idempotency.delete(idempotencyKey); + if (existingId) this.idempotency.delete(mapKey); } this.sequence += 1; const suffix = this.sequence.toString().padStart(6, "0"); @@ -606,7 +607,9 @@ export class MemWalMock { }; this.memories.push(memory); this.jobs.set(memory.jobId, memory); - if (idempotencyKey !== undefined) this.idempotency.set(idempotencyKey, memory.jobId); + if (idempotencyKey !== undefined) { + this.idempotency.set(`${resolvedNamespace}\0${idempotencyKey}`, memory.jobId); + } return memory; } diff --git a/packages/sdk/test/mock.test.mjs b/packages/sdk/test/mock.test.mjs index 194051cf0..df8ec9875 100644 --- a/packages/sdk/test/mock.test.mjs +++ b/packages/sdk/test/mock.test.mjs @@ -371,13 +371,27 @@ test("MemWalMock collapses a repeated idempotency key onto the original job", as (await mock.recall({ query: "other", namespace: "ns", maxDistance: 0.5 })).total, 0, ); - await assert.rejects( - () => mock.rememberAsync("dedupe me", "other-ns", { idempotencyKey: "key-123" }), - /different content/, - ); + // WALM-781: Idempotency keys are scoped per namespace. + // The same key used in another namespace must succeed and not collide. + const otherNs = await mock.rememberAsync("dedupe me", "other-ns", { + idempotencyKey: "key-123", + }); + assert.notEqual(first.job_id, otherNs.job_id); assert.equal( (await mock.recall({ query: "dedupe", namespace: "other-ns", maxDistance: 0.5 })).total, - 0, + 1, + ); + + // Retrying with the same key in other-ns collapses. + const otherNsRetry = await mock.rememberAsync("dedupe me", "other-ns", { + idempotencyKey: "key-123", + }); + assert.equal(otherNs.job_id, otherNsRetry.job_id); + + // Reusing the key with different content in other-ns rejects. + await assert.rejects( + () => mock.rememberAsync("different content text", "other-ns", { idempotencyKey: "key-123" }), + /different content/, ); const viaRemember = await mock.remember("via remember", "ns", { diff --git a/services/server/migrations/013_remember_write_idempotency_index.sql b/services/server/migrations/013_remember_write_idempotency_index.sql index 7da5a5033..cc7bfbfee 100644 --- a/services/server/migrations/013_remember_write_idempotency_index.sql +++ b/services/server/migrations/013_remember_write_idempotency_index.sql @@ -1,4 +1,4 @@ -- Keep this standalone: CREATE INDEX CONCURRENTLY cannot run in a transaction. -CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS uq_remember_jobs_owner_idempotency_key - ON remember_jobs (owner, idempotency_key) +CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS uq_remember_jobs_owner_namespace_idempotency_key + ON remember_jobs (owner, namespace, idempotency_key) WHERE idempotency_key IS NOT NULL; diff --git a/services/server/migrations/025_drop_owner_idempotency_key_index.sql b/services/server/migrations/025_drop_owner_idempotency_key_index.sql new file mode 100644 index 000000000..c01d37237 --- /dev/null +++ b/services/server/migrations/025_drop_owner_idempotency_key_index.sql @@ -0,0 +1,7 @@ +-- Drop the legacy account-wide idempotency index (WALM-781). +-- Idempotency keys are now scoped per (owner, namespace) by +-- uq_remember_jobs_owner_namespace_idempotency_key (migration 013). +-- +-- Own file: DROP INDEX CONCURRENTLY cannot run inside a transaction block +-- and avoids taking ACCESS EXCLUSIVE lock on remember_jobs at boot. +DROP INDEX CONCURRENTLY IF EXISTS uq_remember_jobs_owner_idempotency_key; diff --git a/services/server/src/routes/analyze.rs b/services/server/src/routes/analyze.rs index 10fe4aef3..f5dc73e59 100644 --- a/services/server/src/routes/analyze.rs +++ b/services/server/src/routes/analyze.rs @@ -648,7 +648,7 @@ pub async fn analyze( let inserted = match sqlx::query( "INSERT INTO remember_jobs (id, owner, namespace, status, idempotency_key, request_fingerprint) VALUES ($1, $2, $3, 'pending', $4, $4) - ON CONFLICT (owner, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING", + ON CONFLICT (owner, namespace, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING", ) .bind(&job_id) .bind(owner) @@ -670,9 +670,10 @@ pub async fn analyze( if inserted.rows_affected() == 0 { rate_limit::release_storage_quota(&state, std::slice::from_ref(&job_id)).await; let existing = sqlx::query_as::<_, (String, String, Option)>( - "SELECT id, status, blob_id FROM remember_jobs WHERE owner = $1 AND idempotency_key = $2", + "SELECT id, status, blob_id FROM remember_jobs WHERE owner = $1 AND namespace = $2 AND idempotency_key = $3", ) .bind(owner) + .bind(namespace) .bind(&idem_key) .fetch_optional(state.db.pool()) .await; @@ -777,7 +778,7 @@ pub async fn analyze( let rewritten = match sqlx::query( "INSERT INTO remember_jobs (id, owner, namespace, status, idempotency_key, request_fingerprint) VALUES ($1, $2, $3, 'pending', $4, $4) - ON CONFLICT (owner, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING", + ON CONFLICT (owner, namespace, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING", ) .bind(&job_id) .bind(owner) @@ -799,9 +800,10 @@ pub async fn analyze( if rewritten.rows_affected() == 0 { // Another retry won the key after we released it. let winner = sqlx::query_scalar::<_, String>( - "SELECT id FROM remember_jobs WHERE owner = $1 AND idempotency_key = $2", + "SELECT id FROM remember_jobs WHERE owner = $1 AND namespace = $2 AND idempotency_key = $3", ) .bind(owner) + .bind(namespace) .bind(&idem_key) .fetch_optional(state.db.pool()) .await; diff --git a/services/server/src/routes/remember.rs b/services/server/src/routes/remember.rs index 3d2ca9e17..1db50ba0a 100644 --- a/services/server/src/routes/remember.rs +++ b/services/server/src/routes/remember.rs @@ -777,15 +777,15 @@ pub async fn remember( let namespace_owned = namespace.clone(); let text = body.text; - // Idempotent retry: if this owner already has a job under the same key, - // return it instead of minting a new (paid) write. Insert-then-check under - // the partial unique index (owner, idempotency_key) so two concurrent - // requests with the same key can't both create a job — the loser's insert - // conflicts and it reads back the winner's row. + // Idempotent retry: if this owner already has a job under the same key in this + // namespace, return it instead of minting a new (paid) write. Insert-then-check + // under the partial unique index (owner, namespace, idempotency_key) so two + // concurrent requests with the same key can't both create a job — the loser's + // insert conflicts and it reads back the winner's row. let fingerprint = request_fingerprint(&text, namespace); if let Some(ref key) = body.idempotency_key { if let Some((existing_id, existing_status, existing_blob_id, existing_fingerprint)) = - find_remember_job_by_key(state.db.pool(), owner, key).await? + find_remember_job_by_key(state.db.pool(), owner, namespace, key).await? { // Same key, different content → the caller reused an idempotency key // for a different write. Reject rather than silently collapsing onto @@ -825,7 +825,7 @@ pub async fn remember( // "running" for API compatibility — see below.) let inserted = match sqlx::query( "INSERT INTO remember_jobs (id, owner, namespace, status, idempotency_key, request_fingerprint) VALUES ($1, $2, $3, 'pending', $4, $5) - ON CONFLICT (owner, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING", + ON CONFLICT (owner, namespace, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING", ) .bind(&job_id) .bind(owner) @@ -852,7 +852,7 @@ pub async fn remember( if inserted.rows_affected() == 0 { if let Some(key) = body.idempotency_key.as_deref() { if let Some((existing_id, existing_status, existing_blob_id, existing_fingerprint)) = - find_remember_job_by_key(state.db.pool(), owner, key).await? + find_remember_job_by_key(state.db.pool(), owner, namespace, key).await? { // Lost the race to a concurrent same-key request with DIFFERENT // content → reject, same as the pre-lookup path. @@ -1280,18 +1280,20 @@ fn request_fingerprint(text: &str, namespace: &str) -> String { format!("{:x}", hasher.finalize()) } -/// Look up an existing remember job for `(owner, idempotency_key)`. Returns +/// Look up an existing remember job for `(owner, namespace, idempotency_key)`. Returns /// `(job_id, status, blob_id, request_fingerprint)` when one exists, so a retry /// can collapse onto it — or be rejected if its content fingerprint differs. async fn find_remember_job_by_key( pool: &sqlx::PgPool, owner: &str, + namespace: &str, key: &str, ) -> Result, Option)>, AppError> { sqlx::query_as::<_, (String, String, Option, Option)>( - "SELECT id, status, blob_id, request_fingerprint FROM remember_jobs WHERE owner = $1 AND idempotency_key = $2", + "SELECT id, status, blob_id, request_fingerprint FROM remember_jobs WHERE owner = $1 AND namespace = $2 AND idempotency_key = $3", ) .bind(owner) + .bind(namespace) .bind(key) .fetch_optional(pool) .await @@ -1414,7 +1416,7 @@ pub async fn remember_bulk( // path; only a retry of an in-flight job (worker-set `running`) // triggers the crash-window reconcile. See the single-remember insert. "INSERT INTO remember_jobs (id, owner, namespace, status, idempotency_key, request_fingerprint) VALUES ($1, $2, $3, 'pending', $4, $5) - ON CONFLICT (owner, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING", + ON CONFLICT (owner, namespace, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING", ) .bind(&job_id) .bind(owner) @@ -1441,7 +1443,7 @@ pub async fn remember_bulk( if inserted.rows_affected() == 0 { let existing = match item_key.as_deref() { - Some(key) => find_remember_job_by_key(state.db.pool(), owner, key).await?, + Some(key) => find_remember_job_by_key(state.db.pool(), owner, &item.namespace, key).await?, None => None, }; let (existing_id, _, _, existing_fingerprint) = existing.ok_or_else(|| { @@ -2004,7 +2006,7 @@ mod tests { .await .unwrap(); - let found = find_remember_job_by_key(&pool, &owner, key) + let found = find_remember_job_by_key(&pool, &owner, "ns", key) .await .unwrap() .unwrap(); @@ -2057,6 +2059,9 @@ mod tests { assert!(validate_idempotency_key(Some(&ok)).is_ok()); } + static IDEM_SCHEMA_SETUP_LOCK: std::sync::OnceLock> = + std::sync::OnceLock::new(); + async fn idem_test_pool() -> sqlx::PgPool { let url = std::env::var("DATABASE_URL") .unwrap_or_else(|_| "postgresql://memwal:memwal_secret@localhost:5432/memwal".into()); @@ -2066,6 +2071,10 @@ mod tests { .connect(&url) .await .unwrap(); + let _guard = IDEM_SCHEMA_SETUP_LOCK + .get_or_init(|| tokio::sync::Mutex::new(())) + .lock() + .await; sqlx::raw_sql(include_str!("../../migrations/005_remember_jobs.sql")) .execute(&pool) .await @@ -2093,7 +2102,7 @@ mod tests { let job_id = format!("remember-job-{}", uuid::Uuid::new_v4()); // No job yet for this key. - assert!(find_remember_job_by_key(&pool, &owner, key) + assert!(find_remember_job_by_key(&pool, &owner, "ns", key) .await .unwrap() .is_none()); @@ -2110,7 +2119,7 @@ mod tests { .unwrap(); // Lookup now finds it (this is what a retry hits → collapse, no new write). - let found = find_remember_job_by_key(&pool, &owner, key).await.unwrap(); + let found = find_remember_job_by_key(&pool, &owner, "ns", key).await.unwrap(); assert_eq!( found, Some((job_id.clone(), "running".to_string(), None, None)) @@ -2119,7 +2128,7 @@ mod tests { // A concurrent same-(owner,key) insert with a fresh id must DO NOTHING. let dup = sqlx::query( "INSERT INTO remember_jobs (id, owner, namespace, status, idempotency_key) VALUES ($1, $2, 'ns', 'running', $3) - ON CONFLICT (owner, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING", + ON CONFLICT (owner, namespace, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING", ) .bind(format!("remember-job-{}", uuid::Uuid::new_v4())) .bind(&owner) @@ -2134,7 +2143,7 @@ mod tests { ); // Original row is still the one that survives. - let still = find_remember_job_by_key(&pool, &owner, key).await.unwrap(); + let still = find_remember_job_by_key(&pool, &owner, "ns", key).await.unwrap(); assert_eq!( still, Some((job_id.clone(), "running".to_string(), None, None)) @@ -2146,6 +2155,76 @@ mod tests { .await; } + // WALM-781: Idempotency keys are scoped per (owner, namespace). + // The same key used for identical (or different) text in two different + // namespaces of one account produces two successful writes, and retries + // within a namespace still dedupe. + #[tokio::test] + async fn idempotency_key_is_scoped_per_namespace() { + let pool = idem_test_pool().await; + let owner = format!("0xowner-{}", uuid::Uuid::new_v4()); + let key = "shared-idem-key"; + let text = "Identical fact for two users."; + let fp_a = request_fingerprint(text, "ns-a"); + let fp_b = request_fingerprint(text, "ns-b"); + let job_a = format!("job-a-{}", uuid::Uuid::new_v4()); + let job_b = format!("job-b-{}", uuid::Uuid::new_v4()); + + // 1. First write in ns-a succeeds. + let insert_a = sqlx::query( + "INSERT INTO remember_jobs (id, owner, namespace, status, idempotency_key, request_fingerprint) VALUES ($1, $2, 'ns-a', 'pending', $3, $4) + ON CONFLICT (owner, namespace, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING", + ) + .bind(&job_a) + .bind(&owner) + .bind(key) + .bind(&fp_a) + .execute(&pool) + .await + .unwrap(); + assert_eq!(insert_a.rows_affected(), 1); + + // 2. Second write in ns-b with the SAME key and text also succeeds (does not collide). + let insert_b = sqlx::query( + "INSERT INTO remember_jobs (id, owner, namespace, status, idempotency_key, request_fingerprint) VALUES ($1, $2, 'ns-b', 'pending', $3, $4) + ON CONFLICT (owner, namespace, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING", + ) + .bind(&job_b) + .bind(&owner) + .bind(key) + .bind(&fp_b) + .execute(&pool) + .await + .unwrap(); + assert_eq!(insert_b.rows_affected(), 1, "same key in different namespace must succeed"); + + // 3. Retry within ns-a collides and collapses (insert DO NOTHING). + let retry_a = sqlx::query( + "INSERT INTO remember_jobs (id, owner, namespace, status, idempotency_key, request_fingerprint) VALUES ($1, $2, 'ns-a', 'pending', $3, $4) + ON CONFLICT (owner, namespace, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING", + ) + .bind(format!("job-retry-{}", uuid::Uuid::new_v4())) + .bind(&owner) + .bind(key) + .bind(&fp_a) + .execute(&pool) + .await + .unwrap(); + assert_eq!(retry_a.rows_affected(), 0, "retry within namespace must collapse"); + + // 4. Lookup returns the respective job per namespace. + let found_a = find_remember_job_by_key(&pool, &owner, "ns-a", key).await.unwrap(); + assert_eq!(found_a.unwrap().0, job_a); + + let found_b = find_remember_job_by_key(&pool, &owner, "ns-b", key).await.unwrap(); + assert_eq!(found_b.unwrap().0, job_b); + + let _ = sqlx::query("DELETE FROM remember_jobs WHERE owner = $1") + .bind(&owner) + .execute(&pool) + .await; + } + #[tokio::test] async fn bulk_derived_key_collapses_a_retried_item() { let pool = idem_test_pool().await; @@ -2155,7 +2234,7 @@ mod tests { let job_id = format!("remember-job-{}", uuid::Uuid::new_v4()); let insert = "INSERT INTO remember_jobs (id, owner, namespace, status, idempotency_key, request_fingerprint) VALUES ($1, $2, 'ns', 'pending', $3, $4) - ON CONFLICT (owner, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING"; + ON CONFLICT (owner, namespace, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING"; sqlx::query(insert) .bind(&job_id) @@ -2180,7 +2259,7 @@ mod tests { "retried bulk item must not mint a second job" ); - let found = find_remember_job_by_key(&pool, &owner, item_key) + let found = find_remember_job_by_key(&pool, &owner, "ns", item_key) .await .unwrap() .unwrap(); @@ -2223,7 +2302,7 @@ mod tests { "fixture must start in the orphan state" ); - let existing = find_remember_job_by_key(&pool, &owner, item_key) + let existing = find_remember_job_by_key(&pool, &owner, "ns", item_key) .await .unwrap() .unwrap(); @@ -2260,7 +2339,7 @@ mod tests { for _ in 0..2 { sqlx::query( "INSERT INTO remember_jobs (id, owner, namespace, status, idempotency_key) VALUES ($1, $2, 'ns', 'running', NULL) - ON CONFLICT (owner, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING", + ON CONFLICT (owner, namespace, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING", ) .bind(format!("remember-job-{}", uuid::Uuid::new_v4())) .bind(&owner) @@ -2316,7 +2395,7 @@ mod tests { // Retry #1 after the terminal failure: the key resolves to the failed // row, which is restartable now, not after the TTL. - let (found_id, status, blob, _) = find_remember_job_by_key(&pool, &owner, key) + let (found_id, status, blob, _) = find_remember_job_by_key(&pool, &owner, "ns", key) .await .unwrap() .expect("row bound to key"); @@ -2344,7 +2423,7 @@ mod tests { // Retry #2 while that restart is in flight: same row, no new claim, and // the insert path cannot mint a second job for the key. - let (again_id, again_status, again_blob, _) = find_remember_job_by_key(&pool, &owner, key) + let (again_id, again_status, again_blob, _) = find_remember_job_by_key(&pool, &owner, "ns", key) .await .unwrap() .unwrap(); @@ -2362,7 +2441,7 @@ mod tests { ); let dup = sqlx::query( "INSERT INTO remember_jobs (id, owner, namespace, status, idempotency_key) VALUES ($1, $2, 'ns', 'pending', $3) - ON CONFLICT (owner, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING", + ON CONFLICT (owner, namespace, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING", ) .bind(uuid::Uuid::new_v4().to_string()) .bind(&owner) diff --git a/services/server/src/storage/db.rs b/services/server/src/storage/db.rs index fa87be1e6..d9e5e7ef2 100644 --- a/services/server/src/storage/db.rs +++ b/services/server/src/storage/db.rs @@ -160,6 +160,8 @@ const MIGRATIONS_AFTER_INDEX_RECOVERY: &[Migration] = &[ // deleted_at range index for the admin activity delete counts. // CONCURRENTLY, own file -- see 024's header. migration!("024_memory_tombstones_deleted_at_index.sql"), + // Drop the legacy account-wide idempotency index (WALM-781). + migration!("025_drop_owner_idempotency_key_index.sql"), ]; /// Every migration the pipeline applies, in the order it applies them. @@ -2035,7 +2037,7 @@ mod tests { let rewritten = sqlx::query( "INSERT INTO remember_jobs (id, owner, namespace, status, idempotency_key) VALUES ($1, $2, 'notes', 'pending', $3) - ON CONFLICT (owner, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING", + ON CONFLICT (owner, namespace, idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING", ) .bind(format!("rewrite-job-{suffix}")) .bind(&owner) @@ -2370,8 +2372,8 @@ fn db_status(result: &Result) -> &'static str { /// hitting the GUC. Recovery still exists for a crash or kill mid-build. /// /// Only indexes whose loss is SILENT belong here. Migration 013's -/// `uq_remember_jobs_owner_idempotency_key` is deliberately absent: it is -/// a unique index backing `ON CONFLICT (owner, idempotency_key)`, and an +/// `uq_remember_jobs_owner_namespace_idempotency_key` is deliberately absent: it is +/// a unique index backing `ON CONFLICT (owner, namespace, idempotency_key)`, and an /// invalid one makes that upsert ERROR outright rather than quietly /// degrade, so it reports itself. It also runs in /// `MIGRATIONS_BEFORE_BACKFILL`, ahead of the single recovery pass below. From 9404306e48331397d2582e927a89a2460ecd726b Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Mon, 12 Oct 2026 18:14:07 +0700 Subject: [PATCH 23/70] fix(sdk): derive and forward idempotencyKey in rememberBulk (Fixes #1080) --- docs/relayer/api-reference.md | 10 +- packages/sdk/src/index.ts | 1 + packages/sdk/src/memwal.ts | 69 +++++--- packages/sdk/src/mock.ts | 27 +++- packages/sdk/src/types.ts | 8 +- packages/sdk/src/utils.ts | 13 ++ .../test/remember-bulk-idempotency.test.mjs | 150 ++++++++++++++++++ 7 files changed, 248 insertions(+), 30 deletions(-) create mode 100644 packages/sdk/test/remember-bulk-idempotency.test.mjs diff --git a/docs/relayer/api-reference.md b/docs/relayer/api-reference.md index b402c62d5..56146795d 100644 --- a/docs/relayer/api-reference.md +++ b/docs/relayer/api-reference.md @@ -192,11 +192,12 @@ Submit text as an encrypted memory job. The relayer returns after creating a bac ```json { "text": "User prefers dark mode", - "namespace": "demo" + "namespace": "demo", + "idempotency_key": "my-idempotency-key" } ``` -`namespace` defaults to `"default"` if omitted and is limited to 255 UTF-8 bytes. +`namespace` defaults to `"default"` if omitted and is limited to 255 UTF-8 bytes. `idempotency_key` is an optional string (up to 255 UTF-8 bytes); when supplied, repeated calls with the same key collapse onto the existing job row. **Response:** `202 Accepted` @@ -234,10 +235,13 @@ Submit up to 20 memories in one request. `job_ids[i]` corresponds to `items[i]`. "items": [ { "text": "User prefers dark mode", "namespace": "demo" }, { "text": "User works in TypeScript", "namespace": "demo" } - ] + ], + "idempotency_key": "my-bulk-idempotency-key" } ``` +`idempotency_key` is an optional string (up to 255 UTF-8 bytes). When provided, each item in the batch is assigned `bulk:{idempotency_key}:{i}`, so repeated requests with the same batch key collapse onto the original jobs. + **Response:** `202 Accepted` ```json diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 293dc30f0..0bfb23c7e 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -62,6 +62,7 @@ export type { RestoreResult, RememberBulkItem, RememberBulkOptions, + RememberBulkAsyncOptions, RememberBulkAcceptedResult, RememberBulkStatusItem, RememberBulkStatusResult, diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index 7a85b1fe6..e8685af30 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -51,6 +51,7 @@ import type { ListMemoriesOptions, RememberBulkItem, RememberBulkOptions, + RememberBulkAsyncOptions, RememberBulkResult, RememberAcceptedResult, RememberJobStatus, @@ -71,6 +72,8 @@ import { redactInternalUrls, clockDriftErrorFromResponse, scoringWeightsToWire, + derivedIdempotencyKey, + derivedBulkIdempotencyKey, } from "./utils.js"; import { assertCompatibleRelayer, @@ -135,19 +138,6 @@ function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } -/** Window over which the same (namespace, text) resolves to one key. - * - * `pendingRememberKeys` only dedupes retries that reuse one client instance, - * and callers such as the MCP sidecar build a fresh client per session — so a - * random key let a reconnect replay mint a second paid blob. The bucket bounds - * the collapse: `remember_jobs` rows are never pruned, so an unbucketed key - * would dedupe against a job from any point in history. */ -const IDEMPOTENCY_BUCKET_MS = 30 * 60 * 1000; - -async function derivedIdempotencyKey(requestIdentity: string): Promise { - const bucket = Math.floor(Date.now() / IDEMPOTENCY_BUCKET_MS); - return `r1-${await sha256hex(`${bucket}\0${requestIdentity}`)}`; -} /** Deadline for a request that names no other. * @@ -722,7 +712,10 @@ export class MemWal { * console.log(accepted.job_ids) * ``` */ - async rememberBulkAsync(items: RememberBulkItem[]): Promise { + async rememberBulkAsync( + items: RememberBulkItem[], + options: RememberBulkAsyncOptions = {}, + ): Promise { if (!Array.isArray(items) || items.length === 0) { throw new Error("rememberBulkAsync: items must be a non-empty array"); } @@ -741,12 +734,20 @@ export class MemWal { namespace: item.namespace ?? this.namespace, })); + const requestIdentity = `bulk\0${normalised.map((item) => `${item.namespace}\0${item.text}`).join("\0\0")}`; + const generatedKey = options.idempotencyKey === undefined; + const idempotencyKey = options.idempotencyKey + ?? this.pendingRememberKeys.get(requestIdentity) + ?? (await derivedBulkIdempotencyKey(requestIdentity)); + if (generatedKey) this.pendingRememberKeys.set(requestIdentity, idempotencyKey); + const accepted = await this.signedRequest( "POST", "/api/remember/bulk", - { items: normalised }, + { items: normalised, idempotency_key: idempotencyKey }, [200, 202], ); + if (generatedKey) this.pendingRememberKeys.delete(requestIdentity); if (!accepted.job_ids || accepted.job_ids.length !== normalised.length) { throw new Error( @@ -934,8 +935,11 @@ export class MemWal { /** * Remember multiple memories and return as soon as the server accepts the jobs. */ - async rememberBulk(items: RememberBulkItem[]): Promise { - return this.rememberBulkAsync(items); + async rememberBulk( + items: RememberBulkItem[], + options: RememberBulkAsyncOptions = {}, + ): Promise { + return this.rememberBulkAsync(items, options); } /** @@ -945,9 +949,34 @@ export class MemWal { items: RememberBulkItem[], opts: RememberBulkOptions = {}, ): Promise { - const namespaces = items.map((item) => item.namespace ?? this.namespace); - const accepted = await this.rememberBulkAsync(items); - return this.waitForRememberJobs(accepted.job_ids, namespaces, opts); + if (!Array.isArray(items) || items.length === 0) { + throw new Error("rememberBulkAsync: items must be a non-empty array"); + } + if (items.length > MAX_BULK_ITEMS) { + throw new Error( + `rememberBulkAsync: items exceeds maximum of ${MAX_BULK_ITEMS} per bulk request`, + ); + } + items.forEach((item, i) => { + assertNonBlankText(item?.text, `items[${i}].text cannot be empty`); + }); + + const normalised = items.map((item) => ({ + text: item.text, + namespace: item.namespace ?? this.namespace, + })); + const requestIdentity = `bulk\0${normalised.map((item) => `${item.namespace}\0${item.text}`).join("\0\0")}`; + const generatedKey = opts.idempotencyKey === undefined; + const idempotencyKey = opts.idempotencyKey + ?? this.pendingRememberKeys.get(requestIdentity) + ?? (await derivedBulkIdempotencyKey(requestIdentity)); + if (generatedKey) this.pendingRememberKeys.set(requestIdentity, idempotencyKey); + + const namespaces = normalised.map((item) => item.namespace); + const accepted = await this.rememberBulkAsync(items, { ...opts, idempotencyKey }); + const completed = await this.waitForRememberJobs(accepted.job_ids, namespaces, opts); + if (generatedKey) this.pendingRememberKeys.delete(requestIdentity); + return completed; } /** diff --git a/packages/sdk/src/mock.ts b/packages/sdk/src/mock.ts index ae1c9d43d..55e675e72 100644 --- a/packages/sdk/src/mock.ts +++ b/packages/sdk/src/mock.ts @@ -13,6 +13,7 @@ import type { RememberBulkItem, RememberBulkItemResult, RememberBulkOptions, + RememberBulkAsyncOptions, RememberBulkResult, RememberBulkStatusResult, RememberJobStatus, @@ -23,6 +24,7 @@ import type { } from "./types.js"; import { applyTokenBudget, estimateTokens } from "./tokens.js"; import { resolveRecallCall } from "./recall-args.js"; +import { derivedBulkIdempotencyKey } from "./utils.js"; /** Matches the relayer's `MAX_BULK_ITEMS`. */ const MAX_BULK_ITEMS = 20; @@ -220,7 +222,8 @@ export class MemWalMock { } async rememberBulkAsync( - items: RememberBulkItem[] + items: RememberBulkItem[], + options: RememberBulkAsyncOptions = {} ): Promise { if (!Array.isArray(items) || items.length === 0) { throw new Error("rememberBulkAsync: items must be a non-empty array"); @@ -230,16 +233,28 @@ export class MemWalMock { `rememberBulkAsync: items exceeds maximum of ${MAX_BULK_ITEMS} per bulk request`, ); } - const jobIds = items.map( - (item) => this.store(item.text, item.namespace).jobId + items.forEach((item, i) => { + validateText(item?.text, `items[${i}].text`); + }); + + const normalised = items.map((item) => ({ + text: item.text, + namespace: item.namespace ?? this.namespace, + })); + const requestIdentity = `bulk\0${normalised.map((item) => `${item.namespace}\0${item.text}`).join("\0\0")}`; + const bulkKey = options.idempotencyKey ?? (await derivedBulkIdempotencyKey(requestIdentity)); + const jobIds = normalised.map( + (item, index) => + this.store(item.text, item.namespace, undefined, `bulk:${bulkKey}:${index}`).jobId ); return { job_ids: jobIds, total: jobIds.length, status: "done" }; } async rememberBulk( - items: RememberBulkItem[] + items: RememberBulkItem[], + options: RememberBulkAsyncOptions = {} ): Promise { - return this.rememberBulkAsync(items); + return this.rememberBulkAsync(items, options); } async getRememberBulkStatus( @@ -297,7 +312,7 @@ export class MemWalMock { items: RememberBulkItem[], opts: RememberBulkOptions = {} ): Promise { - const accepted = await this.rememberBulkAsync(items); + const accepted = await this.rememberBulkAsync(items, opts); const namespaces = items.map( (item) => item.namespace ?? this.namespace ); diff --git a/packages/sdk/src/types.ts b/packages/sdk/src/types.ts index f147ea2a7..a52ff4e2c 100644 --- a/packages/sdk/src/types.ts +++ b/packages/sdk/src/types.ts @@ -248,8 +248,14 @@ export interface RememberBulkItem { namespace?: string; } +/** Options for rememberBulk / rememberBulkAsync requests */ +export interface RememberBulkAsyncOptions { + /** Optional caller-supplied idempotency key (defaults to derived key based on items) */ + idempotencyKey?: string; +} + /** Options for remember bulk polling behaviour */ -export interface RememberBulkOptions { +export interface RememberBulkOptions extends RememberBulkAsyncOptions { /** How often to poll each job_id (default: 1500ms) */ pollIntervalMs?: number; /** Max total wait time before throwing (default: 120_000ms) */ diff --git a/packages/sdk/src/utils.ts b/packages/sdk/src/utils.ts index 6634f1cc6..3e58da1e8 100644 --- a/packages/sdk/src/utils.ts +++ b/packages/sdk/src/utils.ts @@ -33,6 +33,19 @@ export async function sha256hex(data: string): Promise { return bytesToHex(sha256(new TextEncoder().encode(data))); } +/** Window over which the same content resolves to one key (30 minutes). */ +export const IDEMPOTENCY_BUCKET_MS = 30 * 60 * 1000; + +export async function derivedIdempotencyKey(requestIdentity: string): Promise { + const bucket = Math.floor(Date.now() / IDEMPOTENCY_BUCKET_MS); + return `r1-${await sha256hex(`${bucket}\0${requestIdentity}`)}`; +} + +export async function derivedBulkIdempotencyKey(requestIdentity: string): Promise { + const bucket = Math.floor(Date.now() / IDEMPOTENCY_BUCKET_MS); + return `b1-${await sha256hex(`${bucket}\0${requestIdentity}`)}`; +} + // ============================================================ // Hex Encoding // ============================================================ diff --git a/packages/sdk/test/remember-bulk-idempotency.test.mjs b/packages/sdk/test/remember-bulk-idempotency.test.mjs new file mode 100644 index 000000000..140fec30c --- /dev/null +++ b/packages/sdk/test/remember-bulk-idempotency.test.mjs @@ -0,0 +1,150 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { MemWal } from "../dist/memwal.js"; +import { MemWalMock } from "../dist/mock.js"; + +const originalFetch = globalThis.fetch; + +test.afterEach(() => { + globalThis.fetch = originalFetch; +}); + +function stubRelayer({ onStatus = () => ({ status: "done", blob_id: "blob-1" }), posted = [] } = {}) { + globalThis.fetch = async (url, init = {}) => { + const path = new URL(url).pathname; + if (path === "/version") { + return Response.json({ + apiVersion: "1.0.0", + relayerVersion: "1.0.0", + minSupportedSdk: { typescript: "0.0.4" }, + }); + } + if (path === "/api/config") { + return Response.json({ packageId: "0x1", network: "testnet" }); + } + if (path === "/api/remember/bulk" && init.method === "POST") { + const body = JSON.parse(init.body); + posted.push({ body, at: Date.now() }); + const job_ids = body.items.map((_, i) => `job-${i + 1}`); + return Response.json({ job_ids, total: job_ids.length, status: "pending" }, { status: 202 }); + } + if (path === "/api/remember/bulk/status" && init.method === "POST") { + const body = JSON.parse(init.body); + const results = body.job_ids.map((job_id) => ({ + job_id, + ...onStatus(job_id), + })); + return Response.json({ results }); + } + throw new Error(`unexpected request ${path}`); + }; + return { posted }; +} + +function newClient() { + const client = MemWal.create({ + key: new Uint8Array(32).fill(1), + accountId: "0x1", + serverUrl: "https://relayer.example", + }); + client.buildSealSession = async () => "test-session"; + return client; +} + +test("rememberBulkAsync derives stable idempotency key for identical items across client instances", async () => { + const posted = []; + stubRelayer({ posted }); + + const items = [ + { text: "first fact to remember" }, + { text: "second fact to remember", namespace: "custom" }, + ]; + + await newClient().rememberBulkAsync(items); + await newClient().rememberBulkAsync(items); + + assert.equal(posted.length, 2); + assert.ok(posted[0].body.idempotency_key, "idempotency_key should be generated"); + assert.equal(posted[0].body.idempotency_key, posted[1].body.idempotency_key); +}); + +test("different bulk items or namespaces generate different derived keys", async () => { + const posted = []; + stubRelayer({ posted }); + + const client = newClient(); + await client.rememberBulkAsync([{ text: "fact A" }]); + await client.rememberBulkAsync([{ text: "fact B" }]); + await client.rememberBulkAsync([{ text: "fact A", namespace: "other" }]); + + const keys = posted.map((p) => p.body.idempotency_key); + assert.equal(new Set(keys).size, 3, "distinct item sets must generate distinct idempotency keys"); +}); + +test("explicit idempotencyKey in rememberBulkAsync and rememberBulk is passed to relayer", async () => { + const posted = []; + stubRelayer({ posted }); + + const client = newClient(); + await client.rememberBulkAsync([{ text: "fact 1" }], { idempotencyKey: "caller-bulk-key-1" }); + await client.rememberBulk([{ text: "fact 2" }], { idempotencyKey: "caller-bulk-key-2" }); + + assert.equal(posted.length, 2); + assert.equal(posted[0].body.idempotency_key, "caller-bulk-key-1"); + assert.equal(posted[1].body.idempotency_key, "caller-bulk-key-2"); +}); + +test("rememberBulkAndWait forwards idempotencyKey option to rememberBulkAsync", async () => { + const posted = []; + stubRelayer({ posted }); + + const client = newClient(); + const result = await client.rememberBulkAndWait( + [{ text: "fact 1" }, { text: "fact 2" }], + { idempotencyKey: "custom-wait-key", pollIntervalMs: 0, timeoutMs: 1000 }, + ); + + assert.equal(result.succeeded, 2); + assert.equal(posted.length, 1); + assert.equal(posted[0].body.idempotency_key, "custom-wait-key"); +}); + +test("MemWalMock supports idempotencyKey in rememberBulk, rememberBulkAsync, and rememberBulkAndWait", async () => { + const mock = MemWalMock.create(); + + const items = [ + { text: "mock fact 1" }, + { text: "mock fact 2" }, + ]; + + const first = await mock.rememberBulk(items, { idempotencyKey: "mock-key-1" }); + const second = await mock.rememberBulk(items, { idempotencyKey: "mock-key-1" }); + + assert.deepEqual(first.job_ids, second.job_ids, "repeated call with same idempotencyKey should return same job_ids"); + + // Attempting same key with different text should throw + await assert.rejects( + () => mock.rememberBulk([{ text: "different fact" }], { idempotencyKey: "mock-key-1" }), + /idempotency_key was already used for a request with different content/, + ); + + // rememberBulkAndWait with idempotencyKey + const waitResult = await mock.rememberBulkAndWait(items, { idempotencyKey: "mock-key-1" }); + assert.equal(waitResult.succeeded, 2); + assert.deepEqual(waitResult.results.map((r) => r.id), first.job_ids); +}); + +test("MemWalMock derives stable idempotency key for identical items when idempotencyKey is omitted", async () => { + const mock = MemWalMock.create(); + const items = [ + { text: "auto derived fact 1" }, + { text: "auto derived fact 2" }, + ]; + const first = await mock.rememberBulk(items); + const second = await mock.rememberBulk(items); + + assert.deepEqual(first.job_ids, second.job_ids); + const recall = await mock.recall({ query: "derived", limit: 10 }); + assert.equal(recall.total, 2); +}); From e27b69168f5eb64f96c4a31075f88d585c725903 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Mon, 12 Oct 2026 18:20:43 +0700 Subject: [PATCH 24/70] fix(python-sdk): anchor memory system instruction to start of prompt and catch cross-loop ValueError (Fixes #1159, WALM-817) --- .../python-sdk-memwal/memwal/middleware.py | 50 +++++++--- .../tests/test_middleware.py | 92 +++++++++++++++++-- 2 files changed, 120 insertions(+), 22 deletions(-) diff --git a/packages/python-sdk-memwal/memwal/middleware.py b/packages/python-sdk-memwal/memwal/middleware.py index 903cfdebf..649ce27f3 100644 --- a/packages/python-sdk-memwal/memwal/middleware.py +++ b/packages/python-sdk-memwal/memwal/middleware.py @@ -188,7 +188,7 @@ async def _drain(tasks: "list[asyncio.Task[Any]]") -> None: try: asyncio.run(_drain(list(self._tasks))) - except RuntimeError: + except (RuntimeError, ValueError): logger.warning( "Walrus Memory flush_sync() could not await %d pending " "task(s) bound to a different event loop (mixing sync " @@ -366,12 +366,19 @@ async def _inject_memories(messages: List[BaseMessage]) -> List[BaseMessage]: last_human_idx = i break - guard_msg = SystemMessage(content=_UNTRUSTED_MEMORY_SYSTEM_INSTRUCTION) memory_msg = HumanMessage(content=memory_context) if last_human_idx > 0: - result[last_human_idx:last_human_idx] = [guard_msg, memory_msg] + result.insert(last_human_idx, memory_msg) else: - result[0:0] = [guard_msg, memory_msg] + result.insert(0, memory_msg) + + if result and isinstance(result[0], SystemMessage): + existing_content = result[0].content + result[0] = SystemMessage( + content=f"{existing_content}\n\n{_UNTRUSTED_MEMORY_SYSTEM_INSTRUCTION}".strip() + ) + else: + result.insert(0, SystemMessage(content=_UNTRUSTED_MEMORY_SYSTEM_INSTRUCTION)) return result except Exception as e: @@ -625,21 +632,34 @@ def _inject_openai_memory( messages: List[Dict[str, Any]], memory_context: str, ) -> List[Dict[str, Any]]: - """Insert a fixed system guard plus memory content in a user message.""" + """Insert a fixed system guard at index 0 and memory content in a user message.""" + result = [dict(m) for m in messages] last_user_idx = -1 - for i in range(len(messages) - 1, -1, -1): - if isinstance(messages[i], dict) and messages[i].get("role") == "user": + for i in range(len(result) - 1, -1, -1): + if isinstance(result[i], dict) and result[i].get("role") == "user": last_user_idx = i break - memory_messages: List[Dict[str, Any]] = [ - {"role": "system", "content": _UNTRUSTED_MEMORY_SYSTEM_INSTRUCTION}, - {"role": "user", "content": memory_context}, - ] - + memory_message = {"role": "user", "content": memory_context} if last_user_idx > 0: - messages[last_user_idx:last_user_idx] = memory_messages + result.insert(last_user_idx, memory_message) + else: + result.insert(0, memory_message) + + if result and isinstance(result[0], dict) and result[0].get("role") == "system": + existing_content = result[0].get("content") + if isinstance(existing_content, list): + result[0] = { + **result[0], + "content": existing_content + [{"type": "text", "text": f"\n\n{_UNTRUSTED_MEMORY_SYSTEM_INSTRUCTION}"}], + } + else: + text_str = str(existing_content or "") + result[0] = { + **result[0], + "content": f"{text_str}\n\n{_UNTRUSTED_MEMORY_SYSTEM_INSTRUCTION}".strip(), + } else: - messages[0:0] = memory_messages + result.insert(0, {"role": "system", "content": _UNTRUSTED_MEMORY_SYSTEM_INSTRUCTION}) - return messages + return result diff --git a/packages/python-sdk-memwal/tests/test_middleware.py b/packages/python-sdk-memwal/tests/test_middleware.py index c95706789..f58e5fec3 100644 --- a/packages/python-sdk-memwal/tests/test_middleware.py +++ b/packages/python-sdk-memwal/tests/test_middleware.py @@ -240,27 +240,53 @@ def test_forged_boundary_and_instruction_stay_inside_actual_boundary(self) -> No class TestInjectOpenAIMemory: """Tests for _inject_openai_memory.""" - def test_inserts_before_last_user_message(self) -> None: + def test_folds_guard_into_system_message_and_inserts_memory_before_user_message(self) -> None: messages = [ {"role": "system", "content": "You are helpful."}, {"role": "user", "content": "question"}, ] result = _inject_openai_memory(messages, "memory context") - # Should be: existing-system | fixed-guard | memory-user | user + # Should be: existing-system-with-guard | memory-user | user + assert len(result) == 3 assert result[0]["role"] == "system" - assert result[1]["role"] == "system" - assert "untrusted data" in result[1]["content"] - assert result[2] == {"role": "user", "content": "memory context"} - assert result[3]["role"] == "user" + assert "You are helpful." in result[0]["content"] + assert "untrusted data" in result[0]["content"] + assert result[1] == {"role": "user", "content": "memory context"} + assert result[2]["role"] == "user" + assert result[2]["content"] == "question" + + def test_multi_turn_never_injects_system_message_after_index_0(self) -> None: + messages = [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "first"}, + {"role": "assistant", "content": "reply"}, + {"role": "user", "content": "second"}, + ] + result = _inject_openai_memory(messages, "recalled fact") + # System message only at index 0 + assert result[0]["role"] == "system" + assert "You are helpful." in result[0]["content"] + assert "untrusted data" in result[0]["content"] + + # No system message after index 0 + assert not any(m.get("role") == "system" for m in result[1:]) + + # Memory is injected as user message right before "second" + assert result[1] == {"role": "user", "content": "first"} + assert result[2] == {"role": "assistant", "content": "reply"} + assert result[3] == {"role": "user", "content": "recalled fact"} + assert result[4] == {"role": "user", "content": "second"} def test_inserts_at_start_when_only_user_message(self) -> None: messages = [{"role": "user", "content": "question"}] result = _inject_openai_memory(messages, "context") - # last_user_idx == 0, so prepend guard + memory-data messages + # When no system message exists, prepends guard system message at index 0 assert result[0]["role"] == "system" + assert "untrusted data" in result[0]["content"] assert result[1]["role"] == "user" assert result[1]["content"] == "context" assert result[2]["role"] == "user" + assert result[2]["content"] == "question" def test_does_not_mutate_original(self) -> None: original = [{"role": "user", "content": "q"}] @@ -335,6 +361,58 @@ async def _capture_agenerate(messages_batch, *a, **kw): assert "User loves coffee" in human_msgs[0].content assert "BEGIN_UNTRUSTED_WALRUS_MEMORY_" in human_msgs[0].content + @respx.mock + async def test_multi_turn_never_injects_system_message_after_index_0(self) -> None: + """In a multi-turn conversation, SystemMessage stays strictly at index 0.""" + _mock_seal_session_prereqs() + from langchain_core.messages import AIMessage, HumanMessage, SystemMessage + from langchain_core.outputs import ChatGeneration, ChatResult + + llm = self._make_llm() + captured: list = [] + + async def _capture_agenerate(messages_batch, *a, **kw): + captured.extend(messages_batch) + return ChatResult( + generations=[ChatGeneration(message=AIMessage(content="ok"))] + ) + + llm._agenerate = _capture_agenerate + + respx.post(_RECALL_URL).mock( + return_value=_mock_recall([ + {"blob_id": "b1", "text": "User loves coffee", "distance": 0.05} + ]) + ) + respx.post(_ANALYZE_URL).mock(return_value=_mock_analyze()) + + smart_llm = with_memwal_langchain( + llm, key=_KEY_HEX, account_id=_ACCOUNT_ID, server_url=_SERVER, auto_save=False + ) + messages = [ + SystemMessage(content="You are helpful."), + HumanMessage(content="I like coffee"), + AIMessage(content="Noted!"), + HumanMessage(content="What do I like?"), + ] + await smart_llm._agenerate([messages]) + + assert len(captured) == 1 + injected = captured[0] + # SystemMessage strictly at index 0, with guard folded in + assert isinstance(injected[0], SystemMessage) + assert "You are helpful." in injected[0].content + assert "untrusted data" in injected[0].content + + # No SystemMessage after index 0 + assert not any(isinstance(m, SystemMessage) for m in injected[1:]) + + # The recalled memory is in a HumanMessage before the final query + assert isinstance(injected[3], HumanMessage) + assert "User loves coffee" in injected[3].content + assert isinstance(injected[4], HumanMessage) + assert injected[4].content == "What do I like?" + @respx.mock async def test_no_memories_found_no_injection(self) -> None: """When no memories match, the message list is passed unchanged.""" From b0b93ed0d39781496aa16b986a98a3d55cc985f8 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Mon, 12 Oct 2026 18:25:14 +0700 Subject: [PATCH 25/70] fix(python-sdk): support HTTP-date Retry-After and prevent float overflow (Fixes #1151, WALM-815) --- packages/python-sdk-memwal/memwal/client.py | 39 ++++- .../tests/test_retry_after_http_date.py | 161 ++++++++++++++++++ 2 files changed, 197 insertions(+), 3 deletions(-) create mode 100644 packages/python-sdk-memwal/tests/test_retry_after_http_date.py diff --git a/packages/python-sdk-memwal/memwal/client.py b/packages/python-sdk-memwal/memwal/client.py index a8416d9ec..0f8292380 100644 --- a/packages/python-sdk-memwal/memwal/client.py +++ b/packages/python-sdk-memwal/memwal/client.py @@ -29,11 +29,13 @@ import base64 import json import logging +import math import random import re import time import uuid from datetime import datetime, timezone +from email.utils import parsedate_to_datetime from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, TypeVar, Union from urllib.parse import ParseResult, urlencode, urlparse @@ -199,18 +201,49 @@ def _positive_retry_ms(raw: object) -> Optional[int]: seconds = float(raw) # type: ignore[arg-type] except (TypeError, ValueError): return None - if seconds <= 0 or seconds != seconds or seconds == float("inf"): + if seconds <= 0 or math.isnan(seconds) or math.isinf(seconds): + return None + try: + ms = seconds * 1000.0 + if math.isinf(ms) or math.isnan(ms): + return None + return int(ms) + except (OverflowError, ValueError): + return None + + +def _http_date_retry_ms(raw: object) -> Optional[int]: + if not isinstance(raw, str): + return None + try: + dt = parsedate_to_datetime(raw) + if dt is None: + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + target_ts = dt.timestamp() + now_ts = time.time() + diff = target_ts - now_ts + if diff <= 0 or math.isnan(diff) or math.isinf(diff): + return None + ms = diff * 1000.0 + if math.isinf(ms) or math.isnan(ms): + return None + return int(ms) + except Exception: return None - return int(seconds * 1000) def _retry_after_ms(err: "_HttpStatusError") -> int: - """Retry-After seconds, else JSON ``retry_after_seconds``, in ms.""" + """Retry-After seconds or HTTP-date, else JSON ``retry_after_seconds``, in ms.""" if err.retry_after: header_ms = _positive_retry_ms(err.retry_after) if header_ms is not None: return header_ms + date_ms = _http_date_retry_ms(err.retry_after) + if date_ms is not None: + return date_ms if not err.body: return 0 try: diff --git a/packages/python-sdk-memwal/tests/test_retry_after_http_date.py b/packages/python-sdk-memwal/tests/test_retry_after_http_date.py new file mode 100644 index 000000000..4678f8a5b --- /dev/null +++ b/packages/python-sdk-memwal/tests/test_retry_after_http_date.py @@ -0,0 +1,161 @@ +"""Tests for HTTP-date Retry-After header parsing and arithmetic overflow protection.""" + +from __future__ import annotations + +import time +from datetime import datetime, timedelta, timezone +from email.utils import format_datetime +from types import SimpleNamespace +from typing import Optional + +import pytest + +from memwal.client import ( + MemWal, + MemWalRememberJobTimeout, + _HttpStatusError, + _clamped_retry_after_ms, + _positive_retry_ms, + _retry_after_ms, +) + + +def _err( + retry_after: Optional[str] = None, + body: Optional[str] = None, +) -> _HttpStatusError: + return _HttpStatusError(429, body or "{}", retry_after=retry_after) + + +def test_positive_retry_ms_delta_seconds() -> None: + assert _positive_retry_ms("45") == 45000 + assert _positive_retry_ms(45) == 45000 + assert _positive_retry_ms("0.5") == 500 + assert _positive_retry_ms(0.25) == 250 + + +def test_positive_retry_ms_invalid_or_non_positive() -> None: + assert _positive_retry_ms(0) is None + assert _positive_retry_ms("0") is None + assert _positive_retry_ms("-5") is None + assert _positive_retry_ms("invalid") is None + assert _positive_retry_ms(None) is None + assert _positive_retry_ms(float("nan")) is None + assert _positive_retry_ms(float("inf")) is None + + +def test_positive_retry_ms_overflow_protection() -> None: + # 1e308 is finite float, but 1e308 * 1000 overflows to infinity + assert _positive_retry_ms("1e308") is None + assert _positive_retry_ms(1e308) is None + + +def test_retry_after_http_date_in_future() -> None: + future_time = datetime.now(timezone.utc) + timedelta(seconds=45) + future_header = format_datetime(future_time, usegmt=True) + + err = _err(retry_after=future_header) + delay = _retry_after_ms(err) + # Delay should be approximately 45,000 ms (allowing slight test execution variance) + assert 40000 <= delay <= 46000 + + +def test_retry_after_http_date_takes_priority_over_json() -> None: + future_time = datetime.now(timezone.utc) + timedelta(seconds=60) + future_header = format_datetime(future_time, usegmt=True) + body = '{"retry_after_seconds": 10}' + + err = _err(retry_after=future_header, body=body) + delay = _retry_after_ms(err) + assert 55000 <= delay <= 61000 + + +def test_retry_after_http_date_in_past_falls_back_to_json() -> None: + past_time = datetime.now(timezone.utc) - timedelta(seconds=10) + past_header = format_datetime(past_time, usegmt=True) + body = '{"retry_after_seconds": 15}' + + err = _err(retry_after=past_header, body=body) + assert _retry_after_ms(err) == 15000 + + +def test_retry_after_http_date_in_past_returns_zero_when_no_json() -> None: + past_time = datetime.now(timezone.utc) - timedelta(seconds=10) + past_header = format_datetime(past_time, usegmt=True) + + err = _err(retry_after=past_header) + assert _retry_after_ms(err) == 0 + + +def test_retry_after_malformed_header_falls_back_to_json() -> None: + err = _err(retry_after="Not A Valid Date Or Number", body='{"retry_after_seconds": 25}') + assert _retry_after_ms(err) == 25000 + + +def test_retry_after_malformed_header_without_json_returns_zero() -> None: + err = _err(retry_after="invalid-header") + assert _retry_after_ms(err) == 0 + + +def test_retry_after_overflowing_number_falls_back_to_json() -> None: + err = _err(retry_after="1e308", body='{"retry_after_seconds": 30}') + assert _retry_after_ms(err) == 30000 + + +def test_clamped_retry_after_ms_with_http_date(monkeypatch: pytest.MonkeyPatch) -> None: + fixed_now = 1_000_000 + monkeypatch.setattr("memwal.client._now_ms", lambda: fixed_now) + + # Future date roughly 50s ahead + future_time = datetime.now(timezone.utc) + timedelta(seconds=50) + future_header = format_datetime(future_time, usegmt=True) + err = _err(retry_after=future_header) + + # If remaining time is only 20s (deadline 1,020,000), clamped to 20,000 + clamped = _clamped_retry_after_ms(err, deadline_ms=fixed_now + 20_000) + assert clamped == 20_000 + + # If remaining time is 100s, delay (~50,000 ms) is used + clamped_large = _clamped_retry_after_ms(err, deadline_ms=fixed_now + 100_000) + assert 45_000 <= clamped_large <= 51_000 + + +@pytest.mark.asyncio +async def test_wait_for_remember_job_honors_http_date_retry_after( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = {"now": int(time.time() * 1000)} + slept: list[int] = [] + + def fake_now_ms() -> int: + return clock["now"] + + async def fake_sleep_ms(ms: int) -> None: + slept.append(ms) + clock["now"] += ms + + monkeypatch.setattr("memwal.client._now_ms", fake_now_ms) + monkeypatch.setattr("memwal.client._sleep_ms", fake_sleep_ms) + + client = MemWal.create(key="11" * 32, account_id="0x1", server_url="http://localhost:8000") + calls = {"n": 0} + + # 429 Retry-After is 2 seconds into future + future_dt = datetime.now(timezone.utc) + timedelta(seconds=2) + http_date = format_datetime(future_dt, usegmt=True) + + async def status(*_args: object, **_kwargs: object) -> dict: + calls["n"] += 1 + if calls["n"] == 1: + raise _HttpStatusError(429, "{}", retry_after=http_date) + return {"job_id": "job-1", "status": "done", "blob_id": "b-1"} + + monkeypatch.setattr(client, "_signed_request", status) + + res = await client.wait_for_remember_job("job-1", timeout_ms=10_000) + assert res.blob_id == "b-1" + assert calls["n"] == 2 + # First sleep is attempt 0 (0 ms), second sleep should be ~2000 ms from Retry-After + assert len(slept) == 2 + assert slept[0] == 0 + assert 1500 <= slept[1] <= 2500 From bde9bbd017b20cc23ee78dfa17887f5fe190cabe Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Mon, 12 Oct 2026 18:45:32 +0700 Subject: [PATCH 26/70] fix(python-sdk): retry transport errors and bound polls by deadline in wait_for_remember_job (Fixes #1000, WALM-696) --- packages/python-sdk-memwal/memwal/client.py | 47 ++++- .../tests/test_transport_retry.py | 192 ++++++++++++++++++ 2 files changed, 231 insertions(+), 8 deletions(-) create mode 100644 packages/python-sdk-memwal/tests/test_transport_retry.py diff --git a/packages/python-sdk-memwal/memwal/client.py b/packages/python-sdk-memwal/memwal/client.py index a8416d9ec..a83cb9c5c 100644 --- a/packages/python-sdk-memwal/memwal/client.py +++ b/packages/python-sdk-memwal/memwal/client.py @@ -463,6 +463,10 @@ async def remember_async( ) -> RememberAcceptedResult: return await self.remember(text, namespace, idempotency_key) + def _poll_deadline_s(self, deadline_ms: int) -> float: + remaining_s = (deadline_ms - _now_ms()) / 1000.0 + return max(0.001, min(60.0, remaining_s)) + async def wait_for_remember_job( self, job_id: str, @@ -498,12 +502,22 @@ async def wait_for_remember_job( retry_after_ms = 0 try: - data = await self._signed_request( - "GET", - f"/api/remember/{job_id}", - {}, - accepted_statuses=(200, 404), - ) + poll_timeout = self._poll_deadline_s(deadline_ms) + try: + data = await self._signed_request( + "GET", + f"/api/remember/{job_id}", + {}, + accepted_statuses=(200, 404), + timeout=poll_timeout, + ) + except TypeError: + data = await self._signed_request( + "GET", + f"/api/remember/{job_id}", + {}, + accepted_statuses=(200, 404), + ) except _HttpStatusError as err: if _is_transient_polling_status(err.status): last_refusal = err.status @@ -512,6 +526,9 @@ async def wait_for_remember_job( retry_after_ms = _clamped_retry_after_ms(err, deadline_ms) continue raise + except httpx.TransportError: + last_refusal = 0 + continue status_str = data.get("status") if status_str is None or status_str == "not_found": @@ -632,7 +649,9 @@ async def remember_bulk( return await self.remember_bulk_async(items) async def get_remember_bulk_status( - self, job_ids: Sequence[str] + self, + job_ids: Sequence[str], + timeout: Optional[float] = None, ) -> RememberBulkStatusResult: """Poll the bulk-status endpoint for a batch of job_ids. @@ -644,6 +663,7 @@ async def get_remember_bulk_status( "POST", "/api/remember/bulk/status", {"job_ids": list(job_ids)}, + timeout=timeout, ) return RememberBulkStatusResult( results=[ @@ -726,7 +746,11 @@ def settle(item: RememberBulkStatusItem) -> bool: retry_after_ms = 0 try: - batch = await self.get_remember_bulk_status(pending) + poll_timeout = self._poll_deadline_s(deadline_ms) + try: + batch = await self.get_remember_bulk_status(pending, timeout=poll_timeout) + except TypeError: + batch = await self.get_remember_bulk_status(pending) except _HttpStatusError as err: if _is_transient_polling_status(err.status): last_refusal = err.status @@ -735,6 +759,9 @@ def settle(item: RememberBulkStatusItem) -> bool: retry_after_ms = _clamped_retry_after_ms(err, deadline_ms) continue raise + except httpx.TransportError: + last_refusal = 0 + continue answered = True terminal_ids: Set[str] = { @@ -757,6 +784,8 @@ def settle(item: RememberBulkStatusItem) -> bool: retry_after = f"{wait_ms / 1000:g}" if wait_ms else None raise MemWalRateLimited(job_ids, retry_after) from err # Any other probe failure is not evidence about the jobs. + except httpx.TransportError: + pass else: answered = True settled = { @@ -1515,6 +1544,7 @@ async def _signed_request( body: Dict[str, Any], accepted_statuses: tuple = (200,), include_seal_session: bool = True, + timeout: Optional[float] = None, ) -> Dict[str, Any]: """Make a signed request to the server. @@ -1575,6 +1605,7 @@ async def _signed_request( url=url, headers=headers, content=None if method_upper == "GET" else body_str, + timeout=timeout, ) if response.status_code not in accepted_statuses: diff --git a/packages/python-sdk-memwal/tests/test_transport_retry.py b/packages/python-sdk-memwal/tests/test_transport_retry.py new file mode 100644 index 000000000..e3356337c --- /dev/null +++ b/packages/python-sdk-memwal/tests/test_transport_retry.py @@ -0,0 +1,192 @@ +"""Tests for network transport error retry and deadline bounding during polling (WALM-696, GH #1000).""" + +from __future__ import annotations + +import httpx +import pytest + +from memwal.client import ( + MemWal, + MemWalRememberJobTimeout, + RememberBulkOptions, +) + + +def _client() -> MemWal: + return MemWal.create(key="11" * 32, account_id="0x1", server_url="http://localhost:8000") + + +def _patch_clock(monkeypatch: pytest.MonkeyPatch) -> tuple[dict[str, int], list[int]]: + clock = {"now": 0} + slept: list[int] = [] + + def now_ms() -> int: + return clock["now"] + + async def sleep_ms(ms: int) -> None: + slept.append(ms) + clock["now"] += ms + + monkeypatch.setattr("memwal.client._now_ms", now_ms) + monkeypatch.setattr("memwal.client._sleep_ms", sleep_ms) + return clock, slept + + +@pytest.mark.asyncio +async def test_wait_for_remember_job_retries_connect_error(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_clock(monkeypatch) + client = _client() + calls = {"n": 0} + + async def status(*_args: object, **_kwargs: object) -> dict: + calls["n"] += 1 + if calls["n"] == 1: + raise httpx.ConnectError("simulated connection drop") + return { + "job_id": "job-1", + "status": "done", + "blob_id": "blob-1", + "owner": "0x1", + "namespace": "default", + } + + monkeypatch.setattr(client, "_signed_request", status) + + done = await client.wait_for_remember_job("job-1", poll_interval_ms=100, timeout_ms=5000) + assert done.blob_id == "blob-1" + assert calls["n"] == 2 + + +@pytest.mark.asyncio +async def test_wait_for_remember_job_retries_read_timeout(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_clock(monkeypatch) + client = _client() + calls = {"n": 0} + + async def status(*_args: object, **_kwargs: object) -> dict: + calls["n"] += 1 + if calls["n"] == 1: + raise httpx.ReadTimeout("simulated socket read timeout") + return { + "job_id": "job-2", + "status": "done", + "blob_id": "blob-2", + "owner": "0x1", + "namespace": "default", + } + + monkeypatch.setattr(client, "_signed_request", status) + + done = await client.wait_for_remember_job("job-2", poll_interval_ms=100, timeout_ms=5000) + assert done.blob_id == "blob-2" + assert calls["n"] == 2 + + +@pytest.mark.asyncio +async def test_wait_for_remember_job_times_out_with_http_0_message( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock, slept = _patch_clock(monkeypatch) + client = _client() + calls = {"n": 0} + + async def failing(*_args: object, **_kwargs: object) -> dict: + calls["n"] += 1 + clock["now"] += 100 + raise httpx.ConnectError("network unreachable") + + monkeypatch.setattr(client, "_signed_request", failing) + + with pytest.raises(MemWalRememberJobTimeout) as exc_info: + await client.wait_for_remember_job("job-fail", poll_interval_ms=100, timeout_ms=500) + + err = exc_info.value + assert err.last_refusal == 0 + assert err.status == 504 + assert "last poll: HTTP 0" in str(err) + assert calls["n"] > 1 + + +@pytest.mark.asyncio +async def test_wait_for_remember_job_bounds_poll_timeout(monkeypatch: pytest.MonkeyPatch) -> None: + clock, _ = _patch_clock(monkeypatch) + client = _client() + captured_timeouts: list[float] = [] + + async def capture(*_args: object, **kwargs: object) -> dict: + timeout = kwargs.get("timeout") + if timeout is not None: + captured_timeouts.append(timeout) + return { + "job_id": "job-bound", + "status": "done", + "blob_id": "blob-bound", + "owner": "0x1", + "namespace": "default", + } + + monkeypatch.setattr(client, "_signed_request", capture) + + clock["now"] = 1000 + # timeout_ms=2000 -> deadline=3000 -> remaining=2.0s + await client.wait_for_remember_job("job-bound", timeout_ms=2000) + assert len(captured_timeouts) == 1 + assert captured_timeouts[0] == pytest.approx(2.0, rel=1e-2) + + +@pytest.mark.asyncio +async def test_wait_for_remember_jobs_bulk_retries_transport_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _patch_clock(monkeypatch) + client = _client() + calls = {"n": 0} + + async def bulk_status(job_ids: list[str], timeout: float | None = None): + from memwal.types import RememberBulkStatusItem, RememberBulkStatusResult + + calls["n"] += 1 + if calls["n"] == 1: + raise httpx.ConnectError("simulated bulk network drop") + return RememberBulkStatusResult( + results=[ + RememberBulkStatusItem( + job_id="bulk-1", + status="done", + blob_id="blob-bulk-1", + ) + ] + ) + + monkeypatch.setattr(client, "get_remember_bulk_status", bulk_status) + + res = await client.wait_for_remember_jobs( + ["bulk-1"], + opts=RememberBulkOptions(poll_interval_ms=100, timeout_ms=5000), + ) + assert len(res.results) == 1 + assert res.results[0].status == "done" + assert res.results[0].blob_id == "blob-bulk-1" + assert calls["n"] == 2 + + +@pytest.mark.asyncio +async def test_wait_for_remember_jobs_bulk_timeout_on_persistent_transport_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock, _ = _patch_clock(monkeypatch) + client = _client() + + async def bulk_failing(job_ids: list[str], timeout: float | None = None): + clock["now"] += 100 + raise httpx.ConnectError("persistent bulk network drop") + + monkeypatch.setattr(client, "get_remember_bulk_status", bulk_failing) + + res = await client.wait_for_remember_jobs( + ["bulk-dead"], + opts=RememberBulkOptions(poll_interval_ms=100, timeout_ms=500), + ) + assert len(res.results) == 1 + assert res.results[0].status == "timeout" + assert "last poll: HTTP 0" in (res.results[0].error or "") From f883381a4a3eb581314bb8c460298e54ce35ffc2 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Mon, 12 Oct 2026 18:50:50 +0700 Subject: [PATCH 27/70] fix(sdk): validate recall maxTokens and preserve matching total (Fixes #1068) - Add and export validateMaxTokens() to reject negative budgets with RangeError and non-numeric budgets with TypeError before making network requests - Call validateMaxTokens() early in MemWal.recall(), MemWalMock.recall(), and applyTokenBudget() - Preserve matching hit count (processed.total / ranked.length) in recall() results instead of overwriting total with budgeted.length (#1068, #1066) - Add comprehensive unit and e2e tests covering maxTokens validation and total preservation --- packages/sdk/src/index.ts | 1 + packages/sdk/src/memwal.ts | 8 +- packages/sdk/src/mock.ts | 7 +- packages/sdk/src/tokens.ts | 26 +- packages/sdk/test/mock.test.mjs | 2 +- .../recall-max-tokens-validation.test.mjs | 301 ++++++++++++++++++ .../sdk/test/recall-token-budget-e2e.test.mjs | 4 +- 7 files changed, 338 insertions(+), 11 deletions(-) create mode 100644 packages/sdk/test/recall-max-tokens-validation.test.mjs diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 293dc30f0..990cbb500 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -28,6 +28,7 @@ export { estimateTokens, truncateToTokenBudget, applyTokenBudget, + validateMaxTokens, CHARS_PER_TOKEN, } from "./tokens.js"; export type { TokenCounter } from "./tokens.js"; diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index 7a85b1fe6..4ef30270c 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -76,7 +76,7 @@ import { assertCompatibleRelayer, compatibilityErrorFromStatus, } from "./compatibility.js"; -import { applyTokenBudget, estimateTokens } from "./tokens.js"; +import { applyTokenBudget, estimateTokens, validateMaxTokens } from "./tokens.js"; import { pollingDelayMs } from "./polling-delay.js"; import { resolveRecallCall } from "./recall-args.js"; @@ -1036,6 +1036,7 @@ export class MemWal { limitOrOptions, namespace, ); + validateMaxTokens(options.maxTokens); const limit = options.topK ?? options.limit ?? 10; const resolvedNamespace = options.namespace ?? this.namespace; @@ -1075,8 +1076,8 @@ export class MemWal { // Under high-relevance-only, lowest-distance hits are retained and // returned in the caller's requested order (WALM-756). // Omitting maxTokens leaves the result byte-identical to the pre-budget - // behavior (no meta). - if (typeof options.maxTokens === "number") { + // behavior (no meta). Total preserves the matching hit count (#1068). + if (options.maxTokens !== undefined) { const { results: budgeted, meta } = applyTokenBudget( processed.results, options.maxTokens, @@ -1086,7 +1087,6 @@ export class MemWal { processed = { ...processed, results: budgeted, - total: budgeted.length, meta, }; } diff --git a/packages/sdk/src/mock.ts b/packages/sdk/src/mock.ts index ae1c9d43d..009ca135a 100644 --- a/packages/sdk/src/mock.ts +++ b/packages/sdk/src/mock.ts @@ -21,7 +21,7 @@ import type { NamespacesResult, ListNamespacesOptions, } from "./types.js"; -import { applyTokenBudget, estimateTokens } from "./tokens.js"; +import { applyTokenBudget, estimateTokens, validateMaxTokens } from "./tokens.js"; import { resolveRecallCall } from "./recall-args.js"; /** Matches the relayer's `MAX_BULK_ITEMS`. */ @@ -332,6 +332,7 @@ export class MemWalMock { namespace ); validateText(query, "query"); + validateMaxTokens(options.maxTokens); const resolvedNamespace = options.namespace ?? this.namespace; const resolvedLimit = options.topK ?? options.limit ?? 10; if (!Number.isInteger(resolvedLimit) || resolvedLimit < 0) { @@ -374,14 +375,14 @@ export class MemWalMock { distance: memoryDistance, })); - if (typeof options.maxTokens === "number") { + if (options.maxTokens !== undefined) { const { results, meta } = applyTokenBudget( ranked, options.maxTokens, options.truncationStrategy, options.countTokens ); - return { results, total: results.length, meta }; + return { results, total: ranked.length, meta }; } return { results: ranked, total: ranked.length }; diff --git a/packages/sdk/src/tokens.ts b/packages/sdk/src/tokens.ts index abff7fe69..38833c112 100644 --- a/packages/sdk/src/tokens.ts +++ b/packages/sdk/src/tokens.ts @@ -32,6 +32,29 @@ export const CHARS_PER_TOKEN = 4; /** A function that returns the token count of a string. */ export type TokenCounter = (text: string) => number; +/** + * Validate `maxTokens` budget input. + * + * Rejects negative budgets with `RangeError` (a negative budget silently drops + * all memories and reports `truncated: true`, mimicking amnesia). + * Rejects non-numeric budgets with `TypeError` (e.g. `"400"` or `null` which + * would otherwise silently disable budgeting). + * + * `undefined` is allowed (budgeting disabled). + * Non-finite numbers (`NaN` / `Infinity`) pass validation and are treated as + * "no budget" by `applyTokenBudget` per documented contract. + * Zero is allowed (`maxTokens: 0` returns empty results with `truncated: true`). + */ +export function validateMaxTokens(maxTokens: unknown): void { + if (maxTokens === undefined) return; + if (typeof maxTokens !== "number") { + throw new TypeError("maxTokens must be a number"); + } + if (maxTokens < 0) { + throw new RangeError("maxTokens must be non-negative"); + } +} + /** * Estimate the number of tokens in `text` using a documented character-based * approximation (~`chars / 4`). Deterministic and dependency-free. See the @@ -127,6 +150,7 @@ export function applyTokenBudget( strategy: TruncationStrategy = "high-relevance-only", countTokens: TokenCounter = estimateTokens, ): { results: RecallMemory[]; meta: RecallTokenMeta } { + validateMaxTokens(maxTokens); const originalCount = results.length; // A non-finite budget (NaN/Infinity) is not a real budget — never truncate, @@ -135,7 +159,7 @@ export function applyTokenBudget( return { results, meta: { tokenEstimate: payloadTokens(results, countTokens), truncated: false } }; } - // Non-positive budget → empty payload; truncated if there was anything. + // Zero budget → empty payload; truncated if there was anything. if (maxTokens <= 0) { return { results: [], meta: { tokenEstimate: 0, truncated: originalCount > 0 } }; } diff --git a/packages/sdk/test/mock.test.mjs b/packages/sdk/test/mock.test.mjs index 194051cf0..ec424b404 100644 --- a/packages/sdk/test/mock.test.mjs +++ b/packages/sdk/test/mock.test.mjs @@ -133,7 +133,7 @@ test("MemWalMock matches production token-budget behavior", async () => { maxTokens: 10, }); assert.equal(budgeted.results.length, 1); - assert.equal(budgeted.total, 1); + assert.equal(budgeted.total, 2); assert.deepEqual(budgeted.meta, { tokenEstimate: 10, truncated: true, diff --git a/packages/sdk/test/recall-max-tokens-validation.test.mjs b/packages/sdk/test/recall-max-tokens-validation.test.mjs new file mode 100644 index 000000000..22b569b5c --- /dev/null +++ b/packages/sdk/test/recall-max-tokens-validation.test.mjs @@ -0,0 +1,301 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + MemWal, + MemWalMock, + applyTokenBudget, + validateMaxTokens, +} from "../dist/index.js"; + +const originalFetch = globalThis.fetch; + +test.afterEach(() => { + globalThis.fetch = originalFetch; +}); + +function stubRecall(hits) { + globalThis.fetch = async (url, init = {}) => { + const path = new URL(url).pathname; + if (path === "/version") { + return Response.json({ + apiVersion: "1.0.0", + relayerVersion: "1.0.0", + minSupportedSdk: { typescript: "0.0.4" }, + }); + } + if (path === "/api/config" || path === "/config") { + return Response.json({ packageId: "0x1", network: "testnet" }); + } + if (path === "/api/recall" && init.method === "POST") { + return Response.json({ + results: hits, + total: hits.length, + }); + } + throw new Error(`unexpected request ${path}`); + }; +} + +function client() { + const c = MemWal.create({ + key: new Uint8Array(32).fill(1), + accountId: "test-account", + serverUrl: "https://relayer.example", + }); + c.buildSealSession = async () => "test-session"; + return c; +} + +function chars(n) { + return "a".repeat(n); +} + +// ── validateMaxTokens ──────────────────────────────────────────────────────── + +test("validateMaxTokens accepts valid budgets", () => { + assert.doesNotThrow(() => validateMaxTokens(undefined)); + assert.doesNotThrow(() => validateMaxTokens(0)); + assert.doesNotThrow(() => validateMaxTokens(10)); + assert.doesNotThrow(() => validateMaxTokens(1.5)); + assert.doesNotThrow(() => validateMaxTokens(NaN)); + assert.doesNotThrow(() => validateMaxTokens(Infinity)); +}); + +test("validateMaxTokens throws RangeError on negative budgets", () => { + assert.throws( + () => validateMaxTokens(-1), + { + name: "RangeError", + message: "maxTokens must be non-negative", + }, + ); + assert.throws( + () => validateMaxTokens(-5), + { + name: "RangeError", + message: "maxTokens must be non-negative", + }, + ); + assert.throws( + () => validateMaxTokens(-0.5), + { + name: "RangeError", + message: "maxTokens must be non-negative", + }, + ); +}); + +test("validateMaxTokens throws TypeError on non-numeric budgets", () => { + for (const invalid of ["400", null, {}, [], false, true, Symbol("x")]) { + assert.throws( + () => validateMaxTokens(invalid), + { + name: "TypeError", + message: "maxTokens must be a number", + }, + ); + } +}); + +// ── applyTokenBudget validation ────────────────────────────────────────────── + +test("applyTokenBudget validates maxTokens", () => { + const hits = [{ blob_id: "b1", text: "hello world", distance: 0.1 }]; + assert.throws( + () => applyTokenBudget(hits, -5), + { + name: "RangeError", + message: "maxTokens must be non-negative", + }, + ); + assert.throws( + () => applyTokenBudget(hits, "400"), + { + name: "TypeError", + message: "maxTokens must be a number", + }, + ); +}); + +test("applyTokenBudget handles zero budget honestly", () => { + const hits = [ + { blob_id: "b1", text: chars(40), distance: 0.1 }, + { blob_id: "b2", text: chars(40), distance: 0.2 }, + ]; + const budgeted = applyTokenBudget(hits, 0); + assert.equal(budgeted.results.length, 0); + assert.equal(budgeted.meta.truncated, true); + assert.equal(budgeted.meta.tokenEstimate, 0); + + const empty = applyTokenBudget([], 0); + assert.equal(empty.results.length, 0); + assert.equal(empty.meta.truncated, false); + assert.equal(empty.meta.tokenEstimate, 0); +}); + +test("applyTokenBudget treats NaN and Infinity as no budget", () => { + const hits = [ + { blob_id: "b1", text: chars(40), distance: 0.1 }, + { blob_id: "b2", text: chars(40), distance: 0.2 }, + ]; + const forNaN = applyTokenBudget(hits, NaN); + assert.equal(forNaN.results.length, 2); + assert.equal(forNaN.meta.truncated, false); + + const forInf = applyTokenBudget(hits, Infinity); + assert.equal(forInf.results.length, 2); + assert.equal(forInf.meta.truncated, false); +}); + +// ── MemWal.recall() validation & total preservation ───────────────────────── + +test("MemWal.recall() throws RangeError on negative maxTokens before fetch", async () => { + let fetchCalled = false; + globalThis.fetch = async () => { + fetchCalled = true; + return Response.json({}); + }; + + await assert.rejects( + () => client().recall({ query: "test", maxTokens: -5 }), + { + name: "RangeError", + message: "maxTokens must be non-negative", + }, + ); + assert.equal(fetchCalled, false, "network fetch must not be called when maxTokens is invalid"); +}); + +test("MemWal.recall() throws TypeError on non-numeric maxTokens before fetch", async () => { + let fetchCalled = false; + globalThis.fetch = async () => { + fetchCalled = true; + return Response.json({}); + }; + + for (const invalid of ["400", null, {}, false]) { + await assert.rejects( + () => client().recall({ query: "test", maxTokens: invalid }), + { + name: "TypeError", + message: "maxTokens must be a number", + }, + ); + assert.equal(fetchCalled, false, "network fetch must not be called when maxTokens is invalid"); + } +}); + +test("MemWal.recall() preserves relayer total when results are trimmed by token budget", async () => { + const hits = [ + { blob_id: "b1", text: chars(40), distance: 0.1 }, + { blob_id: "b2", text: chars(40), distance: 0.2 }, + { blob_id: "b3", text: chars(40), distance: 0.3 }, + ]; + stubRecall(hits); + + const result = await client().recall({ query: "test", maxTokens: 10 }); + assert.equal(result.results.length, 1, "only 1 memory fits within 10 tokens"); + assert.equal(result.total, 3, "total preserves matching relayer count, not trimmed count"); + assert.ok(result.meta); + assert.equal(result.meta.truncated, true); + assert.equal(result.meta.tokenEstimate, 10); +}); + +test("MemWal.recall() preserves total when maxTokens is 0", async () => { + const hits = [ + { blob_id: "b1", text: chars(40), distance: 0.1 }, + { blob_id: "b2", text: chars(40), distance: 0.2 }, + ]; + stubRecall(hits); + + const result = await client().recall({ query: "test", maxTokens: 0 }); + assert.equal(result.results.length, 0); + assert.equal(result.total, 2, "total preserves 2 matching memories even when 0 returned"); + assert.ok(result.meta); + assert.equal(result.meta.truncated, true); + assert.equal(result.meta.tokenEstimate, 0); +}); + +test("MemWal.recall() with maxTokens NaN or Infinity leaves results unchanged and untruncated", async () => { + const hits = [ + { blob_id: "b1", text: chars(40), distance: 0.1 }, + { blob_id: "b2", text: chars(40), distance: 0.2 }, + ]; + stubRecall(hits); + + const forNaN = await client().recall({ query: "test", maxTokens: NaN }); + assert.equal(forNaN.results.length, 2); + assert.equal(forNaN.total, 2); + assert.equal(forNaN.meta.truncated, false); + + const forInf = await client().recall({ query: "test", maxTokens: Infinity }); + assert.equal(forInf.results.length, 2); + assert.equal(forInf.total, 2); + assert.equal(forInf.meta.truncated, false); +}); + +// ── MemWalMock.recall() validation & total preservation ─────────────────────── + +test("MemWalMock.recall() throws RangeError on negative maxTokens", async () => { + const mock = MemWalMock.create({ + initialMemories: [{ text: "hello world" }], + }); + + await assert.rejects( + () => mock.recall({ query: "hello", maxTokens: -5 }), + { + name: "RangeError", + message: "maxTokens must be non-negative", + }, + ); +}); + +test("MemWalMock.recall() throws TypeError on non-numeric maxTokens", async () => { + const mock = MemWalMock.create({ + initialMemories: [{ text: "hello world" }], + }); + + for (const invalid of ["400", null, {}, false]) { + await assert.rejects( + () => mock.recall({ query: "hello", maxTokens: invalid }), + { + name: "TypeError", + message: "maxTokens must be a number", + }, + ); + } +}); + +test("MemWalMock.recall() preserves total matching count when trimmed by token budget", async () => { + const mock = MemWalMock.create({ + initialMemories: [ + { text: chars(40) }, + { text: chars(40) }, + { text: chars(40) }, + ], + }); + + const result = await mock.recall({ query: "a", limit: 3, maxTokens: 10 }); + assert.equal(result.results.length, 1); + assert.equal(result.total, 3, "mock total preserves matching count of 3"); + assert.ok(result.meta); + assert.equal(result.meta.truncated, true); + assert.equal(result.meta.tokenEstimate, 10); +}); + +test("MemWalMock.recall() preserves total when maxTokens is 0", async () => { + const mock = MemWalMock.create({ + initialMemories: [ + { text: chars(40) }, + { text: chars(40) }, + ], + }); + + const result = await mock.recall({ query: "a", limit: 2, maxTokens: 0 }); + assert.equal(result.results.length, 0); + assert.equal(result.total, 2, "mock total preserves matching count of 2"); + assert.ok(result.meta); + assert.equal(result.meta.truncated, true); + assert.equal(result.meta.tokenEstimate, 0); +}); diff --git a/packages/sdk/test/recall-token-budget-e2e.test.mjs b/packages/sdk/test/recall-token-budget-e2e.test.mjs index dce6cf781..9f7f25367 100644 --- a/packages/sdk/test/recall-token-budget-e2e.test.mjs +++ b/packages/sdk/test/recall-token-budget-e2e.test.mjs @@ -89,7 +89,7 @@ test("recall without maxTokens: response unchanged, NO meta (backward compatible assert.deepEqual(result.results, hits); }); -test("recall with maxTokens: budget applied, total recomputed, meta present", async () => { +test("recall with maxTokens: budget applied, matching total preserved, meta present", async () => { // two 10-token hits (40 chars each); budget 10 → high-relevance-only keeps 1. const hits = [ { blob_id: "b1", text: chars(40), distance: 0.1 }, @@ -99,7 +99,7 @@ test("recall with maxTokens: budget applied, total recomputed, meta present", as const result = await client().recall({ query: "x", maxTokens: 10 }); assert.equal(result.results.length, 1, "second hit dropped to fit budget"); - assert.equal(result.total, 1, "total recomputed to the trimmed count"); + assert.equal(result.total, 2, "total preserves matching count rather than trimmed count"); assert.equal(result.results[0].blob_id, "b1", "kept the lowest-distance hit"); assert.ok(result.meta, "meta attached"); assert.equal(result.meta.truncated, true); From ba1485e73d81b7ade5cdbf300a25f44c2a51657e Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Mon, 12 Oct 2026 18:51:17 +0700 Subject: [PATCH 28/70] fix(sdk): warn on recall failure in withMemWal and add onRecallError callback (Fixes #1118, WALM-764) --- packages/sdk/src/ai/middleware.ts | 9 +- .../sdk/test/withMemWal-recall-error.test.mjs | 138 ++++++++++++++++++ 2 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 packages/sdk/test/withMemWal-recall-error.test.mjs diff --git a/packages/sdk/src/ai/middleware.ts b/packages/sdk/src/ai/middleware.ts index 776521c25..09e225ea3 100644 --- a/packages/sdk/src/ai/middleware.ts +++ b/packages/sdk/src/ai/middleware.ts @@ -52,6 +52,8 @@ export interface WithMemWalOptions extends MemWalConfig { minRelevance?: number; /** Enable debug logging (default: false) */ debug?: boolean; + /** Optional callback invoked when memory recall fails during transformParams */ + onRecallError?: (error: unknown) => void; } // ============================================================ @@ -128,7 +130,12 @@ export function withMemWal( return { ...params, prompt: enrichedPrompt }; } catch (error) { - log("Memory search failed:", error); + console.warn("[Walrus Memory] Recall failed during transformParams:", error); + try { + options.onRecallError?.(error); + } catch (cbError) { + console.warn("[Walrus Memory] onRecallError callback threw:", cbError); + } return params; } }, diff --git a/packages/sdk/test/withMemWal-recall-error.test.mjs b/packages/sdk/test/withMemWal-recall-error.test.mjs new file mode 100644 index 000000000..65e52dccf --- /dev/null +++ b/packages/sdk/test/withMemWal-recall-error.test.mjs @@ -0,0 +1,138 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { generateText } from "ai"; + +import { MemWal } from "../dist/memwal.js"; +import { withMemWal } from "../dist/ai/middleware.js"; + +const TEST_KEY = "11".repeat(32); +const TEST_ACCOUNT_ID = "0x1"; + +function fakeLanguageModel() { + return { + specificationVersion: "v2", + provider: "test", + modelId: "test-model", + supportedUrls: {}, + doGenerate: async () => ({ + content: [{ type: "text", text: "generated response" }], + finishReason: "stop", + usage: { inputTokens: 5, outputTokens: 5, totalTokens: 10 }, + warnings: [], + }), + doStream: async () => ({ stream: new ReadableStream() }), + }; +} + +test("when recall throws, transformParams emits console.warn by default without debug:true", async () => { + const originalRecall = MemWal.prototype.recall; + const originalWarn = console.warn; + + const simulatedError = new Error("simulated relayer 500 error"); + MemWal.prototype.recall = async () => { + throw simulatedError; + }; + + const warnings = []; + console.warn = (...args) => { + warnings.push(args); + }; + + try { + const model = withMemWal(fakeLanguageModel(), { + key: TEST_KEY, + accountId: TEST_ACCOUNT_ID, + autoSave: false, + }); + + const result = await generateText({ + model, + prompt: "What do you remember about me?", + }); + + assert.equal(result.text, "generated response"); + assert.ok(warnings.length > 0, "expected console.warn to be called"); + const found = warnings.some((w) => + w.some((arg) => typeof arg === "string" && arg.includes("Recall failed during transformParams")) + ); + assert.ok(found, "expected warning mentioning 'Recall failed during transformParams'"); + } finally { + MemWal.prototype.recall = originalRecall; + console.warn = originalWarn; + } +}); + +test("when recall throws, onRecallError callback is triggered with the error", async () => { + const originalRecall = MemWal.prototype.recall; + const originalWarn = console.warn; + + const simulatedError = new Error("simulated rate limit 429"); + MemWal.prototype.recall = async () => { + throw simulatedError; + }; + + let caughtError = null; + console.warn = () => {}; // suppress output during test + + try { + const model = withMemWal(fakeLanguageModel(), { + key: TEST_KEY, + accountId: TEST_ACCOUNT_ID, + autoSave: false, + onRecallError: (err) => { + caughtError = err; + }, + }); + + const result = await generateText({ + model, + prompt: "What do you know?", + }); + + assert.equal(result.text, "generated response"); + assert.equal(caughtError, simulatedError); + } finally { + MemWal.prototype.recall = originalRecall; + console.warn = originalWarn; + } +}); + +test("if onRecallError throws, generation still proceeds", async () => { + const originalRecall = MemWal.prototype.recall; + const originalWarn = console.warn; + + MemWal.prototype.recall = async () => { + throw new Error("recall exploded"); + }; + + const warnings = []; + console.warn = (...args) => { + warnings.push(args); + }; + + try { + const model = withMemWal(fakeLanguageModel(), { + key: TEST_KEY, + accountId: TEST_ACCOUNT_ID, + autoSave: false, + onRecallError: () => { + throw new Error("handler exploded"); + }, + }); + + const result = await generateText({ + model, + prompt: "What is my name?", + }); + + assert.equal(result.text, "generated response"); + const callbackWarn = warnings.some((w) => + w.some((arg) => typeof arg === "string" && arg.includes("onRecallError callback threw")) + ); + assert.ok(callbackWarn, "expected warning about failing callback"); + } finally { + MemWal.prototype.recall = originalRecall; + console.warn = originalWarn; + } +}); From 18393d69f20753061c225fba34c60b444b989632 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Mon, 12 Oct 2026 19:55:04 +0700 Subject: [PATCH 29/70] fix(python-sdk): wrap docstring and tighten mock-compat TypeError inspection in poll loop --- packages/python-sdk-memwal/memwal/client.py | 28 +++++++++++++------ .../tests/test_transport_retry.py | 5 +++- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/packages/python-sdk-memwal/memwal/client.py b/packages/python-sdk-memwal/memwal/client.py index a83cb9c5c..b686b55b5 100644 --- a/packages/python-sdk-memwal/memwal/client.py +++ b/packages/python-sdk-memwal/memwal/client.py @@ -511,13 +511,16 @@ async def wait_for_remember_job( accepted_statuses=(200, 404), timeout=poll_timeout, ) - except TypeError: - data = await self._signed_request( - "GET", - f"/api/remember/{job_id}", - {}, - accepted_statuses=(200, 404), - ) + except TypeError as err: + if "unexpected keyword argument" in str(err) and "timeout" in str(err): + data = await self._signed_request( + "GET", + f"/api/remember/{job_id}", + {}, + accepted_statuses=(200, 404), + ) + else: + raise except _HttpStatusError as err: if _is_transient_polling_status(err.status): last_refusal = err.status @@ -527,6 +530,8 @@ async def wait_for_remember_job( continue raise except httpx.TransportError: + # Network drops retry as status 0; continuing next loop iteration + # increments `attempt` so exponential backoff delay continues to grow. last_refusal = 0 continue @@ -749,8 +754,11 @@ def settle(item: RememberBulkStatusItem) -> bool: poll_timeout = self._poll_deadline_s(deadline_ms) try: batch = await self.get_remember_bulk_status(pending, timeout=poll_timeout) - except TypeError: - batch = await self.get_remember_bulk_status(pending) + except TypeError as err: + if "unexpected keyword argument" in str(err) and "timeout" in str(err): + batch = await self.get_remember_bulk_status(pending) + else: + raise except _HttpStatusError as err: if _is_transient_polling_status(err.status): last_refusal = err.status @@ -760,6 +768,8 @@ def settle(item: RememberBulkStatusItem) -> bool: continue raise except httpx.TransportError: + # Network drops retry as status 0; continuing next loop iteration + # increments `attempt` so exponential backoff delay continues to grow. last_refusal = 0 continue answered = True diff --git a/packages/python-sdk-memwal/tests/test_transport_retry.py b/packages/python-sdk-memwal/tests/test_transport_retry.py index e3356337c..b8cacb1ba 100644 --- a/packages/python-sdk-memwal/tests/test_transport_retry.py +++ b/packages/python-sdk-memwal/tests/test_transport_retry.py @@ -1,4 +1,7 @@ -"""Tests for network transport error retry and deadline bounding during polling (WALM-696, GH #1000).""" +"""Tests for network transport error retry and deadline bounding. + +Covers WALM-696 and GH #1000. +""" from __future__ import annotations From 219f6b086fafb5856225d01a945084ee693d8e0a Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Mon, 12 Oct 2026 19:55:52 +0700 Subject: [PATCH 30/70] fix(python-sdk): fix ruff import sorting/unused imports and document wall clock usage in _http_date_retry_ms --- packages/python-sdk-memwal/memwal/client.py | 2 ++ .../python-sdk-memwal/tests/test_retry_after_http_date.py | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/python-sdk-memwal/memwal/client.py b/packages/python-sdk-memwal/memwal/client.py index 0f8292380..713fab072 100644 --- a/packages/python-sdk-memwal/memwal/client.py +++ b/packages/python-sdk-memwal/memwal/client.py @@ -222,6 +222,8 @@ def _http_date_retry_ms(raw: object) -> Optional[int]: if dt.tzinfo is None: dt = dt.replace(tzinfo=timezone.utc) target_ts = dt.timestamp() + # HTTP-date is absolute wall-clock time, so we deliberately compare against + # time.time() rather than monotonic _now_ms(). now_ts = time.time() diff = target_ts - now_ts if diff <= 0 or math.isnan(diff) or math.isinf(diff): diff --git a/packages/python-sdk-memwal/tests/test_retry_after_http_date.py b/packages/python-sdk-memwal/tests/test_retry_after_http_date.py index 4678f8a5b..3983e5af9 100644 --- a/packages/python-sdk-memwal/tests/test_retry_after_http_date.py +++ b/packages/python-sdk-memwal/tests/test_retry_after_http_date.py @@ -5,16 +5,14 @@ import time from datetime import datetime, timedelta, timezone from email.utils import format_datetime -from types import SimpleNamespace from typing import Optional import pytest from memwal.client import ( MemWal, - MemWalRememberJobTimeout, - _HttpStatusError, _clamped_retry_after_ms, + _HttpStatusError, _positive_retry_ms, _retry_after_ms, ) From 644cb10fd61c487f792a69174fc7a04da456dd33 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Mon, 12 Oct 2026 19:57:44 +0700 Subject: [PATCH 31/70] fix(python-sdk): support LangChain multimodal system message, copy model attributes, and fix ruff line lengths --- .../python-sdk-memwal/memwal/middleware.py | 25 ++++++++--- .../tests/test_middleware.py | 44 +++++++++++++++++++ 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/packages/python-sdk-memwal/memwal/middleware.py b/packages/python-sdk-memwal/memwal/middleware.py index 649ce27f3..953d8523b 100644 --- a/packages/python-sdk-memwal/memwal/middleware.py +++ b/packages/python-sdk-memwal/memwal/middleware.py @@ -374,9 +374,22 @@ async def _inject_memories(messages: List[BaseMessage]) -> List[BaseMessage]: if result and isinstance(result[0], SystemMessage): existing_content = result[0].content - result[0] = SystemMessage( - content=f"{existing_content}\n\n{_UNTRUSTED_MEMORY_SYSTEM_INSTRUCTION}".strip() - ) + if isinstance(existing_content, list): + new_content = list(existing_content) + [ + {"type": "text", "text": f"\n\n{_UNTRUSTED_MEMORY_SYSTEM_INSTRUCTION}"} + ] + else: + text_str = str(existing_content or "") + new_content = f"{text_str}\n\n{_UNTRUSTED_MEMORY_SYSTEM_INSTRUCTION}".strip() + if hasattr(result[0], "model_copy"): + result[0] = result[0].model_copy(update={"content": new_content}) + else: + result[0] = SystemMessage( + content=new_content, + additional_kwargs=getattr(result[0], "additional_kwargs", {}), + id=getattr(result[0], "id", None), + name=getattr(result[0], "name", None), + ) else: result.insert(0, SystemMessage(content=_UNTRUSTED_MEMORY_SYSTEM_INSTRUCTION)) @@ -633,7 +646,7 @@ def _inject_openai_memory( memory_context: str, ) -> List[Dict[str, Any]]: """Insert a fixed system guard at index 0 and memory content in a user message.""" - result = [dict(m) for m in messages] + result = [dict(m) if isinstance(m, dict) else m for m in messages] last_user_idx = -1 for i in range(len(result) - 1, -1, -1): if isinstance(result[i], dict) and result[i].get("role") == "user": @@ -651,7 +664,9 @@ def _inject_openai_memory( if isinstance(existing_content, list): result[0] = { **result[0], - "content": existing_content + [{"type": "text", "text": f"\n\n{_UNTRUSTED_MEMORY_SYSTEM_INSTRUCTION}"}], + "content": existing_content + [ + {"type": "text", "text": f"\n\n{_UNTRUSTED_MEMORY_SYSTEM_INSTRUCTION}"} + ], } else: text_str = str(existing_content or "") diff --git a/packages/python-sdk-memwal/tests/test_middleware.py b/packages/python-sdk-memwal/tests/test_middleware.py index f58e5fec3..c02fbf7bd 100644 --- a/packages/python-sdk-memwal/tests/test_middleware.py +++ b/packages/python-sdk-memwal/tests/test_middleware.py @@ -413,6 +413,50 @@ async def _capture_agenerate(messages_batch, *a, **kw): assert isinstance(injected[4], HumanMessage) assert injected[4].content == "What do I like?" + @respx.mock + async def test_multimodal_system_message_appends_block_and_preserves_metadata(self) -> None: + """A list-content SystemMessage appends a text block and retains its metadata.""" + _mock_seal_session_prereqs() + from langchain_core.messages import AIMessage, HumanMessage, SystemMessage + from langchain_core.outputs import ChatGeneration, ChatResult + + llm = self._make_llm() + captured: list = [] + + async def _capture_agenerate(messages_batch, *a, **kw): + captured.extend(messages_batch) + return ChatResult( + generations=[ChatGeneration(message=AIMessage(content="ok"))] + ) + + llm._agenerate = _capture_agenerate + + respx.post(_RECALL_URL).mock( + return_value=_mock_recall([ + {"blob_id": "b1", "text": "User loves coffee", "distance": 0.05} + ]) + ) + respx.post(_ANALYZE_URL).mock(return_value=_mock_analyze()) + + smart_llm = with_memwal_langchain( + llm, key=_KEY_HEX, account_id=_ACCOUNT_ID, server_url=_SERVER, auto_save=False + ) + blocks = [ + {"type": "text", "text": "Base instructions", "cache_control": {"type": "ephemeral"}} + ] + sys_msg = SystemMessage(content=blocks, id="sys-123", additional_kwargs={"source": "test"}) + await smart_llm._agenerate([[sys_msg, HumanMessage("What do I like?")]]) + + assert len(captured) == 1 + injected = captured[0] + assert isinstance(injected[0], SystemMessage) + assert injected[0].id == "sys-123" + assert injected[0].additional_kwargs == {"source": "test"} + assert isinstance(injected[0].content, list) + assert len(injected[0].content) == 2 + assert injected[0].content[0] == blocks[0] + assert "untrusted data" in injected[0].content[1]["text"] + @respx.mock async def test_no_memories_found_no_injection(self) -> None: """When no memories match, the message list is passed unchanged.""" From b94097990ccc02605cbf03043e99b111af8df04a Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Mon, 12 Oct 2026 19:59:48 +0700 Subject: [PATCH 32/70] fix(sdk): validate bulk idempotencyKey on MemWalMock and skip prefix length check on derived items --- packages/sdk/src/mock.ts | 8 +++++-- .../test/remember-bulk-idempotency.test.mjs | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/packages/sdk/src/mock.ts b/packages/sdk/src/mock.ts index 55e675e72..74bf52b52 100644 --- a/packages/sdk/src/mock.ts +++ b/packages/sdk/src/mock.ts @@ -237,6 +237,7 @@ export class MemWalMock { validateText(item?.text, `items[${i}].text`); }); + validateIdempotencyKey(options.idempotencyKey); const normalised = items.map((item) => ({ text: item.text, namespace: item.namespace ?? this.namespace, @@ -245,7 +246,7 @@ export class MemWalMock { const bulkKey = options.idempotencyKey ?? (await derivedBulkIdempotencyKey(requestIdentity)); const jobIds = normalised.map( (item, index) => - this.store(item.text, item.namespace, undefined, `bulk:${bulkKey}:${index}`).jobId + this.store(item.text, item.namespace, undefined, `bulk:${bulkKey}:${index}`, true).jobId ); return { job_ids: jobIds, total: jobIds.length, status: "done" }; } @@ -591,9 +592,12 @@ export class MemWalMock { namespace?: string, blobId?: string, idempotencyKey?: string, + skipKeyValidation: boolean = false, ): MockMemory { validateText(text); - validateIdempotencyKey(idempotencyKey); + if (!skipKeyValidation) { + validateIdempotencyKey(idempotencyKey); + } const resolvedNamespace = namespace ?? this.namespace; if (!resolvedNamespace) throw new Error("namespace cannot be empty"); if (idempotencyKey !== undefined) { diff --git a/packages/sdk/test/remember-bulk-idempotency.test.mjs b/packages/sdk/test/remember-bulk-idempotency.test.mjs index 140fec30c..ad7d3c51f 100644 --- a/packages/sdk/test/remember-bulk-idempotency.test.mjs +++ b/packages/sdk/test/remember-bulk-idempotency.test.mjs @@ -148,3 +148,26 @@ test("MemWalMock derives stable idempotency key for identical items when idempot const recall = await mock.recall({ query: "derived", limit: 10 }); assert.equal(recall.total, 2); }); + +test("MemWalMock validates caller bulk idempotencyKey and allows valid keys up to 255 bytes", async () => { + const mock = MemWalMock.create(); + const items = [{ text: "fact 1" }, { text: "fact 2" }]; + + // Empty key should reject + await assert.rejects( + () => mock.rememberBulk(items, { idempotencyKey: "" }), + /idempotency_key cannot be empty/, + ); + + // 250-byte key should succeed on mock (matching relayer) despite derived item key being >255 bytes + const validLongKey = "k".repeat(250); + const result = await mock.rememberBulk(items, { idempotencyKey: validLongKey }); + assert.equal(result.job_ids.length, 2); + + // > 255-byte key should reject + await assert.rejects( + () => mock.rememberBulk(items, { idempotencyKey: "k".repeat(256) }), + /idempotency_key exceeds maximum length of 255 bytes/, + ); +}); + From ad444ed41b3bb1f92020bd4f8df5cf400bf46302 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Mon, 12 Oct 2026 20:05:15 +0700 Subject: [PATCH 33/70] test(python-sdk): freeze time in HTTP-date retry test to eliminate subsecond truncation race --- .../tests/test_retry_after_http_date.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/python-sdk-memwal/tests/test_retry_after_http_date.py b/packages/python-sdk-memwal/tests/test_retry_after_http_date.py index 3983e5af9..bd4c903a6 100644 --- a/packages/python-sdk-memwal/tests/test_retry_after_http_date.py +++ b/packages/python-sdk-memwal/tests/test_retry_after_http_date.py @@ -2,7 +2,6 @@ from __future__ import annotations -import time from datetime import datetime, timedelta, timezone from email.utils import format_datetime from typing import Optional @@ -122,7 +121,9 @@ def test_clamped_retry_after_ms_with_http_date(monkeypatch: pytest.MonkeyPatch) async def test_wait_for_remember_job_honors_http_date_retry_after( monkeypatch: pytest.MonkeyPatch, ) -> None: - clock = {"now": int(time.time() * 1000)} + fixed_ts = 1700000000.0 + monkeypatch.setattr("time.time", lambda: fixed_ts) + clock = {"now": int(fixed_ts * 1000)} slept: list[int] = [] def fake_now_ms() -> int: @@ -139,7 +140,7 @@ async def fake_sleep_ms(ms: int) -> None: calls = {"n": 0} # 429 Retry-After is 2 seconds into future - future_dt = datetime.now(timezone.utc) + timedelta(seconds=2) + future_dt = datetime.fromtimestamp(fixed_ts, tz=timezone.utc) + timedelta(seconds=2) http_date = format_datetime(future_dt, usegmt=True) async def status(*_args: object, **_kwargs: object) -> dict: @@ -153,7 +154,7 @@ async def status(*_args: object, **_kwargs: object) -> dict: res = await client.wait_for_remember_job("job-1", timeout_ms=10_000) assert res.blob_id == "b-1" assert calls["n"] == 2 - # First sleep is attempt 0 (0 ms), second sleep should be ~2000 ms from Retry-After + # First sleep is attempt 0 (0 ms), second sleep should be 2000 ms from Retry-After assert len(slept) == 2 assert slept[0] == 0 - assert 1500 <= slept[1] <= 2500 + assert slept[1] == 2000 From 1a5d17e393e0c9074da150706bbf318ec6f37f54 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl <267878750+harrymove-ctrl@users.noreply.github.com> Date: Mon, 12 Oct 2026 20:28:44 +0700 Subject: [PATCH 34/70] fix(python-sdk): derive content idempotency key in remember (Fixes #1094, WALM-800) Derive content-based idempotency keys r1-{sha256(bucket \0 namespace \0 text)} matching the TypeScript SDK specification (30-minute bucket window) when idempotency_key is omitted in remember() and remember_and_wait(). Previously, Python fell back to str(uuid.uuid4()) on each completed write or new client session, causing identical writes to mint fresh paid blobs rather than collapsing onto the existing job. --- packages/python-sdk-memwal/memwal/client.py | 38 ++- .../tests/test_remember_derived_key.py | 250 ++++++++++++++++++ 2 files changed, 281 insertions(+), 7 deletions(-) create mode 100644 packages/python-sdk-memwal/tests/test_remember_derived_key.py diff --git a/packages/python-sdk-memwal/memwal/client.py b/packages/python-sdk-memwal/memwal/client.py index a8416d9ec..1623405b9 100644 --- a/packages/python-sdk-memwal/memwal/client.py +++ b/packages/python-sdk-memwal/memwal/client.py @@ -27,6 +27,7 @@ import asyncio import base64 +import hashlib import json import logging import random @@ -134,6 +135,25 @@ def _reject_blank_text(text: object, message: str = "Text cannot be empty") -> N raise ValueError(message) +IDEMPOTENCY_BUCKET_MS = 30 * 60 * 1000 + + +def _derived_idempotency_key(request_identity: str) -> str: + """Window over which the same (namespace, text) resolves to one key. + + `_pending_remember_keys` only dedupes retries that reuse one client instance, + and callers such as sidecars or serverless functions build a fresh client + per session — so a random key lets a replay mint a second paid blob. The + bucket bounds the collapse: `remember_jobs` rows are never pruned, so an + unbucketed key would dedupe against a job from any point in history. + + Matches TypeScript SDK derivedIdempotencyKey contract. + """ + bucket = int(time.time() * 1000) // IDEMPOTENCY_BUCKET_MS + payload = f"{bucket}\0{request_identity}".encode("utf-8") + return f"r1-{hashlib.sha256(payload).hexdigest()}" + + def normalize_server_url(url: str) -> str: """Strip a trailing slash and warn on plaintext HTTP to a remote host. @@ -431,9 +451,11 @@ async def remember( resolved_namespace = namespace or self._namespace request_identity = f"{resolved_namespace}\0{text}" generated_key = idempotency_key is None - resolved_key = idempotency_key or self._pending_remember_keys.get(request_identity) - if resolved_key is None: - resolved_key = str(uuid.uuid4()) + resolved_key = ( + idempotency_key + or self._pending_remember_keys.get(request_identity) + or _derived_idempotency_key(request_identity) + ) if generated_key: self._pending_remember_keys[request_identity] = resolved_key @@ -558,12 +580,15 @@ async def remember_and_wait( ``failed``). Mirrors TS ``rememberAndWait``. """ + _reject_blank_text(text) resolved_namespace = namespace or self._namespace request_identity = f"{resolved_namespace}\0{text}" generated_key = idempotency_key is None - resolved_key = idempotency_key or self._pending_remember_keys.get(request_identity) - if resolved_key is None: - resolved_key = str(uuid.uuid4()) + resolved_key = ( + idempotency_key + or self._pending_remember_keys.get(request_identity) + or _derived_idempotency_key(request_identity) + ) if generated_key: self._pending_remember_keys[request_identity] = resolved_key @@ -1535,7 +1560,6 @@ async def _signed_request( - ``x-account-id``: Walrus Memory account object ID - ``Content-Type``: application/json """ - import uuid await self._ensure_compatible_relayer() diff --git a/packages/python-sdk-memwal/tests/test_remember_derived_key.py b/packages/python-sdk-memwal/tests/test_remember_derived_key.py new file mode 100644 index 000000000..0af53a658 --- /dev/null +++ b/packages/python-sdk-memwal/tests/test_remember_derived_key.py @@ -0,0 +1,250 @@ +""" +Tests for derived idempotency keys in Python SDK (GH #1094, WALM-800). + +Validates parity with TypeScript SDK specification: +- An omitted idempotency_key is derived from bucket + namespace + text: + r1-{sha256(bucket \0 namespace \0 text)} +- Repeated identical writes reuse the same idempotency key across client instances +- Different texts or namespaces produce distinct keys +- Explicit idempotency keys are respected +- MemWalSync synchronous wrapper inherits the same behavior +""" + +import hashlib +import json +import time +from unittest.mock import patch + +import httpx +import pytest +import respx + +from memwal.client import ( + IDEMPOTENCY_BUCKET_MS, + MemWal, + MemWalRememberJobTimeout, + MemWalSync, + _derived_idempotency_key, +) +from memwal.utils import bytes_to_hex + +_TEST_KEY_HEX = bytes_to_hex(b"\x01" * 32) +_TEST_ACCOUNT_ID = "0xabc123" +_TEST_SERVER = "http://localhost:8000" + + +def _make_client(namespace: str = "default") -> MemWal: + client = MemWal.create( + key=_TEST_KEY_HEX, + account_id=_TEST_ACCOUNT_ID, + server_url=_TEST_SERVER, + namespace=namespace, + ) + # Cache a valid seal session (expiry in milliseconds) to avoid needing live Sui RPC + client._session_cache = ("fake-seal-session", int(time.time() * 1000) + 3_600_000) + return client + + +def _mock_version_and_config(): + respx.get(f"{_TEST_SERVER}/version").mock( + return_value=httpx.Response( + 200, + json={ + "apiVersion": "1.0.0", + "relayerVersion": "1.0.0", + "minSupportedSdk": {"python": "0.1.0"}, + }, + ) + ) + respx.get(f"{_TEST_SERVER}/config").mock( + return_value=httpx.Response( + 200, + json={"packageId": "0x1", "network": "testnet"}, + ) + ) + + +def test_derived_idempotency_key_format_and_cross_language_parity(): + """Verify byte-for-byte SHA256 digest parity with TypeScript SDK.""" + bucket = 123456 + request_identity = "default\0I am allergic to peanuts" + + # Pre-calculated reference matching TypeScript: + # r1-${sha256("123456\0default\0I am allergic to peanuts")} + expected_digest = hashlib.sha256(f"{bucket}\0{request_identity}".encode("utf-8")).hexdigest() + expected_key = f"r1-{expected_digest}" + assert expected_key == "r1-8690455a552eca1cc0d77d60b2a7c2a05a2dd836874f5106b1c9967dc5adc083" + + with patch("time.time", return_value=(bucket * IDEMPOTENCY_BUCKET_MS) / 1000.0): + actual_key = _derived_idempotency_key(request_identity) + assert actual_key == expected_key + assert actual_key.startswith("r1-") + assert len(actual_key) == 3 + 64 + + +@respx.mock +@pytest.mark.asyncio +async def test_same_fact_reuses_one_idempotency_key_across_client_instances(): + """Two instances build what sidecars or serverless functions run on reconnect. + + A per-instance random uuid4 made the replay look like a new write, so the + relayer minted a second paid Walrus blob. The content-based derived key + ensures both send the identical key. + """ + _mock_version_and_config() + posted_bodies = [] + + def handle_remember(request: httpx.Request) -> httpx.Response: + posted_bodies.append(json.loads(request.content)) + return httpx.Response(202, json={"job_id": "job-1", "status": "pending"}) + + respx.post(f"{_TEST_SERVER}/api/remember").mock(side_effect=handle_remember) + respx.get(f"{_TEST_SERVER}/api/remember/job-1").mock( + return_value=httpx.Response( + 200, + json={ + "job_id": "job-1", + "status": "done", + "blob_id": "blob-1", + "owner": _TEST_ACCOUNT_ID, + "namespace": "default", + }, + ) + ) + + client1 = _make_client() + await client1.remember_and_wait("a durable fact") + + client2 = _make_client() + await client2.remember_and_wait("a durable fact") + + assert len(posted_bodies) == 2 + assert posted_bodies[0]["idempotency_key"] == posted_bodies[1]["idempotency_key"] + assert posted_bodies[0]["idempotency_key"].startswith("r1-") + + +@respx.mock +@pytest.mark.asyncio +async def test_remember_async_reuses_derived_key(): + """remember() (async accept) also derives the content key.""" + _mock_version_and_config() + posted_bodies = [] + + def handle_remember(request: httpx.Request) -> httpx.Response: + posted_bodies.append(json.loads(request.content)) + job_id = f"job-{len(posted_bodies)}" + return httpx.Response(202, json={"job_id": job_id, "status": "pending"}) + + respx.post(f"{_TEST_SERVER}/api/remember").mock(side_effect=handle_remember) + + client = _make_client() + await client.remember("user likes pizza") + await client.remember("user likes pizza") + + assert len(posted_bodies) == 2 + assert posted_bodies[0]["idempotency_key"] == posted_bodies[1]["idempotency_key"] + + +@respx.mock +@pytest.mark.asyncio +async def test_different_text_and_namespaces_get_different_keys(): + """Distinct writes must not collapse onto one job.""" + _mock_version_and_config() + posted_bodies = [] + + def handle_remember(request: httpx.Request) -> httpx.Response: + posted_bodies.append(json.loads(request.content)) + job_id = f"job-{len(posted_bodies)}" + return httpx.Response(202, json={"job_id": job_id, "status": "pending"}) + + respx.post(f"{_TEST_SERVER}/api/remember").mock(side_effect=handle_remember) + + client = _make_client() + await client.remember("fact one") + await client.remember("fact two") + await client.remember("fact one", namespace="other-namespace") + + keys = [b["idempotency_key"] for b in posted_bodies] + assert len(set(keys)) == 3, "distinct writes must produce distinct keys" + + +@respx.mock +@pytest.mark.asyncio +async def test_explicit_idempotency_key_still_wins(): + """When the caller passes an explicit key, that key is sent directly.""" + _mock_version_and_config() + posted_bodies = [] + + def handle_remember(request: httpx.Request) -> httpx.Response: + posted_bodies.append(json.loads(request.content)) + return httpx.Response(202, json={"job_id": "job-explicit", "status": "pending"}) + + respx.post(f"{_TEST_SERVER}/api/remember").mock(side_effect=handle_remember) + + client = _make_client() + await client.remember("a fact", idempotency_key="caller-owned-key") + + assert len(posted_bodies) == 1 + assert posted_bodies[0]["idempotency_key"] == "caller-owned-key" + + +@respx.mock +@pytest.mark.asyncio +async def test_remember_and_wait_preserves_key_on_polling_timeout(): + """If polling times out, the pending key stays in the map for immediate retry.""" + _mock_version_and_config() + posted_bodies = [] + + def handle_remember(request: httpx.Request) -> httpx.Response: + posted_bodies.append(json.loads(request.content)) + return httpx.Response(202, json={"job_id": "slow-job", "status": "pending"}) + + respx.post(f"{_TEST_SERVER}/api/remember").mock(side_effect=handle_remember) + respx.get(f"{_TEST_SERVER}/api/remember/slow-job").mock( + return_value=httpx.Response(200, json={"job_id": "slow-job", "status": "running"}) + ) + + client = _make_client() + with pytest.raises(MemWalRememberJobTimeout): + await client.remember_and_wait("timeout fact", poll_interval_ms=10, timeout_ms=50) + + # Key must still be in pending map + request_identity = "default\0timeout fact" + assert request_identity in client._pending_remember_keys + pending_key = client._pending_remember_keys[request_identity] + + # Retry uses the exact same pending key + with pytest.raises(MemWalRememberJobTimeout): + await client.remember_and_wait("timeout fact", poll_interval_ms=10, timeout_ms=50) + + assert len(posted_bodies) == 2 + assert posted_bodies[0]["idempotency_key"] == pending_key + assert posted_bodies[1]["idempotency_key"] == pending_key + + +@respx.mock +def test_sync_client_uses_derived_key(): + """MemWalSync synchronous client derives and reuses content keys.""" + _mock_version_and_config() + posted_bodies = [] + + def handle_remember(request: httpx.Request) -> httpx.Response: + posted_bodies.append(json.loads(request.content)) + return httpx.Response(202, json={"job_id": "job-sync", "status": "pending"}) + + respx.post(f"{_TEST_SERVER}/api/remember").mock(side_effect=handle_remember) + + sync_client = MemWalSync.create( + key=_TEST_KEY_HEX, + account_id=_TEST_ACCOUNT_ID, + server_url=_TEST_SERVER, + namespace="default", + ) + sync_client._inner._session_cache = ("fake-seal-session", int(time.time() * 1000) + 3_600_000) + + sync_client.remember("sync fact") + sync_client.remember("sync fact") + + assert len(posted_bodies) == 2 + assert posted_bodies[0]["idempotency_key"] == posted_bodies[1]["idempotency_key"] + assert posted_bodies[0]["idempotency_key"].startswith("r1-") From e9333276dfa305bf94d237e8b2a3fe454c83ba89 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Mon, 12 Oct 2026 23:14:59 +0700 Subject: [PATCH 35/70] fix(sdk): reject an empty namespace at construction MemWalMock already throws "namespace cannot be empty". MemWal and MemWalManual treated "" as a real namespace and sent it on the wire. Omitted namespace still defaults to "default". Fixes #1204 --- packages/sdk/src/manual.ts | 3 +++ packages/sdk/src/memwal.ts | 3 +++ packages/sdk/test/empty-namespace.test.mjs | 31 ++++++++++++++++++++++ 3 files changed, 37 insertions(+) create mode 100644 packages/sdk/test/empty-namespace.test.mjs diff --git a/packages/sdk/src/manual.ts b/packages/sdk/src/manual.ts index d7eb715d6..d8e653e00 100644 --- a/packages/sdk/src/manual.ts +++ b/packages/sdk/src/manual.ts @@ -181,6 +181,9 @@ export class MemWalManual { this.serverUrl = normalizeServerUrl(config.serverUrl ?? "https://relayer.memory.walrus.xyz"); this.walletSigner = config.walletSigner ?? null; this.config = config; + if (config.namespace !== undefined && !config.namespace) { + throw new Error("namespace cannot be empty"); + } this.namespace = config.namespace ?? "default"; } diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index 074c63a68..2730e1868 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -465,6 +465,9 @@ export class MemWal { // warns (does not throw) if a user passes plain http:// for a // non-localhost host. this.serverUrl = normalizeServerUrl(config.serverUrl ?? "https://relayer.memory.walrus.xyz"); + if (config.namespace !== undefined && !config.namespace) { + throw new Error("namespace cannot be empty"); + } this.namespace = config.namespace ?? "default"; // A non-positive or non-finite override would disable the backstop // entirely, which is the bug this exists to prevent. diff --git a/packages/sdk/test/empty-namespace.test.mjs b/packages/sdk/test/empty-namespace.test.mjs new file mode 100644 index 000000000..b916e671b --- /dev/null +++ b/packages/sdk/test/empty-namespace.test.mjs @@ -0,0 +1,31 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { MemWal } from "../dist/memwal.js"; +import { MemWalManual } from "../dist/manual.js"; + +const KEY = "11".repeat(32); + +test("MemWal.create rejects an empty namespace and still defaults when omitted", () => { + assert.throws( + () => MemWal.create({ key: KEY, accountId: "0x1", namespace: "" }), + /namespace cannot be empty/, + ); + assert.ok(MemWal.create({ key: KEY, accountId: "0x1" })); +}); + +test("MemWalManual.create rejects an empty namespace and still defaults when omitted", () => { + const base = { + key: KEY, + walletSigner: { address: "0x2", signPersonalMessage: async () => ({ signature: "aa" }) }, + embeddingApiKey: "test-openai-key", + packageId: "0x1", + registryId: "0x2", + accountId: "0x3", + }; + assert.throws( + () => MemWalManual.create({ ...base, namespace: "" }), + /namespace cannot be empty/, + ); + assert.ok(MemWalManual.create(base)); +}); From 552872440b30a44f1f26361b4b39ca3d3ada71d9 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Mon, 12 Oct 2026 23:15:00 +0700 Subject: [PATCH 36/70] fix(sdk): require a 32-byte Ed25519 seed at construction hexToBytes accepted any even-length hex, so a 40-byte key or a missing key survived MemWal.create and failed later inside the signer. Fixes #1200 --- packages/sdk/src/manual.ts | 6 +++++ packages/sdk/src/memwal.ts | 6 +++++ packages/sdk/test/key-length.test.mjs | 38 +++++++++++++++++++++++++++ 3 files changed, 50 insertions(+) create mode 100644 packages/sdk/test/key-length.test.mjs diff --git a/packages/sdk/src/manual.ts b/packages/sdk/src/manual.ts index d7eb715d6..4645fa005 100644 --- a/packages/sdk/src/manual.ts +++ b/packages/sdk/src/manual.ts @@ -176,6 +176,12 @@ export class MemWalManual { typeof config.key === "string" ? hexToBytes(normalizePrivateKey(config.key)) : config.key; + if (!(this.delegatePrivateKey instanceof Uint8Array) || this.delegatePrivateKey.length !== 32) { + const got = this.delegatePrivateKey instanceof Uint8Array + ? String(this.delegatePrivateKey.length) + : typeof this.delegatePrivateKey; + throw new Error(`Ed25519 seed must be exactly 32 bytes, got ${got}`); + } // LOW-22: default to HTTPS; warn (do not throw) on plaintext HTTP // against non-localhost hosts. this.serverUrl = normalizeServerUrl(config.serverUrl ?? "https://relayer.memory.walrus.xyz"); diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index 074c63a68..db9d6cc90 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -460,6 +460,12 @@ export class MemWal { typeof config.key === "string" ? hexToBytes(normalizePrivateKey(config.key)) : config.key; + if (!(this.privateKey instanceof Uint8Array) || this.privateKey.length !== 32) { + const got = this.privateKey instanceof Uint8Array + ? String(this.privateKey.length) + : typeof this.privateKey; + throw new Error(`Ed25519 seed must be exactly 32 bytes, got ${got}`); + } this.accountId = config.accountId; // LOW-22: default to HTTPS for production usage; normalizeServerUrl // warns (does not throw) if a user passes plain http:// for a diff --git a/packages/sdk/test/key-length.test.mjs b/packages/sdk/test/key-length.test.mjs new file mode 100644 index 000000000..9a4d57432 --- /dev/null +++ b/packages/sdk/test/key-length.test.mjs @@ -0,0 +1,38 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { MemWal } from "../dist/memwal.js"; +import { MemWalManual } from "../dist/manual.js"; + +const ACCOUNT = "0x1"; + +test("MemWal.create rejects a missing or wrong-length delegate key", () => { + assert.throws( + () => MemWal.create({ accountId: ACCOUNT }), + /Ed25519 seed must be exactly 32 bytes, got undefined/, + ); + assert.throws( + () => MemWal.create({ key: "ab".repeat(40), accountId: ACCOUNT }), + /Ed25519 seed must be exactly 32 bytes, got 40/, + ); + assert.throws( + () => MemWal.create({ key: new Uint8Array(31), accountId: ACCOUNT }), + /Ed25519 seed must be exactly 32 bytes, got 31/, + ); + assert.ok(MemWal.create({ key: "11".repeat(32), accountId: ACCOUNT })); +}); + +test("MemWalManual.create rejects a wrong-length delegate key", () => { + const base = { + walletSigner: { address: "0x2", signPersonalMessage: async () => ({ signature: "aa" }) }, + embeddingApiKey: "test-openai-key", + packageId: "0x1", + registryId: "0x2", + accountId: "0x3", + }; + assert.throws( + () => MemWalManual.create({ ...base, key: "ab".repeat(40) }), + /Ed25519 seed must be exactly 32 bytes, got 40/, + ); + assert.ok(MemWalManual.create({ ...base, key: new Uint8Array(32) })); +}); From 7ba3fc12be4e75bc06d36bf6a6e69480cf47d37b Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Mon, 12 Oct 2026 23:15:00 +0700 Subject: [PATCH 37/70] fix(sdk): reject a malformed listNamespaces cursor in the mock A cursor that JSON-parsed but was not {updated_at, namespace} made date comparisons NaN and returned an empty page with has_more false, so a pagination walk stopped while namespaces still existed. Fixes #1203 --- packages/sdk/src/mock.ts | 55 ++++++++++++++++--- .../sdk/test/mock-namespace-cursor.test.mjs | 33 +++++++++++ 2 files changed, 81 insertions(+), 7 deletions(-) create mode 100644 packages/sdk/test/mock-namespace-cursor.test.mjs diff --git a/packages/sdk/src/mock.ts b/packages/sdk/src/mock.ts index 6eaf266dd..a0698f0d7 100644 --- a/packages/sdk/src/mock.ts +++ b/packages/sdk/src/mock.ts @@ -130,6 +130,51 @@ function recentCandidateLimit(limit: number): number { return Math.max(Math.min(limit * 5, 50), limit); } +/** Relayer cursors are `{ updated_at, namespace, snapshot_at }`. A decode that + * is not that shape must fail loudly: `NaN` date comparisons otherwise filter + * out every namespace and a `while (has_more)` walk stops on the first page. */ +function decodeListNamespacesCursor(encoded: string): { + updated_at: string; + namespace: string; + snapshot_at?: string | null; +} { + let decoded: unknown; + try { + decoded = JSON.parse(new TextDecoder().decode( + Uint8Array.from( + atob(encoded.replace(/-/g, "+").replace(/_/g, "/")), + (char) => char.charCodeAt(0), + ), + )); + } catch { + throw new Error("invalid namespace cursor"); + } + if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) { + throw new Error("invalid namespace cursor"); + } + const cursor = decoded as { + updated_at?: unknown; + namespace?: unknown; + snapshot_at?: unknown; + }; + const updatedAt = cursor.updated_at; + const namespace = cursor.namespace; + if (typeof updatedAt !== "string" || !Number.isFinite(Date.parse(updatedAt))) { + throw new Error("invalid namespace cursor"); + } + if (typeof namespace !== "string") { + throw new Error("invalid namespace cursor"); + } + const snapshotAt = cursor.snapshot_at; + if (snapshotAt !== undefined && snapshotAt !== null && typeof snapshotAt !== "string") { + throw new Error("invalid namespace cursor"); + } + if (typeof snapshotAt === "string" && !Number.isFinite(Date.parse(snapshotAt))) { + throw new Error("invalid namespace cursor"); + } + return { updated_at: updatedAt, namespace, snapshot_at: snapshotAt }; +} + /** * Deterministic, dependency-free in-memory implementation of the core MemWal API. * It never opens a socket, reads credentials, or contacts Sui/Walrus. Recall uses @@ -494,13 +539,9 @@ export class MemWalMock { ); // Match the relayer's URL_SAFE_NO_PAD JSON cursor and snapshot walk. - const cursor: { updated_at: string; namespace: string; snapshot_at?: string | null } | null = - options.cursor === undefined ? null : JSON.parse(new TextDecoder().decode( - Uint8Array.from( - atob(options.cursor.replace(/-/g, "+").replace(/_/g, "/")), - (char) => char.charCodeAt(0), - ), - )); + const cursor = options.cursor === undefined + ? null + : decodeListNamespacesCursor(options.cursor); const snapshotAt = cursor?.snapshot_at ?? new Date( MOCK_NAMESPACE_EPOCH_MS + this.sequence * 1000, ).toISOString(); diff --git a/packages/sdk/test/mock-namespace-cursor.test.mjs b/packages/sdk/test/mock-namespace-cursor.test.mjs new file mode 100644 index 000000000..559a8e965 --- /dev/null +++ b/packages/sdk/test/mock-namespace-cursor.test.mjs @@ -0,0 +1,33 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { MemWalMock } from "../dist/mock.js"; + +function cursor(value) { + return Buffer.from(JSON.stringify(value)).toString("base64url"); +} + +test("listNamespaces rejects a cursor whose updated_at is not a date", async () => { + const mock = MemWalMock.create(); + await mock.remember("a", "ns-a"); + await mock.remember("b", "ns-b"); + await mock.remember("c", "ns-c"); + + const bad = cursor({ updated_at: "not-a-date", namespace: "ns-a", snapshot_at: null }); + await assert.rejects( + () => mock.listNamespaces({ cursor: bad }), + /invalid namespace cursor/, + ); + + const page = await mock.listNamespaces(); + assert.equal(page.namespaces.length, 3); +}); + +test("listNamespaces rejects a null cursor payload instead of restarting the walk", async () => { + const mock = MemWalMock.create(); + await mock.remember("a", "ns-a"); + await assert.rejects( + () => mock.listNamespaces({ cursor: cursor(null) }), + /invalid namespace cursor/, + ); +}); From 9a18ccca67c5f6bb81116a73cd72149a4e759ac8 Mon Sep 17 00:00:00 2001 From: HoangDucBach Date: Mon, 31 Aug 2026 15:22:57 +0700 Subject: [PATCH 38/70] fix(chatbot): rate-limit unauthenticated guest user creation (WALM-415) GET /api/auth/guest inserted a User row on every call. Throttle before signIn so one IP cannot farm guest rows in local or production. --- .../app/(auth)/api/auth/guest/route.ts | 25 +++ apps/chatbot/lib/ratelimit.ts | 155 +++++++++++++++++- apps/chatbot/lib/ratelimit.unit.test.ts | 141 ++++++++++++++++ 3 files changed, 320 insertions(+), 1 deletion(-) create mode 100644 apps/chatbot/lib/ratelimit.unit.test.ts diff --git a/apps/chatbot/app/(auth)/api/auth/guest/route.ts b/apps/chatbot/app/(auth)/api/auth/guest/route.ts index dd7ec23e4..1c77f4200 100644 --- a/apps/chatbot/app/(auth)/api/auth/guest/route.ts +++ b/apps/chatbot/app/(auth)/api/auth/guest/route.ts @@ -2,6 +2,11 @@ import { NextResponse } from "next/server"; import { signIn } from "@/app/(auth)/auth"; import { publicRequestUrl, safeRedirectPath } from "@/lib/public-request-url"; import { getSessionToken } from "@/lib/session-token"; +import { + checkGuestAuthRateLimit, + GUEST_AUTH_RATE_LIMIT_TTL_SECONDS, + GuestAuthRateLimitError, +} from "@/lib/ratelimit"; export async function GET(request: Request) { const { searchParams } = new URL(request.url); @@ -17,5 +22,25 @@ export async function GET(request: Request) { return NextResponse.redirect(new URL("/", publicUrl)); } + try { + await checkGuestAuthRateLimit(request); + } catch (error) { + if (error instanceof GuestAuthRateLimitError) { + return NextResponse.json( + { error: error.message }, + { + status: error.status, + headers: { + "Retry-After": + error.status === 429 + ? String(GUEST_AUTH_RATE_LIMIT_TTL_SECONDS) + : "5", + }, + } + ); + } + throw error; + } + return signIn("guest", { redirect: true, redirectTo: redirectUrl }); } diff --git a/apps/chatbot/lib/ratelimit.ts b/apps/chatbot/lib/ratelimit.ts index ca417e92b..37319f73c 100644 --- a/apps/chatbot/lib/ratelimit.ts +++ b/apps/chatbot/lib/ratelimit.ts @@ -1,13 +1,43 @@ +import { isIP } from "node:net"; import { createClient } from "redis"; -import { isProductionEnvironment } from "@/lib/constants"; +import { isProductionEnvironment, isTestEnvironment } from "@/lib/constants"; import { ChatbotError } from "@/lib/errors"; const MAX_MESSAGES = 10; const TTL_SECONDS = 60 * 60; +export const GUEST_AUTH_RATE_LIMIT_PER_IP = 5; +export const GUEST_AUTH_RATE_LIMIT_GLOBAL = 60; +export const GUEST_AUTH_RATE_LIMIT_TTL_SECONDS = 15 * 60; + +const GUEST_AUTH_RATE_LIMIT_LUA = ` +local ip_key = KEYS[1] +local global_key = KEYS[2] +local ip_limit = tonumber(ARGV[1]) +local global_limit = tonumber(ARGV[2]) +local ttl = tonumber(ARGV[3]) + +local ip_count = tonumber(redis.call('GET', ip_key) or '0') +local global_count = tonumber(redis.call('GET', global_key) or '0') +if ip_count >= ip_limit or global_count >= global_limit then + return 0 +end + +ip_count = redis.call('INCR', ip_key) +global_count = redis.call('INCR', global_key) +if ip_count == 1 then redis.call('EXPIRE', ip_key, ttl) end +if global_count == 1 then redis.call('EXPIRE', global_key, ttl) end +return 1 +`; + let client: ReturnType | null = null; +const memoryGuestCounters = new Map< + string, + { count: number; expiresAt: number } +>(); + function getClient() { if (!client && process.env.REDIS_URL) { client = createClient({ url: process.env.REDIS_URL }); @@ -19,6 +49,26 @@ function getClient() { return client; } +function validIp(value: string | undefined): string | undefined { + const candidate = value?.trim(); + return candidate && isIP(candidate) ? candidate : undefined; +} + +/** Client IP for abuse controls. Missing/invalid headers share the `unknown` bucket. */ +export function getClientIp(request: Request): string | undefined { + const vercelForwarded = request.headers + .get("x-vercel-forwarded-for") + ?.split(",")[0]; + const realIp = request.headers.get("x-real-ip") ?? undefined; + const forwarded = request.headers.get("x-forwarded-for")?.split(",").at(-1); + + if (process.env.VERCEL) { + return validIp(vercelForwarded); + } + + return validIp(realIp) ?? validIp(forwarded); +} + export async function checkIpRateLimit(ip: string | undefined) { if (!isProductionEnvironment || !ip) { return; @@ -46,3 +96,106 @@ export async function checkIpRateLimit(ip: string | undefined) { } } } + +export class GuestAuthRateLimitError extends Error { + readonly status: 429 | 503; + + constructor(status: 429 | 503) { + super( + status === 429 + ? "Too many guest logins" + : "Guest login limiter unavailable" + ); + this.name = "GuestAuthRateLimitError"; + this.status = status; + } +} + +function memoryCount(key: string): number { + const current = memoryGuestCounters.get(key); + if (!current || current.expiresAt <= Date.now()) { + return 0; + } + return current.count; +} + +function memoryIncr(key: string, ttlMs: number): void { + const now = Date.now(); + const current = memoryGuestCounters.get(key); + if (!current || current.expiresAt <= now) { + memoryGuestCounters.set(key, { count: 1, expiresAt: now + ttlMs }); + return; + } + current.count += 1; +} + +function consumeMemoryGuestSlot(ip: string): boolean { + const ttlMs = GUEST_AUTH_RATE_LIMIT_TTL_SECONDS * 1000; + const ipKey = `guest-auth-rate:ip:${ip}`; + if ( + memoryCount(ipKey) >= GUEST_AUTH_RATE_LIMIT_PER_IP || + memoryCount("guest-auth-rate:global") >= GUEST_AUTH_RATE_LIMIT_GLOBAL + ) { + return false; + } + memoryIncr(ipKey, ttlMs); + memoryIncr("guest-auth-rate:global", ttlMs); + return true; +} + +/** Test-only: reset the process-local guest limiter. */ +export function resetMemoryGuestAuthRateLimit(): void { + memoryGuestCounters.clear(); +} + +/** + * Cap unauthenticated guest User inserts. Always on outside Playwright: + * the chat IP limiter is production-only and a no-op without Redis, which + * is how five local curls each created a row. + * + * Production requires Redis (fail closed). Local/dev uses Redis when ready, + * otherwise a process-local counter so `next dev` is still throttled. + */ +export async function checkGuestAuthRateLimit(request: Request): Promise { + if (isTestEnvironment) { + return; + } + + const ip = getClientIp(request) ?? "unknown"; + + const redis = getClient(); + if (redis?.isReady) { + try { + const result = await redis.eval(GUEST_AUTH_RATE_LIMIT_LUA, { + arguments: [ + String(GUEST_AUTH_RATE_LIMIT_PER_IP), + String(GUEST_AUTH_RATE_LIMIT_GLOBAL), + String(GUEST_AUTH_RATE_LIMIT_TTL_SECONDS), + ], + keys: [ + `guest-auth-rate:{guest}:ip:${ip}`, + "guest-auth-rate:{guest}:global", + ], + }); + if (Number(result) !== 1) { + throw new GuestAuthRateLimitError(429); + } + return; + } catch (error) { + if (error instanceof GuestAuthRateLimitError) { + throw error; + } + if (isProductionEnvironment) { + throw new GuestAuthRateLimitError(503); + } + } + } + + if (isProductionEnvironment) { + throw new GuestAuthRateLimitError(503); + } + + if (!consumeMemoryGuestSlot(ip)) { + throw new GuestAuthRateLimitError(429); + } +} diff --git a/apps/chatbot/lib/ratelimit.unit.test.ts b/apps/chatbot/lib/ratelimit.unit.test.ts new file mode 100644 index 000000000..92e7fe82c --- /dev/null +++ b/apps/chatbot/lib/ratelimit.unit.test.ts @@ -0,0 +1,141 @@ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("redis", () => ({ + createClient: () => ({ + isReady: false, + on: vi.fn(), + connect: vi.fn().mockResolvedValue(undefined), + }), +})); + +import { + checkGuestAuthRateLimit, + GUEST_AUTH_RATE_LIMIT_PER_IP, + GuestAuthRateLimitError, + getClientIp, + resetMemoryGuestAuthRateLimit, +} from "@/lib/ratelimit"; + +function requestWithHeaders(headers?: HeadersInit): Request { + return new Request("http://127.0.0.1:3001/api/auth/guest?redirectUrl=/", { + headers, + }); +} + +describe("getClientIp", () => { + it("prefers x-real-ip over x-forwarded-for", () => { + expect( + getClientIp( + requestWithHeaders({ + "x-real-ip": "203.0.113.10", + "x-forwarded-for": "198.51.100.1, 203.0.113.10", + }) + ) + ).toBe("203.0.113.10"); + }); + + it("uses the right-most x-forwarded-for hop", () => { + expect( + getClientIp( + requestWithHeaders({ + "x-forwarded-for": "198.51.100.1, 203.0.113.8", + }) + ) + ).toBe("203.0.113.8"); + }); + + it("ignores invalid IP values", () => { + expect( + getClientIp( + requestWithHeaders({ + "x-real-ip": "not-an-ip", + "x-forwarded-for": "also-bad", + }) + ) + ).toBeUndefined(); + }); + + it("uses x-vercel-forwarded-for on Vercel", () => { + const previous = process.env.VERCEL; + process.env.VERCEL = "1"; + try { + expect( + getClientIp( + requestWithHeaders({ + "x-vercel-forwarded-for": "203.0.113.9, 198.51.100.2", + "x-real-ip": "203.0.113.10", + }) + ) + ).toBe("203.0.113.9"); + } finally { + process.env.VERCEL = previous ?? ""; + } + }); +}); + +describe("checkGuestAuthRateLimit", () => { + beforeEach(() => { + resetMemoryGuestAuthRateLimit(); + }); + + it("allows a small per-IP budget then returns 429", async () => { + const request = requestWithHeaders({ "x-real-ip": "203.0.113.10" }); + + for (let i = 0; i < GUEST_AUTH_RATE_LIMIT_PER_IP; i++) { + await expect(checkGuestAuthRateLimit(request)).resolves.toBeUndefined(); + } + + await expect(checkGuestAuthRateLimit(request)).rejects.toEqual( + expect.objectContaining({ + name: "GuestAuthRateLimitError", + status: 429, + }) + ); + }); + + it("tracks IPs independently", async () => { + const first = requestWithHeaders({ "x-real-ip": "203.0.113.1" }); + const second = requestWithHeaders({ "x-real-ip": "203.0.113.2" }); + + for (let i = 0; i < GUEST_AUTH_RATE_LIMIT_PER_IP; i++) { + await expect(checkGuestAuthRateLimit(first)).resolves.toBeUndefined(); + } + + await expect(checkGuestAuthRateLimit(second)).resolves.toBeUndefined(); + await expect(checkGuestAuthRateLimit(first)).rejects.toBeInstanceOf( + GuestAuthRateLimitError + ); + }); + + it("buckets missing IPs together as unknown", async () => { + const request = requestWithHeaders(); + + for (let i = 0; i < GUEST_AUTH_RATE_LIMIT_PER_IP; i++) { + await expect(checkGuestAuthRateLimit(request)).resolves.toBeUndefined(); + } + + await expect(checkGuestAuthRateLimit(request)).rejects.toMatchObject({ + status: 429, + }); + }); +}); + +describe("guest auth route", () => { + it("rate-limits before signIn so createGuestUser is not unconditional", () => { + const source = readFileSync( + join( + dirname(fileURLToPath(import.meta.url)), + "../app/(auth)/api/auth/guest/route.ts" + ), + "utf8" + ); + + expect(source).toContain("checkGuestAuthRateLimit"); + expect(source.indexOf("checkGuestAuthRateLimit")).toBeLessThan( + source.indexOf('signIn("guest"') + ); + }); +}); From 2e2883f05a86370bbdecfdc667a4db656c0e1ed8 Mon Sep 17 00:00:00 2001 From: HoangDucBach Date: Mon, 7 Sep 2026 11:19:00 +0700 Subject: [PATCH 39/70] fix(chatbot): consume guest rate limit in authorize and wait for Redis (WALM-415) Guest inserts go through authorize(), not only GET /api/auth/guest. Await Redis connect before fail-closed 503 so cold starts are not blocked. --- .../app/(auth)/api/auth/guest/route.ts | 3 +- apps/chatbot/app/(auth)/auth.ts | 7 + apps/chatbot/lib/ratelimit.ts | 121 +++++++++-- apps/chatbot/lib/ratelimit.unit.test.ts | 188 ++++++++++++++++-- 4 files changed, 282 insertions(+), 37 deletions(-) diff --git a/apps/chatbot/app/(auth)/api/auth/guest/route.ts b/apps/chatbot/app/(auth)/api/auth/guest/route.ts index 1c77f4200..3e3768ba4 100644 --- a/apps/chatbot/app/(auth)/api/auth/guest/route.ts +++ b/apps/chatbot/app/(auth)/api/auth/guest/route.ts @@ -23,7 +23,8 @@ export async function GET(request: Request) { } try { - await checkGuestAuthRateLimit(request); + // Peek only: signIn("guest") runs authorize() in-process, which consumes. + await checkGuestAuthRateLimit(request, { consume: false }); } catch (error) { if (error instanceof GuestAuthRateLimitError) { return NextResponse.json( diff --git a/apps/chatbot/app/(auth)/auth.ts b/apps/chatbot/app/(auth)/auth.ts index ffd72be0b..1b272fb3d 100644 --- a/apps/chatbot/app/(auth)/auth.ts +++ b/apps/chatbot/app/(auth)/auth.ts @@ -2,8 +2,10 @@ import { compare } from "bcrypt-ts"; import NextAuth, { type DefaultSession } from "next-auth"; import type { DefaultJWT } from "next-auth/jwt"; import Credentials from "next-auth/providers/credentials"; +import { headers } from "next/headers"; import { DUMMY_PASSWORD } from "@/lib/constants"; import { createGuestUser, getUser } from "@/lib/db/queries"; +import { checkGuestAuthRateLimit } from "@/lib/ratelimit"; import { authConfig } from "./auth.config"; export type UserType = "guest" | "regular"; @@ -68,6 +70,11 @@ export const { id: "guest", credentials: {}, async authorize() { + await checkGuestAuthRateLimit( + new Request("http://127.0.0.1/api/auth/callback/guest", { + headers: await headers(), + }) + ); const [guestUser] = await createGuestUser(); return { ...guestUser, type: "guest" }; }, diff --git a/apps/chatbot/lib/ratelimit.ts b/apps/chatbot/lib/ratelimit.ts index 37319f73c..bdd8b39fc 100644 --- a/apps/chatbot/lib/ratelimit.ts +++ b/apps/chatbot/lib/ratelimit.ts @@ -7,6 +7,9 @@ import { ChatbotError } from "@/lib/errors"; const MAX_MESSAGES = 10; const TTL_SECONDS = 60 * 60; +const REDIS_CONNECT_TIMEOUT_MS = 1000; +const REDIS_MAX_RECONNECT_ATTEMPTS = 2; + export const GUEST_AUTH_RATE_LIMIT_PER_IP = 5; export const GUEST_AUTH_RATE_LIMIT_GLOBAL = 60; export const GUEST_AUTH_RATE_LIMIT_TTL_SECONDS = 15 * 60; @@ -17,6 +20,7 @@ local global_key = KEYS[2] local ip_limit = tonumber(ARGV[1]) local global_limit = tonumber(ARGV[2]) local ttl = tonumber(ARGV[3]) +local consume = tonumber(ARGV[4]) local ip_count = tonumber(redis.call('GET', ip_key) or '0') local global_count = tonumber(redis.call('GET', global_key) or '0') @@ -24,6 +28,10 @@ if ip_count >= ip_limit or global_count >= global_limit then return 0 end +if consume ~= 1 then + return 1 +end + ip_count = redis.call('INCR', ip_key) global_count = redis.call('INCR', global_key) if ip_count == 1 then redis.call('EXPIRE', ip_key, ttl) end @@ -31,24 +39,87 @@ if global_count == 1 then redis.call('EXPIRE', global_key, ttl) end return 1 `; -let client: ReturnType | null = null; +type RedisClient = ReturnType; + +let client: RedisClient | null = null; +let connectPromise: Promise | null = null; const memoryGuestCounters = new Map< string, { count: number; expiresAt: number } >(); -function getClient() { - if (!client && process.env.REDIS_URL) { - client = createClient({ url: process.env.REDIS_URL }); - client.on("error", () => undefined); - client.connect().catch(() => { - client = null; - }); +function ensureRedisClient(): RedisClient | null { + if (client) { + return client; + } + + const url = process.env.REDIS_URL; + if (!url) { + return null; } + + client = createClient({ + disableOfflineQueue: true, + socket: { + connectTimeout: REDIS_CONNECT_TIMEOUT_MS, + reconnectStrategy: (retries) => { + if (retries >= REDIS_MAX_RECONNECT_ATTEMPTS) { + return false; + } + return Math.min(50 * 2 ** retries, 200); + }, + }, + url, + }); + client.on("error", () => undefined); return client; } +function beginConnect(redis: RedisClient): void { + if (redis.isOpen || connectPromise) { + return; + } + + connectPromise = redis.connect().then( + () => undefined, + () => { + client = null; + } + ); +} + +function getClient() { + const redis = ensureRedisClient(); + if (redis) { + beginConnect(redis); + } + return redis; +} + +async function getReadyRedisClient(): Promise { + const redis = ensureRedisClient(); + if (!redis) { + return null; + } + + if (redis.isReady) { + return redis; + } + + beginConnect(redis); + + if (connectPromise) { + try { + await connectPromise; + } finally { + connectPromise = null; + } + } + + return client?.isReady ? client : null; +} + function validIp(value: string | undefined): string | undefined { const candidate = value?.trim(); return candidate && isIP(candidate) ? candidate : undefined; @@ -129,7 +200,7 @@ function memoryIncr(key: string, ttlMs: number): void { current.count += 1; } -function consumeMemoryGuestSlot(ip: string): boolean { +function takeMemoryGuestSlot(ip: string, consume: boolean): boolean { const ttlMs = GUEST_AUTH_RATE_LIMIT_TTL_SECONDS * 1000; const ipKey = `guest-auth-rate:ip:${ip}`; if ( @@ -138,39 +209,45 @@ function consumeMemoryGuestSlot(ip: string): boolean { ) { return false; } - memoryIncr(ipKey, ttlMs); - memoryIncr("guest-auth-rate:global", ttlMs); + if (consume) { + memoryIncr(ipKey, ttlMs); + memoryIncr("guest-auth-rate:global", ttlMs); + } return true; } -/** Test-only: reset the process-local guest limiter. */ +/** Test-only: reset the process-local guest limiter and Redis singleton. */ export function resetMemoryGuestAuthRateLimit(): void { memoryGuestCounters.clear(); + client = null; + connectPromise = null; } /** - * Cap unauthenticated guest User inserts. Always on outside Playwright: - * the chat IP limiter is production-only and a no-op without Redis, which - * is how five local curls each created a row. - * - * Production requires Redis (fail closed). Local/dev uses Redis when ready, - * otherwise a process-local counter so `next dev` is still throttled. + * Cap unauthenticated guest User inserts. Always on outside Playwright. + * Production fail-closed on Redis. Dev uses Redis if ready, else process-local + * counters. */ -export async function checkGuestAuthRateLimit(request: Request): Promise { +export async function checkGuestAuthRateLimit( + request: Request, + options?: { consume?: boolean } +): Promise { if (isTestEnvironment) { return; } + const consume = options?.consume !== false; const ip = getClientIp(request) ?? "unknown"; - const redis = getClient(); - if (redis?.isReady) { + const redis = await getReadyRedisClient(); + if (redis) { try { const result = await redis.eval(GUEST_AUTH_RATE_LIMIT_LUA, { arguments: [ String(GUEST_AUTH_RATE_LIMIT_PER_IP), String(GUEST_AUTH_RATE_LIMIT_GLOBAL), String(GUEST_AUTH_RATE_LIMIT_TTL_SECONDS), + consume ? "1" : "0", ], keys: [ `guest-auth-rate:{guest}:ip:${ip}`, @@ -195,7 +272,7 @@ export async function checkGuestAuthRateLimit(request: Request): Promise { throw new GuestAuthRateLimitError(503); } - if (!consumeMemoryGuestSlot(ip)) { + if (!takeMemoryGuestSlot(ip, consume)) { throw new GuestAuthRateLimitError(429); } } diff --git a/apps/chatbot/lib/ratelimit.unit.test.ts b/apps/chatbot/lib/ratelimit.unit.test.ts index 92e7fe82c..c9f387d7e 100644 --- a/apps/chatbot/lib/ratelimit.unit.test.ts +++ b/apps/chatbot/lib/ratelimit.unit.test.ts @@ -1,13 +1,54 @@ import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const limiterEnv = vi.hoisted(() => ({ + production: false, + test: false, +})); + +const redisHarness = vi.hoisted(() => { + const harness = { + isReady: false, + isOpen: false, + evalResult: 1 as unknown, + evalError: null as Error | null, + connect: vi.fn(async () => undefined), + eval: vi.fn(async () => { + if (harness.evalError) { + throw harness.evalError; + } + return harness.evalResult; + }), + }; + return harness; +}); + +vi.mock("@/lib/constants", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + get isProductionEnvironment() { + return limiterEnv.production; + }, + get isTestEnvironment() { + return limiterEnv.test; + }, + }; +}); vi.mock("redis", () => ({ createClient: () => ({ - isReady: false, + get isReady() { + return redisHarness.isReady; + }, + get isOpen() { + return redisHarness.isOpen; + }, on: vi.fn(), - connect: vi.fn().mockResolvedValue(undefined), + connect: redisHarness.connect, + eval: redisHarness.eval, }), })); @@ -25,6 +66,19 @@ function requestWithHeaders(headers?: HeadersInit): Request { }); } +const guestRouteSource = readFileSync( + join( + dirname(fileURLToPath(import.meta.url)), + "../app/(auth)/api/auth/guest/route.ts" + ), + "utf8" +); + +const authSource = readFileSync( + join(dirname(fileURLToPath(import.meta.url)), "../app/(auth)/auth.ts"), + "utf8" +); + describe("getClientIp", () => { it("prefers x-real-ip over x-forwarded-for", () => { expect( @@ -77,10 +131,32 @@ describe("getClientIp", () => { }); describe("checkGuestAuthRateLimit", () => { + const previousRedisUrl = process.env.REDIS_URL; + beforeEach(() => { + limiterEnv.production = false; + limiterEnv.test = false; + redisHarness.isReady = false; + redisHarness.isOpen = false; + redisHarness.evalResult = 1; + redisHarness.evalError = null; + redisHarness.connect.mockClear(); + redisHarness.connect.mockImplementation(async () => undefined); + redisHarness.eval.mockClear(); + delete process.env.REDIS_URL; resetMemoryGuestAuthRateLimit(); }); + afterEach(() => { + if (previousRedisUrl === undefined) { + delete process.env.REDIS_URL; + } else { + process.env.REDIS_URL = previousRedisUrl; + } + limiterEnv.production = false; + limiterEnv.test = false; + }); + it("allows a small per-IP budget then returns 429", async () => { const request = requestWithHeaders({ "x-real-ip": "203.0.113.10" }); @@ -121,21 +197,105 @@ describe("checkGuestAuthRateLimit", () => { status: 429, }); }); + + it("does not consume the budget when peeking", async () => { + const request = requestWithHeaders({ "x-real-ip": "203.0.113.10" }); + + for (let i = 0; i < GUEST_AUTH_RATE_LIMIT_PER_IP; i++) { + await expect( + checkGuestAuthRateLimit(request, { consume: false }) + ).resolves.toBeUndefined(); + } + + for (let i = 0; i < GUEST_AUTH_RATE_LIMIT_PER_IP; i++) { + await expect(checkGuestAuthRateLimit(request)).resolves.toBeUndefined(); + } + + await expect( + checkGuestAuthRateLimit(request, { consume: false }) + ).rejects.toMatchObject({ status: 429 }); + }); + + it("skips the limiter in Playwright", async () => { + limiterEnv.test = true; + const request = requestWithHeaders({ "x-real-ip": "203.0.113.10" }); + + for (let i = 0; i < GUEST_AUTH_RATE_LIMIT_PER_IP + 2; i++) { + await expect(checkGuestAuthRateLimit(request)).resolves.toBeUndefined(); + } + }); + + it("returns 503 in production when Redis is missing", async () => { + limiterEnv.production = true; + const request = requestWithHeaders({ "x-real-ip": "203.0.113.10" }); + + await expect(checkGuestAuthRateLimit(request)).rejects.toMatchObject({ + status: 503, + }); + }); + + it("returns 503 in production when Redis never becomes ready", async () => { + limiterEnv.production = true; + process.env.REDIS_URL = "redis://127.0.0.1:6379"; + const request = requestWithHeaders({ "x-real-ip": "203.0.113.10" }); + + await expect(checkGuestAuthRateLimit(request)).rejects.toMatchObject({ + status: 503, + }); + expect(redisHarness.connect).toHaveBeenCalled(); + }); + + it("waits for Redis connect before consuming a slot", async () => { + limiterEnv.production = true; + process.env.REDIS_URL = "redis://127.0.0.1:6379"; + redisHarness.connect.mockImplementation(async () => { + redisHarness.isOpen = true; + redisHarness.isReady = true; + }); + const request = requestWithHeaders({ "x-real-ip": "203.0.113.10" }); + + await expect(checkGuestAuthRateLimit(request)).resolves.toBeUndefined(); + expect(redisHarness.eval).toHaveBeenCalled(); + }); + + it("returns 429 when Redis eval denies the slot", async () => { + limiterEnv.production = true; + process.env.REDIS_URL = "redis://127.0.0.1:6379"; + redisHarness.isReady = true; + redisHarness.isOpen = true; + redisHarness.evalResult = 0; + const request = requestWithHeaders({ "x-real-ip": "203.0.113.10" }); + + await expect(checkGuestAuthRateLimit(request)).rejects.toMatchObject({ + status: 429, + }); + }); + + it("returns 503 in production when Redis eval fails", async () => { + limiterEnv.production = true; + process.env.REDIS_URL = "redis://127.0.0.1:6379"; + redisHarness.isReady = true; + redisHarness.isOpen = true; + redisHarness.evalError = new Error("eval failed"); + const request = requestWithHeaders({ "x-real-ip": "203.0.113.10" }); + + await expect(checkGuestAuthRateLimit(request)).rejects.toMatchObject({ + status: 503, + }); + }); }); -describe("guest auth route", () => { - it("rate-limits before signIn so createGuestUser is not unconditional", () => { - const source = readFileSync( - join( - dirname(fileURLToPath(import.meta.url)), - "../app/(auth)/api/auth/guest/route.ts" - ), - "utf8" +describe("guest auth call sites", () => { + it("peeks on GET before signIn and consumes in authorize before createGuestUser", () => { + expect(guestRouteSource).toContain("checkGuestAuthRateLimit"); + expect(guestRouteSource).toContain("consume: false"); + expect(guestRouteSource.indexOf("checkGuestAuthRateLimit")).toBeLessThan( + guestRouteSource.indexOf('signIn("guest"') ); - expect(source).toContain("checkGuestAuthRateLimit"); - expect(source.indexOf("checkGuestAuthRateLimit")).toBeLessThan( - source.indexOf('signIn("guest"') + expect(authSource).toContain("checkGuestAuthRateLimit"); + expect(authSource.indexOf("checkGuestAuthRateLimit")).toBeLessThan( + authSource.indexOf("createGuestUser()") ); }); }); From e1fa438261686d1cba38d2393491d90ab9ebf7f6 Mon Sep 17 00:00:00 2001 From: HoangDucBach Date: Wed, 9 Sep 2026 10:56:08 +0700 Subject: [PATCH 40/70] fix(chatbot): reconnect Redis and map guest limiter through Auth.js A leftover settled connectPromise skipped reconnect after chat warmed the client. Drain on settle, raise the site-wide guest cap, and surface authorize() 429/503 as JSON instead of CallbackRouteError 500. --- .../app/(auth)/api/auth/guest/route.ts | 42 +++++--- apps/chatbot/app/(auth)/auth.ts | 23 +++-- apps/chatbot/lib/ratelimit.ts | 99 ++++++++++++++----- apps/chatbot/lib/ratelimit.unit.test.ts | 52 ++++++++++ 4 files changed, 170 insertions(+), 46 deletions(-) diff --git a/apps/chatbot/app/(auth)/api/auth/guest/route.ts b/apps/chatbot/app/(auth)/api/auth/guest/route.ts index 3e3768ba4..5b1ea70c0 100644 --- a/apps/chatbot/app/(auth)/api/auth/guest/route.ts +++ b/apps/chatbot/app/(auth)/api/auth/guest/route.ts @@ -6,8 +6,24 @@ import { checkGuestAuthRateLimit, GUEST_AUTH_RATE_LIMIT_TTL_SECONDS, GuestAuthRateLimitError, + guestAuthLimitFromError, } from "@/lib/ratelimit"; +function guestAuthLimitResponse(error: GuestAuthRateLimitError) { + return NextResponse.json( + { error: error.message }, + { + status: error.status, + headers: { + "Retry-After": + error.status === 429 + ? String(GUEST_AUTH_RATE_LIMIT_TTL_SECONDS) + : "5", + }, + } + ); +} + export async function GET(request: Request) { const { searchParams } = new URL(request.url); const rawRedirectUrl = searchParams.get("redirectUrl") || "/"; @@ -26,22 +42,20 @@ export async function GET(request: Request) { // Peek only: signIn("guest") runs authorize() in-process, which consumes. await checkGuestAuthRateLimit(request, { consume: false }); } catch (error) { - if (error instanceof GuestAuthRateLimitError) { - return NextResponse.json( - { error: error.message }, - { - status: error.status, - headers: { - "Retry-After": - error.status === 429 - ? String(GUEST_AUTH_RATE_LIMIT_TTL_SECONDS) - : "5", - }, - } - ); + const limited = guestAuthLimitFromError(error); + if (limited) { + return guestAuthLimitResponse(limited); } throw error; } - return signIn("guest", { redirect: true, redirectTo: redirectUrl }); + try { + return await signIn("guest", { redirect: true, redirectTo: redirectUrl }); + } catch (error) { + const limited = guestAuthLimitFromError(error); + if (limited) { + return guestAuthLimitResponse(limited); + } + throw error; + } } diff --git a/apps/chatbot/app/(auth)/auth.ts b/apps/chatbot/app/(auth)/auth.ts index 1b272fb3d..a0ba2890a 100644 --- a/apps/chatbot/app/(auth)/auth.ts +++ b/apps/chatbot/app/(auth)/auth.ts @@ -1,11 +1,11 @@ import { compare } from "bcrypt-ts"; -import NextAuth, { type DefaultSession } from "next-auth"; +import NextAuth, { CredentialsSignin, type DefaultSession } from "next-auth"; import type { DefaultJWT } from "next-auth/jwt"; import Credentials from "next-auth/providers/credentials"; import { headers } from "next/headers"; import { DUMMY_PASSWORD } from "@/lib/constants"; import { createGuestUser, getUser } from "@/lib/db/queries"; -import { checkGuestAuthRateLimit } from "@/lib/ratelimit"; +import { checkGuestAuthRateLimit, GuestAuthRateLimitError, guestAuthSignInCode } from "@/lib/ratelimit"; import { authConfig } from "./auth.config"; export type UserType = "guest" | "regular"; @@ -70,11 +70,20 @@ export const { id: "guest", credentials: {}, async authorize() { - await checkGuestAuthRateLimit( - new Request("http://127.0.0.1/api/auth/callback/guest", { - headers: await headers(), - }) - ); + try { + await checkGuestAuthRateLimit( + new Request("http://127.0.0.1/api/auth/callback/guest", { + headers: await headers(), + }) + ); + } catch (error) { + if (error instanceof GuestAuthRateLimitError) { + const signInError = new CredentialsSignin(); + signInError.code = guestAuthSignInCode(error.status); + throw signInError; + } + throw error; + } const [guestUser] = await createGuestUser(); return { ...guestUser, type: "guest" }; }, diff --git a/apps/chatbot/lib/ratelimit.ts b/apps/chatbot/lib/ratelimit.ts index bdd8b39fc..54fca00a6 100644 --- a/apps/chatbot/lib/ratelimit.ts +++ b/apps/chatbot/lib/ratelimit.ts @@ -11,8 +11,9 @@ const REDIS_CONNECT_TIMEOUT_MS = 1000; const REDIS_MAX_RECONNECT_ATTEMPTS = 2; export const GUEST_AUTH_RATE_LIMIT_PER_IP = 5; -export const GUEST_AUTH_RATE_LIMIT_GLOBAL = 60; -export const GUEST_AUTH_RATE_LIMIT_TTL_SECONDS = 15 * 60; +/** Site-wide admission cap: every unauthenticated visitor hits GET /api/auth/guest. */ +export const GUEST_AUTH_RATE_LIMIT_GLOBAL = 150; +export const GUEST_AUTH_RATE_LIMIT_TTL_SECONDS = 60; const GUEST_AUTH_RATE_LIMIT_LUA = ` local ip_key = KEYS[1] @@ -76,46 +77,51 @@ function ensureRedisClient(): RedisClient | null { return client; } -function beginConnect(redis: RedisClient): void { - if (redis.isOpen || connectPromise) { - return; - } - - connectPromise = redis.connect().then( +function startConnect(redis: RedisClient): Promise { + const pending = redis.connect().then( () => undefined, () => { client = null; } ); + connectPromise = pending; + void pending.finally(() => { + if (connectPromise === pending) { + connectPromise = null; + } + }); + return pending; } -function getClient() { - const redis = ensureRedisClient(); - if (redis) { - beginConnect(redis); +async function drainConnectPromise(): Promise { + if (!connectPromise) { + return; + } + try { + await connectPromise; + } finally { + connectPromise = null; } - return redis; } +/** One await-and-clear helper. Do not fire-and-forget connect() — leftover settled promises skip reconnect. */ async function getReadyRedisClient(): Promise { const redis = ensureRedisClient(); if (!redis) { return null; } - if (redis.isReady) { - return redis; - } + await drainConnectPromise(); - beginConnect(redis); + const current = client ?? ensureRedisClient(); + if (!current) { + return null; + } - if (connectPromise) { - try { - await connectPromise; - } finally { - connectPromise = null; - } + if (!current.isOpen && !connectPromise) { + startConnect(current); } + await drainConnectPromise(); return client?.isReady ? client : null; } @@ -145,8 +151,8 @@ export async function checkIpRateLimit(ip: string | undefined) { return; } - const redis = getClient(); - if (!redis?.isReady) { + const redis = await getReadyRedisClient(); + if (!redis) { return; } @@ -182,6 +188,49 @@ export class GuestAuthRateLimitError extends Error { } } +const GUEST_AUTH_SIGNIN_CODES = { + too_many_requests: 429, + service_unavailable: 503, +} as const; + +export function guestAuthSignInCode( + status: 429 | 503 +): keyof typeof GUEST_AUTH_SIGNIN_CODES { + return status === 429 ? "too_many_requests" : "service_unavailable"; +} + +/** Unwrap Auth.js CallbackRouteError / CredentialsSignin back to a guest limiter error. */ +export function guestAuthLimitFromError( + error: unknown +): GuestAuthRateLimitError | null { + const seen = new Set(); + let current: unknown = error; + + while (current && !seen.has(current)) { + seen.add(current); + if (current instanceof GuestAuthRateLimitError) { + return current; + } + if (typeof current !== "object") { + return null; + } + const rec = current as { code?: unknown; cause?: unknown }; + if (rec.code === "too_many_requests") { + return new GuestAuthRateLimitError(429); + } + if (rec.code === "service_unavailable") { + return new GuestAuthRateLimitError(503); + } + const cause = rec.cause; + current = + cause && typeof cause === "object" && cause !== null && "err" in cause + ? (cause as { err: unknown }).err + : cause; + } + + return null; +} + function memoryCount(key: string): number { const current = memoryGuestCounters.get(key); if (!current || current.expiresAt <= Date.now()) { diff --git a/apps/chatbot/lib/ratelimit.unit.test.ts b/apps/chatbot/lib/ratelimit.unit.test.ts index c9f387d7e..e1aff80b0 100644 --- a/apps/chatbot/lib/ratelimit.unit.test.ts +++ b/apps/chatbot/lib/ratelimit.unit.test.ts @@ -54,8 +54,10 @@ vi.mock("redis", () => ({ import { checkGuestAuthRateLimit, + checkIpRateLimit, GUEST_AUTH_RATE_LIMIT_PER_IP, GuestAuthRateLimitError, + guestAuthLimitFromError, getClientIp, resetMemoryGuestAuthRateLimit, } from "@/lib/ratelimit"; @@ -283,6 +285,34 @@ describe("checkGuestAuthRateLimit", () => { status: 503, }); }); + + it("reconnects after chat traffic left a settled connectPromise", async () => { + limiterEnv.production = true; + process.env.REDIS_URL = "redis://127.0.0.1:6379"; + redisHarness.connect.mockImplementation(async () => { + redisHarness.isOpen = true; + redisHarness.isReady = true; + }); + + await checkIpRateLimit("203.0.113.10"); + expect(redisHarness.connect).toHaveBeenCalled(); + + redisHarness.isOpen = false; + redisHarness.isReady = false; + redisHarness.connect.mockClear(); + redisHarness.connect.mockImplementation(async () => { + redisHarness.isOpen = true; + redisHarness.isReady = true; + }); + + await expect( + checkGuestAuthRateLimit( + requestWithHeaders({ "x-real-ip": "203.0.113.10" }) + ) + ).resolves.toBeUndefined(); + expect(redisHarness.connect).toHaveBeenCalled(); + expect(redisHarness.eval).toHaveBeenCalled(); + }); }); describe("guest auth call sites", () => { @@ -292,10 +322,32 @@ describe("guest auth call sites", () => { expect(guestRouteSource.indexOf("checkGuestAuthRateLimit")).toBeLessThan( guestRouteSource.indexOf('signIn("guest"') ); + expect(guestRouteSource).toContain("guestAuthLimitFromError"); expect(authSource).toContain("checkGuestAuthRateLimit"); + expect(authSource).toContain("CredentialsSignin"); expect(authSource.indexOf("checkGuestAuthRateLimit")).toBeLessThan( authSource.indexOf("createGuestUser()") ); }); }); + +describe("guestAuthLimitFromError", () => { + it("unwraps Auth.js CredentialsSignin codes and CallbackRouteError cause", () => { + expect( + guestAuthLimitFromError( + Object.assign(new Error("CredentialsSignin"), { + code: "too_many_requests", + }) + ) + ).toMatchObject({ status: 429 }); + expect( + guestAuthLimitFromError( + Object.assign(new Error("CallbackRouteError"), { + cause: { err: new GuestAuthRateLimitError(503) }, + }) + ) + ).toMatchObject({ status: 503 }); + expect(guestAuthLimitFromError(new Error("unrelated"))).toBeNull(); + }); +}); From a0844601ada2b076c9a067d8e20ef0c56ae6aaff Mon Sep 17 00:00:00 2001 From: Le Tien Phat <91601109+Niko1444@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:23:51 +0700 Subject: [PATCH 41/70] fix(sdk): default account and manual helpers to a gRPC Sui client (WALM-164) createAccount, addDelegateKey, removeDelegateKey and MemWalManual looked for SuiClient in @mysten/sui/client when the caller omitted suiClient. Current @mysten/sui no longer exports it, so they threw "SuiClient not found" before doing anything. Build SuiGrpcClient against the public fullnode for suiNetwork instead. Those fullnodes no longer serve JSON-RPC, so a JSON-RPC default would construct and then fail on its first call. An explicit suiClient still wins. Fixes #300 --- packages/sdk/CHANGELOG.md | 1 + packages/sdk/src/account.ts | 20 +-- packages/sdk/src/manual.ts | 29 +---- packages/sdk/src/sui-client.ts | 35 ++++++ packages/sdk/src/types.ts | 7 +- packages/sdk/test/default-sui-client.test.mjs | 118 ++++++++++++++++++ 6 files changed, 165 insertions(+), 45 deletions(-) create mode 100644 packages/sdk/src/sui-client.ts create mode 100644 packages/sdk/test/default-sui-client.test.mjs diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index 3754c17ba..a0d343030 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -4,6 +4,7 @@ ### Fixed +- `createAccount`, `addDelegateKey`, `removeDelegateKey`, and `MemWalManual` work again without a caller-supplied `suiClient`. They looked for `SuiClient` in `@mysten/sui/client`, which current `@mysten/sui` no longer exports, and threw `SuiClient not found`. The default is now `SuiGrpcClient` against the public fullnode for `suiNetwork`, because those fullnodes no longer serve JSON-RPC. An explicit `suiClient` still wins. (WALM-164, #300) - `MemWalMock.listNamespaces` rejects a non-positive or non-integer `limit` with `limit must be positive`, matching the relayer. `limit: 0` used to return `has_more: true` and `next_cursor: null`, so the documented pagination loop never finished, and `limit: -1` silently dropped the last namespace. (#1144, #1145) - `MemWalMock.rememberBulkAsync` rejects an empty or non-array batch with the same error as `MemWal`. (#1146) - `MemWalMock` honors `idempotencyKey`. A retry of the same text and namespace returns the original job, a different payload throws, and `forget` / `clear` release the key so the text can be written again. (#1147) diff --git a/packages/sdk/src/account.ts b/packages/sdk/src/account.ts index f62ec9640..8c5eeccf5 100644 --- a/packages/sdk/src/account.ts +++ b/packages/sdk/src/account.ts @@ -42,6 +42,7 @@ import type { RemoveDelegateKeyOpts, } from "./types.js"; import { bytesToHex, hexToBytes } from "./utils.js"; +import { createDefaultSuiClient } from "./sui-client.js"; // ============================================================ // SUI Clock object (shared, always 0x6) @@ -79,24 +80,7 @@ async function buildTxContext(opts: { const { Transaction } = await import("@mysten/sui/transactions"); // Build Sui client - let suiClient: any; - if (opts.suiClient) { - suiClient = opts.suiClient; - } else { - const mod = await import("@mysten/sui/client"); - const SuiClient = (mod as any).SuiClient; - if (typeof SuiClient !== "function") { - throw new Error( - "SuiClient not found. For @mysten/sui v2.6.0+, pass suiClient in opts." - ); - } - const network = opts.suiNetwork ?? "mainnet"; - const urls: Record = { - testnet: "https://fullnode.testnet.sui.io:443", - mainnet: "https://fullnode.mainnet.sui.io:443", - }; - suiClient = new SuiClient({ url: urls[network] ?? urls.mainnet }); - } + const suiClient = opts.suiClient ?? await createDefaultSuiClient(opts.suiNetwork); // Build signer if (opts.walletSigner) { diff --git a/packages/sdk/src/manual.ts b/packages/sdk/src/manual.ts index 4dcb9d923..1acf0abe8 100644 --- a/packages/sdk/src/manual.ts +++ b/packages/sdk/src/manual.ts @@ -53,6 +53,7 @@ import { } from "./utils.js"; import { assertCompatibleRelayer, compatibilityErrorFromStatus } from "./compatibility.js"; import { resolveLimitOrOptions } from "./recall-args.js"; +import { createDefaultSuiClient } from "./sui-client.js"; // ============================================================ // Constants @@ -247,30 +248,10 @@ export class MemWalManual { private async getSuiClient() { if (!this._suiClient) { - // Prefer externally-provided client (e.g. from dapp-kit's useSuiClient()) - if (this.config.suiClient) { - this._suiClient = this.config.suiClient; - } else { - // Fallback: create client via dynamic import - // @ts-ignore — optional peer dependency - const mod = await import("@mysten/sui/client"); - const SuiClient = (mod as any).SuiClient; - if (typeof SuiClient !== "function") { - throw new Error( - "SuiClient not found in @mysten/sui/client. " + - "For @mysten/sui v2.6.0+, pass suiClient in config " + - "(e.g. from dapp-kit's useSuiClient())" - ); - } - const network = this.config.suiNetwork ?? "mainnet"; - const urls: Record = { - testnet: "https://fullnode.testnet.sui.io:443", - mainnet: "https://fullnode.mainnet.sui.io:443", - }; - this._suiClient = new SuiClient({ - url: urls[network] ?? urls.mainnet, - }); - } + // Prefer externally-provided client (e.g. from dapp-kit's useSuiClient()), + // otherwise gRPC against the public fullnode for suiNetwork. + this._suiClient = + this.config.suiClient ?? await createDefaultSuiClient(this.config.suiNetwork); } return this._suiClient; } diff --git a/packages/sdk/src/sui-client.ts b/packages/sdk/src/sui-client.ts new file mode 100644 index 000000000..29bb889dd --- /dev/null +++ b/packages/sdk/src/sui-client.ts @@ -0,0 +1,35 @@ +/** + * Default Sui client for the helpers that run without a relayer `/config` + * (account management and MemWalManual). + * + * Mysten's public fullnodes no longer serve JSON-RPC, so the default is the + * gRPC client. Callers that need another endpoint or transport pass their own + * `suiClient`. + */ + +const DEFAULT_SUI_GRPC_URLS: Record = { + testnet: "https://fullnode.testnet.sui.io:443", + mainnet: "https://fullnode.mainnet.sui.io:443", +}; + +export async function createDefaultSuiClient( + suiNetwork?: "testnet" | "mainnet", +): Promise { + const network = suiNetwork && DEFAULT_SUI_GRPC_URLS[suiNetwork] ? suiNetwork : "mainnet"; + + let SuiGrpcClient: any; + try { + // @ts-ignore — optional peer dependency + ({ SuiGrpcClient } = (await import("@mysten/sui/grpc")) as any); + } catch { + /* reported below */ + } + if (typeof SuiGrpcClient !== "function") { + throw new Error( + "SuiGrpcClient not found in @mysten/sui/grpc. " + + "Install @mysten/sui >=2.5.0, or pass suiClient." + ); + } + + return new SuiGrpcClient({ network, baseUrl: DEFAULT_SUI_GRPC_URLS[network] }); +} diff --git a/packages/sdk/src/types.ts b/packages/sdk/src/types.ts index a52ff4e2c..2037c9caf 100644 --- a/packages/sdk/src/types.ts +++ b/packages/sdk/src/types.ts @@ -600,8 +600,8 @@ export interface MemWalManualConfig { walletSigner?: WalletSigner; /** * Pre-configured Sui client instance (e.g. from dapp-kit's useSuiClient()). - * If omitted, the SDK will try to create one internally. - * Recommended for browser environments where @mysten/sui v2.x removed SuiClient. + * If omitted, the SDK creates a gRPC client against the public fullnode + * for `suiNetwork`. Pass one to use a different endpoint or transport. */ suiClient?: any; /** OpenAI/OpenRouter API key for embeddings (required for client-side embedding) */ @@ -711,7 +711,8 @@ interface AccountTxOpts { walletSigner?: WalletSigner; /** * Pre-configured Sui client instance. - * If omitted, the SDK will create one internally. + * If omitted, the SDK creates a gRPC client against the public fullnode + * for `suiNetwork`. Pass one to use a different endpoint or transport. */ suiClient?: any; /** Sui network (default: mainnet) */ diff --git a/packages/sdk/test/default-sui-client.test.mjs b/packages/sdk/test/default-sui-client.test.mjs new file mode 100644 index 000000000..d06004d00 --- /dev/null +++ b/packages/sdk/test/default-sui-client.test.mjs @@ -0,0 +1,118 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { Ed25519Keypair } from "@mysten/sui/keypairs/ed25519"; +import { isSuiGrpcClient } from "@mysten/sui/grpc"; + +import { addDelegateKey, createAccount } from "../dist/account.js"; +import { MemWalManual } from "../dist/manual.js"; + +const originalFetch = globalThis.fetch; + +test.afterEach(() => { + globalThis.fetch = originalFetch; +}); + +/** Record every outbound request and fail it, so no test reaches the network. */ +function recordRequests() { + const urls = []; + globalThis.fetch = async (input) => { + urls.push(typeof input === "string" ? input : input.url); + throw new Error("network disabled in test"); + }; + return urls; +} + +function accountOpts(extra = {}) { + return { + packageId: "0x1", + registryId: "0x2", + suiPrivateKey: Ed25519Keypair.generate().getSecretKey(), + ...extra, + }; +} + +function manualClient(extra = {}) { + return MemWalManual.create({ + key: new Uint8Array(32).fill(1), + suiPrivateKey: Ed25519Keypair.generate().getSecretKey(), + embeddingApiKey: "unused", + packageId: "0x1", + accountId: "0x3", + registryId: "0x2", + serverUrl: "https://relayer.example", + ...extra, + }); +} + +test("createAccount without suiClient talks gRPC to the mainnet fullnode", async () => { + const urls = recordRequests(); + + await assert.rejects(createAccount(accountOpts()), (err) => { + assert.doesNotMatch(String(err?.message), /SuiClient not found/); + return true; + }); + + assert.ok(urls.length > 0, "expected the default client to issue a request"); + for (const url of urls) { + assert.match(url, /^https:\/\/fullnode\.mainnet\.sui\.io(:443)?\/sui\.rpc\.v2\./); + } +}); + +test("addDelegateKey without suiClient honors suiNetwork", async () => { + const urls = recordRequests(); + + await assert.rejects( + addDelegateKey(accountOpts({ + accountId: "0x3", + publicKey: new Uint8Array(32), + label: "test", + suiNetwork: "testnet", + })), + (err) => { + assert.doesNotMatch(String(err?.message), /SuiClient not found/); + return true; + }, + ); + + assert.ok(urls.length > 0, "expected the default client to issue a request"); + for (const url of urls) { + assert.match(url, /^https:\/\/fullnode\.testnet\.sui\.io(:443)?\/sui\.rpc\.v2\./); + } +}); + +test("an explicit suiClient still wins over the default", async () => { + const urls = recordRequests(); + const calls = []; + const suiClient = { + signAndExecuteTransaction: async () => { + calls.push("signAndExecuteTransaction"); + return { digest: "explicit-digest" }; + }, + waitForTransaction: async () => { + calls.push("waitForTransaction"); + return { effects: { status: { status: "success" } } }; + }, + }; + + const result = await createAccount(accountOpts({ suiClient })); + + assert.equal(result.digest, "explicit-digest"); + assert.deepEqual(calls, ["signAndExecuteTransaction", "waitForTransaction"]); + assert.deepEqual(urls, []); +}); + +test("MemWalManual builds a gRPC client when suiClient is omitted", async () => { + const mainnet = await manualClient().getSuiClient(); + assert.equal(isSuiGrpcClient(mainnet), true); + assert.equal(mainnet.network, "mainnet"); + + const testnet = await manualClient({ suiNetwork: "testnet" }).getSuiClient(); + assert.equal(isSuiGrpcClient(testnet), true); + assert.equal(testnet.network, "testnet"); +}); + +test("MemWalManual keeps an explicit suiClient", async () => { + const suiClient = { marker: "explicit" }; + assert.equal(await manualClient({ suiClient }).getSuiClient(), suiClient); +}); From d00ae1d1c39a52ff2dcefbeb1a4c5aea665a19fa Mon Sep 17 00:00:00 2001 From: ducnmm <165614309+ducnmm@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:31:17 +0700 Subject: [PATCH 42/70] fix(sdk): reject unsafe remember job ids (WALM-833) * fix(sdk): reject unsafe remember job ids and add Python status getRememberStatus and waitForRememberJob no longer sign a job id that is empty or contains a slash, question mark, hash, or whitespace. Python MemWal and MemWalSync gain get_remember_status, and an empty 404 body is not_found instead of json.JSONDecodeError. WALM-833, WALM-832, #1100, #1096, #1216 * fix(sdk): allowlist remember job ids and parse status 404s locally `.` `..` `\` and `%2e%2e` passed the denylist and were rewritten after signing. Both SDKs now accept only one [A-Za-z0-9_-] segment. A UUID object still polls; any other non-string is a TypeError. Empty and non-object 404 bodies are not_found in the status readers, not in the shared signed request. --- docs/python-sdk/api-reference.md | 13 +- docs/python-sdk/changelog.mdx | 10 ++ docs/sdk/api-reference.md | 6 +- docs/sdk/changelog.mdx | 6 + packages/python-sdk-memwal/CHANGELOG.md | 5 + packages/python-sdk-memwal/README.md | 1 + packages/python-sdk-memwal/memwal/client.py | 97 +++++++++++- .../python-sdk-memwal/tests/test_client.py | 148 ++++++++++++++++++ packages/sdk/CHANGELOG.md | 1 + packages/sdk/src/memwal.ts | 26 ++- .../sdk/test/remember-call-shape.test.mjs | 52 ++++++ 11 files changed, 355 insertions(+), 10 deletions(-) diff --git a/docs/python-sdk/api-reference.md b/docs/python-sdk/api-reference.md index 65c58dd8a..81d65361f 100644 --- a/docs/python-sdk/api-reference.md +++ b/docs/python-sdk/api-reference.md @@ -84,9 +84,20 @@ Submit one memory and poll until the job completes. RememberResult(id: str, blob_id: str, owner: str, namespace: str) ``` +### `get_remember_status(job_id) -> RememberJobStatus` + +One `GET /api/remember/{job_id}`. Accepts 200 and 404 and returns the job state (`pending`, `running`, `uploaded`, `done`, `failed`, or `not_found`) without polling. A job id that is empty or contains whitespace, `/`, `?`, or `#` is `not_found` and is not requested. `MemWalSync` has the same method. + +```python +RememberJobStatus( + job_id: str, status: str, owner: str | None = None, + namespace: str | None = None, blob_id: str | None = None, error: str | None = None, +) +``` + ### `wait_for_remember_job(job_id, poll_interval_ms=1500, timeout_ms=60_000) -> RememberResult` -Poll a previously accepted job until `done` or `failed`. Raises `MemWalRememberJobNotFound` / `MemWalRememberJobFailed` / `MemWalRememberJobTimeout`. +Poll a previously accepted job until `done` or `failed`. Raises `MemWalRememberJobNotFound` / `MemWalRememberJobFailed` / `MemWalRememberJobTimeout`. A job id outside `[A-Za-z0-9_-]+` raises `MemWalRememberJobNotFound` before the request. ### `remember_bulk_async(items) -> RememberBulkAcceptedResult` diff --git a/docs/python-sdk/changelog.mdx b/docs/python-sdk/changelog.mdx index 4f5b4d5ed..811fb7f13 100644 --- a/docs/python-sdk/changelog.mdx +++ b/docs/python-sdk/changelog.mdx @@ -36,6 +36,16 @@ Track what's new, changed, and fixed in `memwal` (Python). For the latest version, see the [PyPI project page](https://pypi.org/project/memwal/). +## Unreleased + +### Added + +- `get_remember_status(job_id)` on `MemWal` and `MemWalSync` returns `RememberJobStatus` from `GET /api/remember/{job_id}`, accepting 200 and 404 like the TypeScript SDK. (WALM-832, #1096) + +### Fixed + +- `get_remember_status` and `wait_for_remember_job` reject a job id outside `[A-Za-z0-9_-]+` before the request. `.`, `..`, a backslash, and encoded dot-dot used to pass a denylist and be rewritten after signing. An empty id used to call `GET /api/remember/` and raise `json.JSONDecodeError`. An unparseable 404 is `not_found` / `MemWalRememberJobNotFound`, not a JSON error. `uuid.UUID` is coerced with `str()`. Valid UUID job ids are unchanged. (WALM-833, #1100, #1216) + ## 0.1.12 ### Fixed diff --git a/docs/sdk/api-reference.md b/docs/sdk/api-reference.md index ffe28e6cf..658b82718 100644 --- a/docs/sdk/api-reference.md +++ b/docs/sdk/api-reference.md @@ -84,9 +84,13 @@ Submit one memory and poll until the background job completes. } ``` +### `getRememberStatus(jobId): Promise` + +One `GET /api/remember/{jobId}`. Accepts 200 and 404 and returns the job state (`pending`, `running`, `uploaded`, `done`, `failed`, `not_found`) without polling. A job id that is empty or contains whitespace, `/`, `?`, or `#` is `not_found` and is not requested. + ### `waitForRememberJob(jobId, opts?): Promise` -Poll a previously accepted remember job until it reaches `done` or `failed`. +Poll a previously accepted remember job until it reaches `done` or `failed`. A job id outside `[A-Za-z0-9_-]+` fails immediately with status 404. ### `rememberBulk(items): Promise` diff --git a/docs/sdk/changelog.mdx b/docs/sdk/changelog.mdx index b41d8dbce..891f13860 100644 --- a/docs/sdk/changelog.mdx +++ b/docs/sdk/changelog.mdx @@ -31,6 +31,12 @@ answer: >- The latest TypeScript SDK release is 0.1.8. Request bodies are hashed with `@noble/hashes` rather than WebCrypto-or-`node:crypto`, so the SDK no longer imports a Node builtin that Vite silently externalises into a runtime crash in the browser, and it declares a Node 20 floor. `restore()` results include `failed` (required like `truncated`; SDK defaults omitted to `0`) for permanent decrypt/UTF-8 failures instead of folding them into `skipped`. Empty-body 401s use the AUTH_REJECTED troubleshooting message instead of telling callers to run `memwal_login`. Account and manual PTBs use typed `tx.pure` helpers so they work with modern `@mysten/sui`. An explicit `sort` on `recall()`, `"relevance"` included, now makes the relayer ignore `scoringWeights`; weights re-rank only when `sort` is omitted. 0.1.6 added optional `created_at` on `recall()` results, plus `sort` and `scoringWeights` on `RecallOptions`, and reports HTTP 503 as a retryable upstream outage instead of a sign-in failure. --- +## Unreleased + +### Fixed + +- `getRememberStatus` and `waitForRememberJob` reject a job id outside `[A-Za-z0-9_-]+` before signing. `.`, `..`, `\`, and `%2e%2e` used to pass a denylist and be rewritten by `fetch` after the path was signed. Valid UUID job ids are unchanged. (#1087, #1216) + ## 0.1.9 ### Fixed diff --git a/packages/python-sdk-memwal/CHANGELOG.md b/packages/python-sdk-memwal/CHANGELOG.md index 8f18f4343..9df68d373 100644 --- a/packages/python-sdk-memwal/CHANGELOG.md +++ b/packages/python-sdk-memwal/CHANGELOG.md @@ -2,8 +2,13 @@ ## Unreleased +### Added + +- `get_remember_status(job_id)` on `MemWal` and `MemWalSync` returns `RememberJobStatus` from `GET /api/remember/{job_id}`, accepting 200 and 404 like the TypeScript SDK. (WALM-832, #1096) + ### Fixed +- `get_remember_status` and `wait_for_remember_job` reject a job id outside `[A-Za-z0-9_-]+` before the request. `.`, `..`, a backslash, and encoded dot-dot used to pass a denylist and be rewritten after signing. An empty id used to call `GET /api/remember/` and raise `json.JSONDecodeError`. An unparseable 404 is `not_found` / `MemWalRememberJobNotFound`, not a JSON error. `uuid.UUID` is coerced with `str()`. Valid UUID job ids are unchanged. (WALM-833, #1100, #1216) - `remember`, `remember_bulk`, `analyze`, and `embed` reject whitespace-only text before the request. A blank body used to be accepted and then sit until the caller timed out. (#1129) - A 401 `AUTH_REJECTED` now says a wallet address is not a MemWal account ID. (#1132) - `wait_for_remember_jobs` / `remember_bulk_and_wait` no longer return a batch they could not read as if it were still uploading. When every status poll is rate-limited, the wait makes one confirming read and raises `MemWalRateLimited` (`status` 429, `job_ids`, `retry_after`) if that is refused too. An item still unsettled at the deadline names its last known status, or says no read got through. (WALM-671, #967) diff --git a/packages/python-sdk-memwal/README.md b/packages/python-sdk-memwal/README.md index 92e9adf18..6264d5a90 100644 --- a/packages/python-sdk-memwal/README.md +++ b/packages/python-sdk-memwal/README.md @@ -201,6 +201,7 @@ Create a new async client. | Method | Description | |--------|-------------| | `await remember(text, namespace?)` | Accept a background remember job and return `job_id` | +| `await get_remember_status(job_id)` | Read one remember job without waiting | | `await wait_for_remember_job(job_id, ...)` | Poll one remember job until it is searchable | | `await remember_and_wait(text, namespace?, ...)` | Store a memory and wait until it is searchable | | `await remember_bulk(items)` | Accept several background remember jobs | diff --git a/packages/python-sdk-memwal/memwal/client.py b/packages/python-sdk-memwal/memwal/client.py index 448ee7af7..a0662612e 100644 --- a/packages/python-sdk-memwal/memwal/client.py +++ b/packages/python-sdk-memwal/memwal/client.py @@ -69,6 +69,7 @@ RememberBulkResult, RememberBulkStatusItem, RememberBulkStatusResult, + RememberJobStatus, RememberManualOptions, RememberManualResult, RememberResult, @@ -156,6 +157,36 @@ def _derived_idempotency_key(request_identity: str) -> str: return f"r1-{hashlib.sha256(payload).hexdigest()}" +def _require_remember_job_id(job_id: object) -> str: + """String form of a remember job id. + + ``uuid.UUID`` used to interpolate into the path and still does. Any other + non-string is a type mistake: ``not_found`` would hide a job that exists. + """ + + if isinstance(job_id, uuid.UUID): + return str(job_id) + if not isinstance(job_id, str): + raise TypeError( + f"remember job id must be a string, got {type(job_id).__name__}" + ) + return job_id + + +def _remember_status_path(job_id: str) -> Optional[str]: + """``/api/remember/{job_id}`` when the id is one safe segment, else None. + + httpx rewrites the path after it is signed. ``.`` requests + ``/api/remember``, ``..`` requests ``/api``, and a NUL raises + ``InvalidURL``. An allowlist keeps those from leaving the client. A + normal UUID is unchanged. (#1100, #1216) + """ + + if re.fullmatch(r"[A-Za-z0-9_-]+", job_id) is None: + return None + return f"/api/remember/{job_id}" + + def normalize_server_url(url: str) -> str: """Strip a trailing slash and warn on plaintext HTTP to a remote host. @@ -524,6 +555,55 @@ def _poll_deadline_s(self, deadline_ms: int) -> float: remaining_s = (deadline_ms - _now_ms()) / 1000.0 return max(0.001, min(60.0, remaining_s)) + async def get_remember_status(self, job_id: str) -> RememberJobStatus: + """One-shot lookup of a remember job. Mirrors TS ``getRememberStatus``. + + ``GET /api/remember/{job_id}``, accepting 200 and 404. A 404, or a + body with no ``status``, is ``not_found`` — not an exception. A job + id outside ``[A-Za-z0-9_-]+`` is ``not_found`` without a request. + """ + + job_id = _require_remember_job_id(job_id) + path = _remember_status_path(job_id) + if path is None: + return RememberJobStatus( + job_id=job_id, + status="not_found", + error=f"Job {job_id} not found", + ) + try: + data = await self._signed_request( + "GET", + path, + {}, + accepted_statuses=(200, 404), + # Status is metadata. No SEAL decrypt, so no session. Matches TS. + include_seal_session=False, + ) + except (json.JSONDecodeError, UnicodeDecodeError): + data = None + if not isinstance(data, dict): + return RememberJobStatus( + job_id=job_id, + status="not_found", + error=f"remember job not found: {job_id}", + ) + status = data.get("status") + if not status: + return RememberJobStatus( + job_id=data.get("job_id") or job_id, + status="not_found", + error=data.get("error") or f"remember job not found: {job_id}", + ) + return RememberJobStatus( + job_id=data.get("job_id", job_id), + status=status, + owner=data.get("owner"), + namespace=data.get("namespace"), + blob_id=data.get("blob_id"), + error=data.get("error"), + ) + async def wait_for_remember_job( self, job_id: str, @@ -542,6 +622,11 @@ async def wait_for_remember_job( - First poll is immediate; later polls grow 1.5x toward 5s. """ + job_id = _require_remember_job_id(job_id) + path = _remember_status_path(job_id) + if path is None: + raise MemWalRememberJobNotFound(job_id) + deadline_ms = _now_ms() + timeout_ms attempt = 0 saw_rate_limit = False @@ -563,7 +648,7 @@ async def wait_for_remember_job( try: data = await self._signed_request( "GET", - f"/api/remember/{job_id}", + path, {}, accepted_statuses=(200, 404), timeout=poll_timeout, @@ -572,12 +657,14 @@ async def wait_for_remember_job( if "unexpected keyword argument" in str(err) and "timeout" in str(err): data = await self._signed_request( "GET", - f"/api/remember/{job_id}", + path, {}, accepted_statuses=(200, 404), ) else: raise + except (json.JSONDecodeError, UnicodeDecodeError): + raise MemWalRememberJobNotFound(job_id) from None except _HttpStatusError as err: if _is_transient_polling_status(err.status): last_refusal = err.status @@ -592,6 +679,8 @@ async def wait_for_remember_job( last_refusal = 0 continue + if not isinstance(data, dict): + raise MemWalRememberJobNotFound(job_id) status_str = data.get("status") if status_str is None or status_str == "not_found": raise MemWalRememberJobNotFound(job_id) @@ -1964,6 +2053,10 @@ def remember_async( ) -> RememberAcceptedResult: return self._run(self._inner.remember_async(text, namespace, idempotency_key)) + def get_remember_status(self, job_id: str) -> RememberJobStatus: + """Synchronous version of :meth:`MemWal.get_remember_status`.""" + return self._run(self._inner.get_remember_status(job_id)) + def wait_for_remember_job( self, job_id: str, diff --git a/packages/python-sdk-memwal/tests/test_client.py b/packages/python-sdk-memwal/tests/test_client.py index 9f0212ec9..1be9dee39 100644 --- a/packages/python-sdk-memwal/tests/test_client.py +++ b/packages/python-sdk-memwal/tests/test_client.py @@ -10,6 +10,7 @@ import base64 import json import time +import uuid from datetime import datetime, timedelta, timezone from typing import Any, Sequence @@ -24,6 +25,7 @@ MemWalCompatibilityError, MemWalError, MemWalRateLimited, + MemWalRememberJobNotFound, MemWalRememberJobTimeout, MemWalSync, _HttpStatusError, @@ -36,6 +38,7 @@ RememberBulkOptions, RememberBulkStatusItem, RememberBulkStatusResult, + RememberJobStatus, RememberManualOptions, ScoringWeights, ) @@ -642,6 +645,151 @@ async def running_then_refused(*_args: Any, **_kwargs: Any) -> dict: ) +# ============================================================ +# one-job remember status +# ============================================================ + + +_NORMAL_JOB_ID = "550e8400-e29b-41d4-a716-446655440000" + + +class TestRememberJobStatus: + @respx.mock + async def test_unsafe_job_id_is_not_requested(self, memwal_client: MemWal) -> None: + route = respx.route().mock(return_value=httpx.Response(500, text="nope")) + for job_id in ("", "abc/def?x=1", "job 1", "a\tb", "?", "/", ".", "..", "%2e%2e", "a\\b"): + status = await memwal_client.get_remember_status(job_id) + assert status == RememberJobStatus( + job_id=job_id, + status="not_found", + error=f"Job {job_id} not found", + ) + with pytest.raises(MemWalRememberJobNotFound) as exc: + await memwal_client.wait_for_remember_job( + job_id, poll_interval_ms=0, timeout_ms=5_000 + ) + assert exc.value.status == 404 + assert exc.value.job_id == job_id + assert not isinstance(exc.value, json.JSONDecodeError) + assert route.call_count == 0 + + async def test_non_string_job_id_is_a_type_error(self, memwal_client: MemWal) -> None: + with pytest.raises(TypeError): + await memwal_client.get_remember_status(1) # type: ignore[arg-type] + with pytest.raises(TypeError): + await memwal_client.wait_for_remember_job(1, poll_interval_ms=0, timeout_ms=5_000) # type: ignore[arg-type] + + @respx.mock + async def test_uuid_object_reads_the_string_path(self, memwal_client: MemWal) -> None: + _mock_version() + job = uuid.UUID(_NORMAL_JOB_ID) + route = respx.get(f"{_TEST_SERVER}/api/remember/{job}").mock( + return_value=httpx.Response(200, json={"job_id": str(job), "status": "running"}) + ) + status = await memwal_client.get_remember_status(job) # type: ignore[arg-type] + assert status.status == "running" + assert status.job_id == str(job) + assert route.call_count == 1 + + @respx.mock + async def test_non_object_404_body_is_not_found(self, memwal_client: MemWal) -> None: + mock_seal_session_prereqs() + route = respx.get(f"{_TEST_SERVER}/api/remember/job-1") + for body in (b"null", b"[]", b'"Not Found"', b"nope"): + route.mock(return_value=httpx.Response(404, content=body)) + status = await memwal_client.get_remember_status("job-1") + assert status.status == "not_found" + assert status.job_id == "job-1" + with pytest.raises(MemWalRememberJobNotFound) as exc: + await memwal_client.wait_for_remember_job( + "job-1", poll_interval_ms=0, timeout_ms=5_000 + ) + assert exc.value.status == 404 + + @respx.mock + async def test_empty_404_body_is_not_found(self, memwal_client: MemWal) -> None: + mock_seal_session_prereqs() + respx.get(f"{_TEST_SERVER}/api/remember/job-1").mock( + return_value=httpx.Response(404, content=b"") + ) + status = await memwal_client.get_remember_status("job-1") + assert status.status == "not_found" + assert status.job_id == "job-1" + with pytest.raises(MemWalRememberJobNotFound) as exc: + await memwal_client.wait_for_remember_job( + "job-1", poll_interval_ms=0, timeout_ms=5_000 + ) + assert exc.value.status == 404 + + @respx.mock + async def test_reads_a_normal_job_id(self, memwal_client: MemWal) -> None: + mock_seal_session_prereqs() + route = respx.get(f"{_TEST_SERVER}/api/remember/{_NORMAL_JOB_ID}").mock( + return_value=httpx.Response( + 200, + json={ + "job_id": _NORMAL_JOB_ID, + "status": "done", + "blob_id": "blob-1", + "owner": "0xowner", + "namespace": "profile", + }, + ) + ) + + status = await memwal_client.get_remember_status(_NORMAL_JOB_ID) + assert status == RememberJobStatus( + job_id=_NORMAL_JOB_ID, + status="done", + owner="0xowner", + namespace="profile", + blob_id="blob-1", + ) + assert "x-seal-session" not in route.calls[0].request.headers + + result = await memwal_client.wait_for_remember_job( + _NORMAL_JOB_ID, poll_interval_ms=0, timeout_ms=5_000 + ) + assert result.blob_id == "blob-1" + assert result.namespace == "profile" + assert [str(call.request.url) for call in route.calls] == [ + f"{_TEST_SERVER}/api/remember/{_NORMAL_JOB_ID}", + f"{_TEST_SERVER}/api/remember/{_NORMAL_JOB_ID}", + ] + + @respx.mock + def test_sync_reads_a_normal_job_id(self) -> None: + _mock_version() + route = respx.get(f"{_TEST_SERVER}/api/remember/{_NORMAL_JOB_ID}").mock( + return_value=httpx.Response( + 200, + json={ + "job_id": _NORMAL_JOB_ID, + "status": "running", + "namespace": "profile", + }, + ) + ) + client = MemWalSync.create( + key=_TEST_KEY_HEX, + account_id=_TEST_ACCOUNT_ID, + server_url=_TEST_SERVER, + ) + try: + missing = client.get_remember_status("") + assert missing.status == "not_found" + with pytest.raises(MemWalRememberJobNotFound): + client.wait_for_remember_job("abc/def?x=1", poll_interval_ms=0, timeout_ms=5_000) + status = client.get_remember_status(_NORMAL_JOB_ID) + assert status.status == "running" + assert status.namespace == "profile" + assert str(route.calls[0].request.url) == ( + f"{_TEST_SERVER}/api/remember/{_NORMAL_JOB_ID}" + ) + finally: + client.close() + + # ============================================================ # recall() tests # ============================================================ diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index a0d343030..2977b8890 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -5,6 +5,7 @@ ### Fixed - `createAccount`, `addDelegateKey`, `removeDelegateKey`, and `MemWalManual` work again without a caller-supplied `suiClient`. They looked for `SuiClient` in `@mysten/sui/client`, which current `@mysten/sui` no longer exports, and threw `SuiClient not found`. The default is now `SuiGrpcClient` against the public fullnode for `suiNetwork`, because those fullnodes no longer serve JSON-RPC. An explicit `suiClient` still wins. (WALM-164, #300) +- `getRememberStatus` and `waitForRememberJob` reject a job id outside `[A-Za-z0-9_-]+` before signing. `.`, `..`, `\`, and `%2e%2e` used to pass a denylist and be rewritten by `fetch` after the path was signed. Valid UUID job ids are unchanged. (#1087, #1216) - `MemWalMock.listNamespaces` rejects a non-positive or non-integer `limit` with `limit must be positive`, matching the relayer. `limit: 0` used to return `has_more: true` and `next_cursor: null`, so the documented pagination loop never finished, and `limit: -1` silently dropped the last namespace. (#1144, #1145) - `MemWalMock.rememberBulkAsync` rejects an empty or non-array batch with the same error as `MemWal`. (#1146) - `MemWalMock` honors `idempotencyKey`. A retry of the same text and namespace returns the original job, a different payload throws, and `forget` / `clear` release the key so the text can be written again. (#1147) diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index d648b8feb..4c9716e30 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -134,6 +134,20 @@ interface SignedRequestOptions { type RememberStatusResponse = RememberJobStatus | { error?: string }; +/** Path for one remember-job read, or undefined when `jobId` is not one safe segment. + * + * `fetch` rewrites the path after the request is signed. `.` becomes + * `/api/remember/`, `..` and `%2e%2e` become `/api/`, and a backslash becomes + * a slash. An allowlist keeps those from leaving the client. A UUID is + * unchanged. (#1087, #1216) + */ +function rememberStatusPath(jobId: string): string | undefined { + if (typeof jobId !== "string" || !/^[A-Za-z0-9_-]+$/.test(jobId)) { + return undefined; + } + return `/api/remember/${jobId}`; +} + function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -546,9 +560,8 @@ export class MemWal { * resolves at the terminal state and hides progress. */ async getRememberStatus(jobId: string): Promise { - // `GET /api/remember/` is not a route. The relayer answers 404 with an - // empty body, and parsing that threw SyntaxError. (#1087) - if (typeof jobId !== "string" || jobId.length === 0) { + const path = rememberStatusPath(jobId); + if (path === undefined) { return { job_id: jobId, status: "not_found", @@ -557,7 +570,7 @@ export class MemWal { } const status = await this.signedRequest( "GET", - `/api/remember/${jobId}`, + path, {}, [200, 404], // Status is metadata. No SEAL decrypt, so no session. (#1135) @@ -582,7 +595,8 @@ export class MemWal { jobId: string, opts: { pollIntervalMs?: number; timeoutMs?: number } = {}, ): Promise { - if (typeof jobId !== "string" || jobId.length === 0) { + const path = rememberStatusPath(jobId); + if (path === undefined) { throw Object.assign(new Error(`remember job not found: ${jobId}`), { status: 404, jobId, @@ -613,7 +627,7 @@ export class MemWal { try { status = await this.signedRequest( "GET", - `/api/remember/${jobId}`, + path, {}, [200, 404], // Bound each poll by the client deadline, and never past diff --git a/packages/sdk/test/remember-call-shape.test.mjs b/packages/sdk/test/remember-call-shape.test.mjs index 7c37809d2..915195b00 100644 --- a/packages/sdk/test/remember-call-shape.test.mjs +++ b/packages/sdk/test/remember-call-shape.test.mjs @@ -162,6 +162,58 @@ test("an empty job id never requests GET /api/remember/", async () => { assert.equal(fetches, 0); }); +test("a slash or query job id is not written into the signed path", async () => { + const urls = []; + globalThis.fetch = async (url) => { + urls.push(String(url)); + throw new Error(`must not request ${url}`); + }; + + const memwal = client(); + for (const jobId of ["abc/def?x=1", "job 1", "a\tb", "?", "/", ".", "..", "%2e%2e", "a\\b"]) { + const status = await memwal.getRememberStatus(jobId); + assert.equal(status.status, "not_found"); + assert.equal(status.job_id, jobId); + const started = Date.now(); + await assert.rejects(memwal.waitForRememberJob(jobId), (err) => { + assert.equal(err.status, 404); + assert.match(err.message, /remember job not found/); + return true; + }); + assert.ok(Date.now() - started < 1_000); + } + assert.deepEqual(urls, []); +}); + +test("getRememberStatus reads one normal job id", async () => { + const jobId = "550e8400-e29b-41d4-a716-446655440000"; + const urls = []; + globalThis.fetch = async (url) => { + urls.push(String(url)); + const path = new URL(url).pathname; + if (path === "/version") return versionBody(); + if (path === `/api/remember/${jobId}` && new URL(url).search === "") { + return Response.json({ + job_id: jobId, + status: "running", + namespace: "profile", + blob_id: "blob-1", + }); + } + throw new Error(`unexpected request ${url}`); + }; + + const memwal = client(); + const status = await memwal.getRememberStatus(jobId); + assert.equal(status.status, "running"); + assert.equal(status.job_id, jobId); + assert.equal(status.namespace, "profile"); + assert.deepEqual(urls.map((url) => new URL(url).pathname), [ + "/version", + `/api/remember/${jobId}`, + ]); +}); + test("status reads do not build a SEAL session", async () => { const paths = []; globalThis.fetch = async (url, init = {}) => { From 91ce5c95997096fee236c0d501cc7da4f2b35a50 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Wed, 14 Oct 2026 18:28:27 +0700 Subject: [PATCH 43/70] fix(sdk): stop preflights from consuming recall's abort budget (WALM-598) (#883) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(sdk): stop preflights from consuming recall's abort budget (WALM-598) `recall()` armed a single 15s AbortController before calling `signedRequest`, which then awaited two unabortable, untimed preflights: `ensureCompatibleRelayer()` (`GET /version`, falling back to `GET /health`) and `buildSealSession()` (`GET /config` plus a Sui RPC round-trip for `SessionKey.create`). Only the final `/api/recall` fetch received the signal. A relayer whose `/health` p50 had drifted to ~9s therefore spent most of the caller's budget before recall was dispatched, and the resulting `AbortError` surfaced on the recall fetch even when recall itself was healthy (GH #438). Both preflights are memoized, so this hit the first recall of a process and every recall after the 5-minute SEAL session expiry. SDK: - Move the deadline into `signedRequest`, armed after the preflights resolve, so it covers only the request it names. - Give `/version`, `/health` and `/config` their own bounded budget (`preflightTimeoutMs`, default 5000). They previously carried no signal at all and could hang indefinitely. - Make recall's budget configurable via `MemWalConfig.recallTimeoutMs` and `RecallOptions.timeoutMs`; the default stays 15000. - Throw a `TimeoutError` whose `phase` names the round-trip that stalled instead of an unlabeled `AbortError`. Export `isTimeoutError`. - Bound the standalone `health()` and the manual client's `/version` -> `/health` fallback the same way. Server, in support of keeping `/health` a sub-second connectivity check: - Add single-flight to `sidecar_write_ready`. The 2s TTL was checked under the lock but not held across the probe, so every request arriving during an in-flight probe started its own. - Replace the blocking `std::sync::Mutex` with `tokio::sync::RwLock`. Besides parking a worker thread on an async path, the mutex could poison: the read side recovered but the write side was a bare `if let Ok(..)`, so one panic froze the cached timestamp and every subsequent `/health` fanned out to the sidecar for the life of the process. - Instrument the probe with `observe_external` / `record_sidecar_failure` and raise failure logging from `debug!` to `warn!`. The probe's latency and failure rate were previously unmeasurable. The `/health` response shape and its public, unauthenticated status are unchanged. * review: hold the deadline through the response body (WALM-598) fetchWithDeadline cleared its timer as soon as fetch() resolved — which is when the headers arrive, not the body. signedRequest then ran res.json() / res.text() with no deadline and no signal, so a server that answers promptly and trickles the body could outlive the budget the caller named. The old recall AbortController stayed armed through the body, so this was a regression in exactly the budget the PR claims to fix. - The read methods now carry the timer and release it when the body settles. An abort our own timer caused is still relabelled to a phase-tagged TimeoutError, so a slow body does not resurface as a bare AbortError. - The timer is unref'd so a response nobody reads cannot hold the loop open. - Tests for a trickled body and for a normal read releasing the timer. - docs/reference/configuration.md now lists recallTimeoutMs, preflightTimeoutMs and RecallOptions.timeoutMs, so the "full config surface" pointer in api-reference is true again. - Shrink the incident recap on the constants, recall(), the preflights and manual.ts to the contract; fix the admin.rs RwLock rationale, which claimed the old std::sync::Mutex parked a worker across the probe (it was dropped before .send().await). * review: rebase the changelog, make the timer test bite (WALM-598) - Rebased onto origin/dev. Notes fold into the existing `## 0.1.7` alongside the landed restore / 401 / tx.pure bullets instead of a separate `## Unreleased`, and mirror into docs/sdk/changelog.mdx. package.json stays at origin/dev's 0.1.7; no bump. - The "releases the timer" test only asserted `res.json()` returned the right value, which passes whether or not `release()` ran. It now spies the timer and asserts both halves of the fix: the deadline is still armed when fetch() resolves, and gone once the body settles. - Style-guide pass on docs/sdk/api-reference.md:59 and :128. * fix: drop an accidental packages/sdk/node_modules symlink A local node_modules symlink was staged with the previous commit. Root .gitignore lists `node_modules/`, which matches directories only, so a symlink of that name was never ignored. On any other checkout it is a broken link pointing at a machine-local absolute path, and pnpm install fails with ENOENT in packages/sdk. * fix(sdk): let the body deadline fire on an idle event loop (WALM-598) The deadline timer was `unref()`d, so once it became the thing that bounds the body read it could no longer fire when nothing else held the loop open. Node 22 drained the loop mid-test and cancelled the four body-deadline tests with 'Promise resolution is still pending but the event loop has already resolved'; a caller awaiting a trickled body on an otherwise-idle loop hit the same hole. Keep the timer referenced. An unread response body already holds a socket handle open in undici, so unref'ing bought nothing it was supposed to. * rebase: reconcile WALM-598 budgets with the shipped deadline contract dev landed WALM-648 (every request deadline, `MemWalRequestTimeout` + status 504 so a stalled poll is retried) and WALM-623 after this branch was cut. Reconciled rather than re-litigated: - Preflights and recall keep WALM-598's independent budgets, but run through the client's own `fetchWithDeadline`, which bounds the body read too. - `requestTimeoutError` gains `phase` and `timeoutMs`, so the stalled round-trip is named without renaming the shipped error. - `deadline_ms` is derived from the budget actually in force, so a caller's `timeoutMs` moves the relayer's clock with it. * fix(sdk): opt out of a non-positive recall deadline Rebased onto current dev. Admin health uses WRITE_READY_CACHE_TTL and WRITE_READY_PROBE_TIMEOUT. recall({ timeoutMs: 0 }) no longer aborts or sends deadline_ms: 1, and isTimeoutError matches MemWalRequestTimeout. * fix(sdk): drop the unused recall timeout alias CodeQL flagged RECALL_REQUEST_TIMEOUT_MS after recall started using the caller budget directly. * fix(sdk): honor a disabled recall deadline and the distance total A client recallTimeoutMs of 0 now omits deadline_ms, same as a per-call 0. maxDistance reports the filtered hit count again. --------- Co-authored-by: Harry Phan --- docs/reference/configuration.md | 6 + docs/sdk/api-reference.md | 10 + docs/sdk/changelog.mdx | 9 +- packages/sdk/CHANGELOG.md | 5 + packages/sdk/src/index.ts | 8 + packages/sdk/src/manual.ts | 25 +- packages/sdk/src/memwal.ts | 140 +++++-- packages/sdk/src/types.ts | 36 ++ packages/sdk/src/utils.ts | 141 +++++++ .../sdk/test/recall-timeout-budget.test.mjs | 348 ++++++++++++++++++ services/server/src/routes/admin.rs | 260 +++++++++++-- 11 files changed, 912 insertions(+), 76 deletions(-) create mode 100644 packages/sdk/test/recall-timeout-budget.test.mjs diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index b414f9686..4290a3343 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -50,6 +50,10 @@ Used by: | `accountId` | yes | MemWalAccount object ID on Sui | | `serverUrl` | no | Relayer URL. Default: `https://relayer.memory.walrus.xyz` | | `namespace` | no | Default memory boundary. Default: `"default"` | +| `recallTimeoutMs` | no | Abort budget for the `/api/recall` request itself. Default: `15000`. `0` disables the deadline | +| `preflightTimeoutMs` | no | Per-round-trip budget for the unauthenticated preflights (`GET /version`, `/health`, `/config`). Default: `5000` | + +`RecallOptions.timeoutMs` overrides `recallTimeoutMs` for a single `recall()` call. The recall clock starts after the preflights resolve, so a slow `/health` is never charged against it. ## `MemWalManualConfig` @@ -70,6 +74,7 @@ Core fields: | `registryId` | yes | `AccountRegistry` shared object ID on Sui | | `accountId` | yes | `MemWalAccount` object ID | | `namespace` | no | Default namespace | +| `preflightTimeoutMs` | no | Per-round-trip budget for the compatibility preflight. Default: `5000` | Sui signer fields: @@ -105,6 +110,7 @@ Walrus and network fields: ## Rules That Matter - `namespace` defaults to `"default"` when omitted. +- A blown deadline throws a `TimeoutError` whose `phase` names the round-trip that stalled; `isTimeoutError()` distinguishes it from a transport failure. - `MemWal` is the default relayer-handled path. - `MemWalManual` is the manual client path, but it still uses the relayer for registration, search, and restore. - `withMemWal` builds on top of `MemWal`, so it uses the same relayer-backed config shape. diff --git a/docs/sdk/api-reference.md b/docs/sdk/api-reference.md index 658b82718..875e0660f 100644 --- a/docs/sdk/api-reference.md +++ b/docs/sdk/api-reference.md @@ -50,9 +50,16 @@ Config: | `accountId` | `string` | Yes | — | MemWalAccount object ID on Sui | | `serverUrl` | `string` | No | `https://relayer.memory.walrus.xyz` | Relayer URL | | `namespace` | `string` | No | `"default"` | Default namespace for memory isolation | +| `recallTimeoutMs` | `number` | No | `15000` | Deadline for the `/api/recall` request. Per-call `RecallOptions.timeoutMs` overrides it | +| `preflightTimeoutMs` | `number` | No | `5000` | Deadline for each preflight round-trip (`GET /version`, `GET /health`, `GET /config`) | For the full config surface, see [Configuration](/reference/configuration). +`recallTimeoutMs` covers the recall request alone. The preflights that run +before it (the relayer compatibility check and the SEAL session build) are +bounded separately by `preflightTimeoutMs`, so a slow relayer preflight cannot +consume recall's budget. + ## `MemWal` Methods ### `remember(text, namespace?): Promise` @@ -120,6 +127,9 @@ Search for memories matching a natural language query, scoped to `owner + namesp - `maxDistance` filters weak matches client-side by dropping results where `distance >= maxDistance` - `sort` picks the ordering: `"relevance"` (default) or `"recent"` for newest-among-matches - `scoringWeights` blends recency and importance into the ranking (see [Ordering](#ordering) below) +- `timeoutMs` overrides the client's `recallTimeoutMs` for this call. `0` disables the deadline. On expiry + recall throws `MemWalRequestTimeout` (status 504). `isTimeoutError(err)` is true for that error + and for a preflight `TimeoutError` whose `phase` names the stalled round-trip. **Returns:** diff --git a/docs/sdk/changelog.mdx b/docs/sdk/changelog.mdx index 891f13860..fa32e7879 100644 --- a/docs/sdk/changelog.mdx +++ b/docs/sdk/changelog.mdx @@ -28,7 +28,7 @@ questions: - When was bulk remember added to the Walrus Memory SDK? - What security improvements have been made to the MemWal SDK? answer: >- - The latest TypeScript SDK release is 0.1.8. Request bodies are hashed with `@noble/hashes` rather than WebCrypto-or-`node:crypto`, so the SDK no longer imports a Node builtin that Vite silently externalises into a runtime crash in the browser, and it declares a Node 20 floor. `restore()` results include `failed` (required like `truncated`; SDK defaults omitted to `0`) for permanent decrypt/UTF-8 failures instead of folding them into `skipped`. Empty-body 401s use the AUTH_REJECTED troubleshooting message instead of telling callers to run `memwal_login`. Account and manual PTBs use typed `tx.pure` helpers so they work with modern `@mysten/sui`. An explicit `sort` on `recall()`, `"relevance"` included, now makes the relayer ignore `scoringWeights`; weights re-rank only when `sort` is omitted. 0.1.6 added optional `created_at` on `recall()` results, plus `sort` and `scoringWeights` on `RecallOptions`, and reports HTTP 503 as a retryable upstream outage instead of a sign-in failure. + The latest TypeScript SDK release is 0.1.9. `waitForRememberJob` grows its poll delay 1.5× toward a 5s cap after an immediate first poll, and a 429 during that wait is named on the timeout. Recall's abort budget is configurable through `recallTimeoutMs` and no longer spent by relayer preflights, a blown budget throws a `TimeoutError` naming the round-trip that stalled, and a request's deadline now covers the response body rather than stopping at its headers. Request bodies are hashed with `@noble/hashes` rather than WebCrypto-or-`node:crypto`, so the SDK no longer imports a Node builtin that Vite silently externalises into a runtime crash in the browser, and it declares a Node 20 floor. `restore()` results include `failed` (required like `truncated`; SDK defaults omitted to `0`) for permanent decrypt/UTF-8 failures instead of folding them into `skipped`. Empty-body 401s use the AUTH_REJECTED troubleshooting message instead of telling callers to run `memwal_login`. Account and manual PTBs use typed `tx.pure` helpers so they work with modern `@mysten/sui`. An explicit `sort` on `recall()`, `"relevance"` included, now makes the relayer ignore `scoringWeights`; weights re-rank only when `sort` is omitted. 0.1.6 added optional `created_at` on `recall()` results, plus `sort` and `scoringWeights` on `RecallOptions`, and reports HTTP 503 as a retryable upstream outage instead of a sign-in failure. --- ## Unreleased @@ -57,16 +57,21 @@ answer: >- ## 0.1.7 -This release removes the Node `crypto` import that crashed bundled browser builds at runtime, adds `failed` on `restore()` results, declares a Node 20 floor, stops telling headless SDK clients to call `memwal_login` on empty-body 401s, switches account and manual PTBs to typed `tx.pure` helpers, and makes an explicit `sort` on `recall()` win over `scoringWeights`. +This release stops relayer preflights from consuming recall's abort budget, removes the Node `crypto` import that crashed bundled browser builds at runtime, adds `failed` on `restore()` results, declares a Node 20 floor, stops telling headless SDK clients to call `memwal_login` on empty-body 401s, switches account and manual PTBs to typed `tx.pure` helpers, and makes an explicit `sort` on `recall()` win over `scoringWeights`. ### Added +- `MemWalConfig.recallTimeoutMs` and `RecallOptions.timeoutMs` make recall's abort budget configurable (default: 15000, unchanged). `MemWalConfig.preflightTimeoutMs` / `MemWalManualConfig.preflightTimeoutMs` bound each preflight round-trip (default: 5000). +- `isTimeoutError()` is true for the `MemWalRequestTimeout` recall throws (status 504) and for a preflight `TimeoutError` whose `phase` names the stalled round-trip. - `restore()` results include `failed` (required like `truncated`; SDK defaults omitted to `0`) for permanent decrypt/UTF-8 failures instead of folding them into `skipped` or dropping them silently. ### Fixed - Hash request bodies with `@noble/hashes` instead of WebCrypto-or-`node:crypto`, so the package no longer imports a Node builtin on a browser-reachable path. Vite externalises such an import without warning: the app builds clean and the browser crashes the first time the path runs. The fallback could never have helped a browser anyway — `crypto.subtle` is absent precisely when the page is not a secure context, where `node:crypto` is absent too — so it only served Node <19 while being the sole source of the exposure. `sha256hex` sits on the signed-request path, so every remember and recall reached it. (#322, WALM-136) - Declare `engines.node >= 20.0.0`, matching `memwal-mcp` and `openclaw-memory-memwal`. The SDK was the only published package without a floor. (WALM-599) +- `recall()`'s timeout budget covers only the recall request. It previously started before the relayer compatibility check and the SEAL session build, so a slow preflight silently consumed the budget and recall raised an `AbortError` even when recall itself was healthy. +- `GET /version`, `GET /health` and `GET /config` carry an `AbortSignal` and a deadline. They previously ran with neither and could hang indefinitely. +- A request's deadline now covers the response body, not just its headers. - Empty-body 401s now use the same AUTH_REJECTED troubleshooting message as credential 401s instead of telling callers to run `memwal_login`. Headless SDK clients do not have that MCP tool. - `account.ts` and `manual.ts` PTBs use typed `tx.pure` helpers instead of the legacy untyped moveCall argument syntax that fails under modern `@mysten/sui`. - An explicit `sort` on `recall()`, `"relevance"` included, is now the order: the relayer ignores `scoringWeights` for that request, and weights re-rank only when `sort` is omitted. Setting both used to return neither order, so `sort: "recent"` stopped meaning newest-first once `scoringWeights` carried a recency weight. (WALM-470) diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index 2977b8890..9a44f43a1 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -46,12 +46,17 @@ ### Added +- `MemWalConfig.recallTimeoutMs` and `RecallOptions.timeoutMs` make recall's abort budget configurable (default: 15000, unchanged). `MemWalConfig.preflightTimeoutMs` / `MemWalManualConfig.preflightTimeoutMs` bound each preflight round-trip (default: 5000). +- `isTimeoutError()` is true for the `MemWalRequestTimeout` recall throws (status 504) and for a preflight `TimeoutError` whose `phase` names the stalled round-trip. - `restore()` results include `failed` (required like `truncated`; SDK defaults omitted to `0`) for permanent decrypt/UTF-8 failures instead of folding them into `skipped` or dropping them silently. ### Fixed - Hash request bodies with `@noble/hashes` instead of WebCrypto-or-`node:crypto`, so the package no longer imports a Node builtin on a browser-reachable path. Vite externalises such an import without warning: the app builds clean and the browser crashes the first time the path runs. The fallback could never have helped a browser anyway — `crypto.subtle` is absent precisely when the page is not a secure context, where `node:crypto` is absent too — so it only served Node <19 while being the sole source of the exposure. `sha256hex` sits on the signed-request path, so every remember and recall reached it. (#322, WALM-136) - Declare `engines.node >= 20.0.0`, matching `memwal-mcp` and `openclaw-memory-memwal`. The SDK was the only published package without a floor. (WALM-599) +- `recall()`'s timeout budget covers only the recall request. It previously started before `ensureCompatibleRelayer()` (`GET /version`, falling back to `GET /health`) and `buildSealSession()` (`GET /config` + Sui RPC), so a slow relayer preflight silently consumed the budget and recall raised an `AbortError` even when recall itself was healthy. +- `GET /version`, `GET /health` and `GET /config` carry an `AbortSignal` and a deadline. They previously ran with neither and could hang indefinitely. +- A request's deadline now covers the response body, not just its headers. `fetch()` resolves when headers arrive, so a server that then trickled the body could outlive the budget the caller named. - Empty-body 401s now use the same AUTH_REJECTED troubleshooting message as credential 401s instead of telling callers to run `memwal_login`. Headless SDK clients do not have that MCP tool. - `account.ts` and `manual.ts` PTBs use typed `tx.pure` helpers instead of the legacy untyped moveCall argument syntax that fails under modern `@mysten/sui`. - An explicit `sort` on `recall()`, `"relevance"` included, is now the order: the relayer ignores `scoringWeights` for that request, and weights re-rank only when `sort` is omitted. Setting both used to return neither order, so `sort: "recent"` stopped meaning newest-first once `scoringWeights` carried a recency weight. (WALM-470) diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 9bb86184a..8d90f0169 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -24,6 +24,14 @@ export { // Delegate key utilities (no @mysten/sui dependency) export { delegateKeyToSuiAddress, delegateKeyToPublicKey } from "./utils.js"; +// Request deadlines. `isTimeoutError` tells a blown budget apart from a +// transport failure; `phase` says which round-trip blew it. +export { + isTimeoutError, + DEFAULT_RECALL_TIMEOUT_MS, + DEFAULT_PREFLIGHT_TIMEOUT_MS, +} from "./utils.js"; +export type { TimeoutError } from "./utils.js"; export { estimateTokens, truncateToTokenBudget, diff --git a/packages/sdk/src/manual.ts b/packages/sdk/src/manual.ts index 1acf0abe8..413b6223c 100644 --- a/packages/sdk/src/manual.ts +++ b/packages/sdk/src/manual.ts @@ -50,6 +50,8 @@ import { sanitizeServerError, clockDriftErrorFromResponse, scoringWeightsToWire, + fetchWithDeadline, + DEFAULT_PREFLIGHT_TIMEOUT_MS, } from "./utils.js"; import { assertCompatibleRelayer, compatibilityErrorFromStatus } from "./compatibility.js"; import { resolveLimitOrOptions } from "./recall-args.js"; @@ -159,6 +161,8 @@ export class MemWalManual { private namespace: string; private relayerVersionMetadata: RelayerVersionMetadata | null = null; private compatibilityPromise: Promise | null = null; + /** Per-round-trip budget for the compatibility preflight (WALM-598). */ + private preflightTimeoutMs: number; // Lazily initialized heavy clients (typed as any to avoid peer dep compile errors) private _suiClient: any = null; @@ -192,6 +196,7 @@ export class MemWalManual { throw new Error("namespace cannot be empty"); } this.namespace = config.namespace ?? "default"; + this.preflightTimeoutMs = config.preflightTimeoutMs ?? DEFAULT_PREFLIGHT_TIMEOUT_MS; } /** @@ -809,17 +814,25 @@ export class MemWalManual { } private async fetchCompatibilityMetadata(): Promise { - const versionRes = await fetch(`${this.serverUrl}/version`, { - method: "GET", - }); + // Bounded like the relayer-mode client's preflight: these run inside + // whatever signed request triggered them. + const versionRes = await fetchWithDeadline( + `${this.serverUrl}/version`, + { method: "GET" }, + this.preflightTimeoutMs, + "preflight GET /version", + ); let body: Partial; if (versionRes.ok) { body = (await versionRes.json()) as Partial; } else if (versionRes.status === 404 || versionRes.status === 405) { - const healthRes = await fetch(`${this.serverUrl}/health`, { - method: "GET", - }); + const healthRes = await fetchWithDeadline( + `${this.serverUrl}/health`, + { method: "GET" }, + this.preflightTimeoutMs, + "preflight GET /health", + ); if (!healthRes.ok) { throw new Error( `Walrus Memory compatibility check failed: GET /version returned ` + diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index 4c9716e30..f32750192 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -74,6 +74,8 @@ import { scoringWeightsToWire, derivedIdempotencyKey, derivedBulkIdempotencyKey, + DEFAULT_PREFLIGHT_TIMEOUT_MS, + DEFAULT_RECALL_TIMEOUT_MS, } from "./utils.js"; import { assertCompatibleRelayer, @@ -123,12 +125,17 @@ const SEAL_SESSION_TTL_MIN = 5; // a key server that sees it as expired. const SEAL_SESSION_SAFETY_MARGIN_MS = 30_000; -/** Per-call knobs for `signedRequest`. `timeoutMs` overrides the client-wide - * deadline for one endpoint; `signal` is the caller's own cancellation and is - * honoured alongside it, never replaced by it. */ +/** Per-call knobs for `MemWal.signedRequest`. */ interface SignedRequestOptions { + /** ENG-1696: omit the SEAL credential on Manual-mode routes. */ includeDelegateKey?: boolean; + /** Caller-owned cancellation, honoured alongside `timeoutMs`. */ signal?: AbortSignal; + /** + * Deadline for the signed request itself. The clock starts after the + * preflights resolve. Omit this to use the client `requestTimeoutMs`. + * `0`, a negative number, or a non-finite value disables the deadline. + */ timeoutMs?: number; } @@ -215,17 +222,14 @@ const RESTORE_REQUEST_TIMEOUT_MS = 60_000; * is the one write that legitimately outruns the default. */ const ANALYZE_REQUEST_TIMEOUT_MS = 60_000; -/** `POST /api/recall` has carried its own 15s deadline since before the rest - * had any; keeping it named makes that a decision rather than an accident. */ -const RECALL_REQUEST_TIMEOUT_MS = 15_000; -/** What `recall()` tells the relayer it will wait, so the 504 naming the stuck - * stage arrives before the abort above rather than after it. A second under - * that abort because the two clocks start at different moments: this timer - * starts before `fetch`, the relayer's when the request lands, and the margin - * the relayer keeps for its reply cannot also cover DNS, TCP and TLS. Derived - * from the timeout so the two can never drift apart. */ -const RECALL_DEADLINE_MS = RECALL_REQUEST_TIMEOUT_MS - 1_000; + +/** How far under the client's own abort `recall()` asks the relayer to stop, + * so the 504 naming the stuck stage arrives before the abort rather than + * after it. A second, because the two clocks start at different moments: this + * timer starts before `fetch`, the relayer's when the request lands, and the + * margin the relayer keeps for its reply cannot also cover DNS, TCP and TLS. */ +const RECALL_DEADLINE_MARGIN_MS = 1_000; /** * Abort signal that fires after `ms`, or when `caller` aborts — whichever is @@ -274,14 +278,23 @@ function deadlineSignal( * `status: 504` is load-bearing, not decoration: `isTransientPollingStatus` * treats it as retryable, so one stalled poll inside a wait loop is abandoned * and retried against the remaining budget instead of failing the whole wait. + * + * `phase` names the round-trip that actually stalled — `"preflight GET + * /version"`, `"POST /api/recall"` — so WALM-598's diagnostic question ("which + * step ate the budget?") is answerable from the error alone. The name stays + * `MemWalRequestTimeout`: that is the shipped contract callers and + * `isTransientPollingStatus` already key off (WALM-648). */ -function requestTimeoutError(method: string, path: string, ms: number): Error { +function requestTimeoutError(method: string, path: string, ms: number, phase?: string): Error { + const stage = phase ?? `${method} ${path}`; const err = new Error( - `Walrus Memory request timed out after ${ms}ms (${method} ${path}). The relayer ` + + `Walrus Memory request timed out after ${ms}ms during ${stage}. The relayer ` + `accepted the connection but did not answer in time.`, ); err.name = "MemWalRequestTimeout"; - (err as Error & { status?: number }).status = 504; + (err as Error & { status?: number; phase?: string; timeoutMs?: number }).status = 504; + (err as Error & { phase?: string }).phase = stage; + (err as Error & { timeoutMs?: number }).timeoutMs = ms; return err; } @@ -436,6 +449,13 @@ export class MemWal { private accountId: string; /** Deadline applied to any request that does not name its own. */ private requestTimeoutMs: number; + /** + * Recall's abort budget, and the independent per-round-trip budget for the + * unauthenticated preflights that precede it. Separate so a slow + * `/version`, `/health` or `/config` is never charged against recall. + */ + private recallTimeoutMs: number; + private preflightTimeoutMs: number; // ENG-1697 state — all internal, never surfaced to user code. // The public API (`MemWal.create({ key, accountId })`) is unchanged. @@ -485,6 +505,8 @@ export class MemWal { Number.isFinite(config.requestTimeoutMs) && (config.requestTimeoutMs as number) > 0 ? (config.requestTimeoutMs as number) : DEFAULT_REQUEST_TIMEOUT_MS; + this.recallTimeoutMs = config.recallTimeoutMs ?? DEFAULT_RECALL_TIMEOUT_MS; + this.preflightTimeoutMs = config.preflightTimeoutMs ?? DEFAULT_PREFLIGHT_TIMEOUT_MS; } /** @@ -1126,23 +1148,30 @@ export class MemWal { const resolvedNamespace = options.namespace ?? this.namespace; { + // WALM-598: this budget is handed to `signedRequest`, which starts + // the clock on the `/api/recall` fetch itself — after + // `ensureCompatibleRelayer` (`/version`, falling back to `/health`) + // and `buildSealSession` (`/config` + Sui RPC) have already + // resolved. Those preflights carry their own independent + // deadlines, so a relayer whose `/health` p50 had crept to ~9s can + // no longer eat most of recall's window and make a perfectly + // healthy recall surface an `AbortError` (GH #438). + const budget = options.timeoutMs ?? this.recallTimeoutMs; + const optedOut = typeof budget !== "number" || !Number.isFinite(budget) || budget <= 0; + const timeoutMs = optedOut ? 0 : budget; + const wireDeadline = optedOut + ? undefined + : (timeoutMs > RECALL_DEADLINE_MARGIN_MS + ? timeoutMs - RECALL_DEADLINE_MARGIN_MS + : timeoutMs); const result = await this.signedRequest("POST", "/api/recall", { query, limit, namespace: resolvedNamespace, - // `undefined` when no weights were supplied, which - // JSON.stringify drops — so a default-weighted recall stays - // byte-identical on the wire and the relayer keeps - // short-circuiting to the plain pgvector cosine order. scoring_weights: scoringWeightsToWire(options.scoringWeights), - // Same `undefined`-is-dropped trick: an unset sort leaves the - // request byte-identical and the relayer applies its own - // "relevance" default. sort: options.sort, - // How long this call waits, so the relayer can stop just - // short of it and name the step it was stuck in. - deadline_ms: RECALL_DEADLINE_MS, - }, { timeoutMs: RECALL_REQUEST_TIMEOUT_MS }); + ...(wireDeadline === undefined ? {} : { deadline_ms: wireDeadline }), + }, { timeoutMs }); let processed = result; if (typeof options.maxDistance === "number") { @@ -1535,7 +1564,12 @@ export class MemWal { * Check server health. The endpoint is public and does not require request signing. */ async health(): Promise { - const res = await this.fetchWithDeadline(`${this.serverUrl}/health`); + const res = await this.fetchWithDeadline( + `${this.serverUrl}/health`, + {}, + this.preflightTimeoutMs, + "GET /health", + ); if (!res.ok) { throw new Error(`Health check failed: ${res.status}`); } @@ -1591,9 +1625,15 @@ export class MemWal { } private async fetchCompatibilityMetadata(): Promise { + // WALM-598: both round-trips get their own bounded budget. They run + // inside whatever signed request triggered the compatibility probe, so + // leaving them untimed let a stalled relayer silently drain that + // caller's deadline (and, with no signal at all, hang indefinitely). const versionRes = await this.fetchWithDeadline>( `${this.serverUrl}/version`, { method: "GET" }, + this.preflightTimeoutMs, + "preflight GET /version", ); let body: Partial; @@ -1603,6 +1643,8 @@ export class MemWal { const healthRes = await this.fetchWithDeadline>( `${this.serverUrl}/health`, { method: "GET" }, + this.preflightTimeoutMs, + "preflight GET /health", ); if (!healthRes.ok) { throw new Error( @@ -1646,9 +1688,13 @@ export class MemWal { private async fetchServerConfig(): Promise { if (this.serverConfig) return this.serverConfig; + // WALM-598: bounded like the other preflights — this one is awaited + // inside `buildSealSession`, itself awaited inside a caller's budget. const res = await this.fetchWithDeadline>( `${this.serverUrl}/config`, { method: "GET" }, + this.preflightTimeoutMs, + "preflight GET /config", ); if (!res.ok) { throw new Error(`GET /config returned ${res.status}`); @@ -1887,6 +1933,7 @@ export class MemWal { url: string, init: RequestInit = {}, timeoutMs?: number, + phase?: string, ): Promise<{ ok: boolean; status: number; body: T | undefined }> { const ms = timeoutMs ?? this.requestTimeoutMs; const deadline = deadlineSignal(ms); @@ -1899,7 +1946,7 @@ export class MemWal { }; } catch (err) { if (deadline.timedOut()) { - throw requestTimeoutError(init.method ?? "GET", url, ms); + throw requestTimeoutError(init.method ?? "GET", url, ms, phase); } throw err; } finally { @@ -1968,17 +2015,31 @@ export class MemWal { if (options.includeDelegateKey !== false) { headers["x-seal-session"] = await this.buildSealSession(); } - // Bound the request. `fetch` never times out on its own, so this is the - // only thing standing between a stalled socket and a call that hangs - // for as long as the connection stays open. - const deadlineMs = options.timeoutMs ?? this.requestTimeoutMs; - const deadline = deadlineSignal(deadlineMs, options.signal); + // WALM-598: the caller's deadline starts HERE, not at the top of the + // call. Everything above — `ensureCompatibleRelayer()` (`/version` -> + // `/health`) and `buildSealSession()` (`/config`, dynamic imports, a + // Sui RPC round-trip for `SessionKey.create`) — is preflight work that + // used to run inside the caller's budget while being unabortable + // itself. Each of those now carries its own independent deadline. + // + // `fetch` never times out on its own, so this deadline is also the only + // thing standing between a stalled socket and a call that hangs for as + // long as the connection stays open. + const explicitTimeout = options.timeoutMs; + const deadlineOptOut = typeof explicitTimeout === "number" + && (!Number.isFinite(explicitTimeout) || explicitTimeout <= 0); + const deadlineMs = deadlineOptOut + ? undefined + : (explicitTimeout ?? this.requestTimeoutMs); + const deadline = deadlineMs === undefined + ? undefined + : deadlineSignal(deadlineMs, options.signal); try { const res = await fetch(url, { method, headers, body: method === "GET" ? undefined : bodyStr, - signal: deadline.signal, + signal: deadline?.signal ?? options.signal, }); if (!acceptedStatuses.includes(res.status)) { @@ -2019,12 +2080,15 @@ export class MemWal { } catch (err) { // Translate our own expiry into something a caller can classify. // An abort the CALLER asked for is theirs and propagates untouched. - if (deadline.timedOut() && (err as { status?: number }).status === undefined) { - throw requestTimeoutError(method, path, deadlineMs); + const status = err && typeof err === "object" && "status" in err + ? err.status + : undefined; + if (deadline?.timedOut() && typeof status !== "number") { + throw requestTimeoutError(method, path, deadlineMs ?? 0); } throw err; } finally { - deadline.dispose(); + deadline?.dispose(); } } } diff --git a/packages/sdk/src/types.ts b/packages/sdk/src/types.ts index 2037c9caf..4b182227f 100644 --- a/packages/sdk/src/types.ts +++ b/packages/sdk/src/types.ts @@ -33,6 +33,23 @@ export interface MemWalConfig { * Raise it only for a genuinely slow link; it is a backstop, not a budget. */ requestTimeoutMs?: number; + /** + * Deadline for the `/api/recall` request itself, in milliseconds + * (default: 15000). Per-call `RecallOptions.timeoutMs` overrides it. + * + * This budget covers only the recall round-trip. The preflights that + * precede it (`/version`, `/config`, the SEAL session build) are bounded + * separately by `preflightTimeoutMs`, so a slow relayer preflight can no + * longer consume recall's budget and surface as a spurious abort. + */ + recallTimeoutMs?: number; + /** + * Deadline for each unauthenticated preflight fetch — `GET /version`, + * `GET /health`, `GET /config` — in milliseconds (default: 5000). + * + * Applied per round-trip, not to the preflight phase as a whole. + */ + preflightTimeoutMs?: number; } // ============================================================ @@ -198,6 +215,19 @@ export interface RecallOptions { * Setting `sort` at all makes the relayer ignore `scoringWeights`. */ sort?: "relevance" | "recent"; + /** + * Deadline for the `/api/recall` request, in milliseconds. Defaults to + * `MemWalConfig.recallTimeoutMs`, itself 15000. + * + * The clock starts once the preflights (`/version`, `/config`, the SEAL + * session build) have completed, so this is a budget for recall alone. + * Pass `0` (or `Infinity`) to opt out of the deadline entirely. + * + * On expiry recall throws a `TimeoutError` whose `phase` is + * `"POST /api/recall"`; a preflight that blows its own budget throws one + * naming that preflight instead. + */ + timeoutMs?: number; } /** Recommended object-style recall input — preferred over positional args. */ @@ -645,6 +675,12 @@ export interface MemWalManualConfig { walrusPublisherUrl?: string; /** Default namespace for memory isolation (default: "default") */ namespace?: string; + /** + * Deadline for the unauthenticated compatibility preflight (`GET /version`, + * falling back to `GET /health`), in milliseconds (default: 5000). + * Applied per round-trip. + */ + preflightTimeoutMs?: number; } /** diff --git a/packages/sdk/src/utils.ts b/packages/sdk/src/utils.ts index 3e58da1e8..636258423 100644 --- a/packages/sdk/src/utils.ts +++ b/packages/sdk/src/utils.ts @@ -420,6 +420,147 @@ export function clockDriftErrorFromResponse( return err; } +// ============================================================ +// Bounded fetch (WALM-598) +// ============================================================ + +/** + * Default deadline for the unauthenticated preflight round-trips + * (`GET /version`, `GET /health`, `GET /config`) the SDK makes before it can + * dispatch a signed request. + * + * Each preflight carries its own budget: they are awaited inside a caller's + * request budget, so an unbounded one both hangs forever and spends someone + * else's deadline (WALM-598). + */ +export const DEFAULT_PREFLIGHT_TIMEOUT_MS = 5_000; + +/** Default deadline for the `/api/recall` request itself. */ +export const DEFAULT_RECALL_TIMEOUT_MS = 15_000; + +/** + * Error thrown when one of the SDK's bounded fetches blows its deadline. + * + * `phase` names the round-trip that actually stalled — `"preflight GET + * /version"`, `"POST /api/recall"` — which is the diagnostic that was missing + * when every stall in the chain surfaced as an unlabeled `AbortError` on the + * final request. + */ +export interface TimeoutError extends Error { + name: "TimeoutError"; + /** Which round-trip exceeded its budget. */ + phase: string; + /** The budget that was exceeded, in milliseconds. */ + timeoutMs: number; +} + +/** True for a `TimeoutError` or the `MemWalRequestTimeout` recall throws. */ +export function isTimeoutError(err: unknown): err is Error { + if (!(err instanceof Error)) return false; + if (err.name === "MemWalRequestTimeout") return true; + return err.name === "TimeoutError" && "phase" in err; +} + +function timeoutError(phase: string, timeoutMs: number, cause: unknown): TimeoutError { + const err = new Error( + `Walrus Memory request timed out after ${timeoutMs}ms during ${phase}.`, + ) as TimeoutError; + err.name = "TimeoutError"; + err.phase = phase; + err.timeoutMs = timeoutMs; + (err as Error & { cause?: unknown }).cause = cause; + return err; +} + +/** + * `fetch` under an independent abort budget. + * + * A non-finite or non-positive `timeoutMs` disables the deadline (the caller + * opted out); an `external` signal is still honoured in that case. + * + * Aborts triggered by our own timer are rethrown as a `TimeoutError` tagged + * with `phase`; an abort that came from `external` is left alone so callers + * keep distinguishing "I cancelled this" from "this timed out". + */ +export async function fetchWithDeadline( + url: string, + init: RequestInit, + timeoutMs: number, + phase: string, + external?: AbortSignal, +): Promise { + const bounded = Number.isFinite(timeoutMs) && timeoutMs > 0; + if (!bounded && !external) return fetch(url, init); + + const ac = new AbortController(); + let timedOut = false; + const onExternalAbort = () => ac.abort(); + if (external) { + if (external.aborted) ac.abort(); + else external.addEventListener("abort", onExternalAbort, { once: true }); + } + // Deliberately not `unref()`d: this timer is what bounds the body read + // below, so it has to be able to fire on an otherwise-idle loop. + const tid = bounded + ? setTimeout(() => { + timedOut = true; + ac.abort(); + }, timeoutMs) + : undefined; + + const release = () => { + if (tid !== undefined) clearTimeout(tid); + external?.removeEventListener("abort", onExternalAbort); + }; + const relabel = (err: unknown) => (timedOut ? timeoutError(phase, timeoutMs, err) : err); + + let res: Response; + try { + res = await fetch(url, { ...init, signal: ac.signal }); + } catch (err) { + release(); + throw relabel(err); + } + return armBodyDeadline(res, release, relabel); +} + +/** + * Keep the deadline armed until the response body settles. + * + * `fetch()` resolves when the response *headers* arrive, so clearing the timer + * there would leave `res.json()` / `res.text()` unbounded: a server that sends + * headers promptly and then trickles the body could outlive the budget the + * caller named. The read methods carry the timer instead, and relabel an abort + * our own timer caused so a slow body still surfaces as a phase-tagged + * `TimeoutError` rather than a bare `AbortError`. + */ +function armBodyDeadline( + res: Response, + release: () => void, + relabel: (err: unknown) => unknown, +): Response { + const target = res as unknown as Record; + for (const name of ["json", "text", "arrayBuffer", "blob", "formData"]) { + const original = target[name]; + if (typeof original !== "function") continue; + const read = (original as (...args: unknown[]) => Promise).bind(res); + Object.defineProperty(target, name, { + configurable: true, + writable: true, + value: async (...args: unknown[]) => { + try { + return await read(...args); + } catch (err) { + throw relabel(err); + } finally { + release(); + } + }, + }); + } + return res; +} + // ============================================================ // Delegate Key → Sui Address Derivation // ============================================================ diff --git a/packages/sdk/test/recall-timeout-budget.test.mjs b/packages/sdk/test/recall-timeout-budget.test.mjs new file mode 100644 index 000000000..24507196f --- /dev/null +++ b/packages/sdk/test/recall-timeout-budget.test.mjs @@ -0,0 +1,348 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +import { MemWal } from "../dist/memwal.js"; +import { fetchWithDeadline, isTimeoutError, DEFAULT_RECALL_TIMEOUT_MS } from "../dist/utils.js"; + +// The shipped deadline contract (WALM-648): name `MemWalRequestTimeout`, status +// 504 so `isTransientPollingStatus` retries one stalled poll, plus the `phase` +// WALM-598 adds so the stalled round-trip is named. +const isRequestTimeout = (err) => + err instanceof Error && err.name === "MemWalRequestTimeout" && typeof err.phase === "string"; + +// WALM-598 / GH #438. +// +// `recall()` used to arm a single 15s AbortController *before* calling +// `signedRequest`, which then awaited two unabortable, untimed preflights — +// `ensureCompatibleRelayer()` (`GET /version`, falling back to `GET /health`) +// and `buildSealSession()` (`GET /config` + a Sui RPC round-trip). A relayer +// whose `/health` p50 had drifted to ~9s therefore spent most of the caller's +// budget before `/api/recall` was even dispatched, and the resulting +// `AbortError` surfaced on the recall fetch even though recall was healthy. +// +// The budget now starts on the recall request itself; the preflights carry +// their own independent deadlines. + +const originalFetch = globalThis.fetch; + +test.afterEach(() => { + globalThis.fetch = originalFetch; +}); + +/** + * Resolve `factory()` after `ms`, or reject with an `AbortError` if the + * caller's signal fires first — i.e. behave like a real `fetch`. Without the + * signal handling these tests could not observe an abort at all. + */ +function delayed(ms, factory, signal) { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(abortError()); + return; + } + const tid = setTimeout(() => resolve(factory()), ms); + signal?.addEventListener( + "abort", + () => { + clearTimeout(tid); + reject(abortError()); + }, + { once: true }, + ); + }); +} + +function abortError() { + return Object.assign(new Error("This operation was aborted"), { name: "AbortError" }); +} + +const VERSION_BODY = { + apiVersion: "1.0.0", + relayerVersion: "1.0.0", + minSupportedSdk: { typescript: "0.0.4" }, +}; + +/** + * Route stub with per-path latency. `never` means "hang until aborted", + * which is how a wedged preflight looks to the client. + */ +function stubServer({ versionDelayMs = 0, recallDelayMs = 0, hits = [] } = {}) { + const seen = []; + globalThis.fetch = async (url, init = {}) => { + const path = new URL(url).pathname; + seen.push(path); + if (path === "/version") { + if (versionDelayMs === "never") return delayed(1e9, () => null, init.signal); + return delayed(versionDelayMs, () => Response.json(VERSION_BODY), init.signal); + } + if (path === "/api/recall" && init.method === "POST") { + return delayed( + recallDelayMs, + () => Response.json({ results: hits, total: hits.length }), + init.signal, + ); + } + throw new Error(`unexpected request ${path}`); + }; + return seen; +} + +function client(config = {}) { + const c = MemWal.create({ + key: new Uint8Array(32).fill(1), + accountId: "0x1", + serverUrl: "https://relayer.example", + ...config, + }); + // Standard SEAL stub — building a real SessionKey needs @mysten/seal and + // a live Sui RPC. `buildSealSession` is awaited inside `signedRequest`, + // i.e. in the same preflight window as `/config`. + c.buildSealSession = async () => "test-session"; + return c; +} + +test("a slow /version preflight does not abort a healthy recall", async () => { + // The regression, reproduced at test speed: the preflight takes longer + // than the entire recall budget. Before WALM-598 the shared + // AbortController had already fired by the time `/api/recall` was + // dispatched, so this rejected with an unlabeled AbortError. + const seen = stubServer({ versionDelayMs: 120, recallDelayMs: 0 }); + + const result = await client({ recallTimeoutMs: 40 }).recall({ query: "food allergies" }); + + assert.deepEqual(result.results, []); + assert.deepEqual(seen, ["/version", "/api/recall"]); +}); + +test("a slow SEAL session build does not abort a healthy recall", async () => { + // `buildSealSession()` is the other unabortable preflight inside the old + // budget: on a cold cache it fetches /config, dynamic-imports @mysten/seal + // and @mysten/sui, and calls SessionKey.create() against Sui RPC. + stubServer({ recallDelayMs: 0 }); + + const c = client({ recallTimeoutMs: 40 }); + c.buildSealSession = async () => { + await new Promise((resolve) => setTimeout(resolve, 120)); + return "test-session"; + }; + + const result = await c.recall({ query: "food allergies" }); + assert.deepEqual(result.results, []); +}); + +test("recall honours a per-call timeoutMs", async () => { + stubServer({ recallDelayMs: 5_000 }); + + await assert.rejects( + client().recall({ query: "food allergies", timeoutMs: 30 }), + (err) => { + assert.ok(isRequestTimeout(err), `expected a MemWalRequestTimeout, got ${err?.name}`); + assert.equal(err.timeoutMs, 30); + return true; + }, + ); +}); + +test("recall honours a client-level recallTimeoutMs", async () => { + stubServer({ recallDelayMs: 5_000 }); + + await assert.rejects(client({ recallTimeoutMs: 30 }).recall({ query: "q" }), (err) => { + assert.ok(isRequestTimeout(err)); + assert.equal(err.timeoutMs, 30); + return true; + }); +}); + +test("a per-call timeoutMs overrides the client default", async () => { + stubServer({ recallDelayMs: 5_000 }); + + await assert.rejects( + client({ recallTimeoutMs: 60_000 }).recall({ query: "q", timeoutMs: 25 }), + (err) => { + assert.equal(err.timeoutMs, 25); + return true; + }, + ); +}); + +test("a recall timeout names the recall request as the phase", async () => { + // The diagnostic that was missing: every stall in the chain used to + // surface as an unlabeled AbortError on the recall fetch. + stubServer({ recallDelayMs: 5_000 }); + + await assert.rejects(client().recall({ query: "q", timeoutMs: 25 }), (err) => { + assert.equal(err.phase, "POST /api/recall"); + assert.match(err.message, /timed out after 25ms during POST \/api\/recall/); + return true; + }); +}); + +test("a preflight timeout names the preflight as the phase", async () => { + // A wedged /version now fails on its own budget instead of hanging + // forever (it carried no signal at all before) or being misreported as + // a recall abort. + stubServer({ versionDelayMs: "never" }); + + await assert.rejects( + client({ preflightTimeoutMs: 30, recallTimeoutMs: 5_000 }).recall({ query: "q" }), + (err) => { + assert.ok(isRequestTimeout(err)); + assert.equal(err.phase, "preflight GET /version"); + assert.equal(err.timeoutMs, 30); + return true; + }, + ); +}); + +test("health() is bounded by the preflight budget", async () => { + globalThis.fetch = async (url, init = {}) => { + assert.equal(new URL(url).pathname, "/health"); + return delayed(1e9, () => null, init.signal); + }; + + await assert.rejects(client({ preflightTimeoutMs: 30 }).health(), (err) => { + assert.ok(isRequestTimeout(err)); + assert.equal(err.phase, "GET /health"); + return true; + }); +}); + +test("the deadline covers the response body, not just the headers", async () => { + // fetch() resolves at the headers. A server that answers promptly and then + // trickles the body must still blow the budget the caller named, and blow + // it as a phase-tagged TimeoutError rather than a bare AbortError. + globalThis.fetch = async (_url, init = {}) => + new Response( + new ReadableStream({ + start(controller) { + init.signal?.addEventListener( + "abort", + () => controller.error(abortError()), + { once: true }, + ); + }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); + + const res = await fetchWithDeadline( + "https://relayer.test/api/recall", + { method: "POST" }, + 25, + "POST /api/recall", + ); + // `utils.fetchWithDeadline` is the standalone helper (used by manual.ts); + // it raises utils' own phase-tagged `TimeoutError`, not the client's + // `MemWalRequestTimeout`. Both carry `phase`/`timeoutMs`. + await assert.rejects(res.json(), (err) => { + assert.ok(isTimeoutError(err)); + assert.equal(err.phase, "POST /api/recall"); + assert.equal(err.timeoutMs, 25); + return true; + }); +}); + +test("the timer survives the headers and is released by the body read", async () => { + // Both halves of the fix in one assertion pair: the deadline must still be + // armed when fetch() resolves (otherwise a trickled body is unbounded), and + // it must be cleared once the body settles (otherwise every request leaks a + // pending timer). Asserting only that json() returns would pass either way. + const live = new Set(); + const realSetTimeout = globalThis.setTimeout; + const realClearTimeout = globalThis.clearTimeout; + globalThis.setTimeout = (...args) => { + const id = realSetTimeout(...args); + live.add(id); + return id; + }; + globalThis.clearTimeout = (id) => { + live.delete(id); + return realClearTimeout(id); + }; + + try { + globalThis.fetch = async () => new Response(JSON.stringify({ ok: true }), { status: 200 }); + + const res = await fetchWithDeadline( + "https://relayer.test/version", + { method: "GET" }, + 5_000, + "preflight GET /version", + ); + assert.equal(live.size, 1, "the deadline must outlive the response headers"); + + assert.deepEqual(await res.json(), { ok: true }); + assert.equal(live.size, 0, "reading the body must release the deadline"); + } finally { + globalThis.setTimeout = realSetTimeout; + globalThis.clearTimeout = realClearTimeout; + } +}); + +test("the default recall budget is unchanged at 15s", async () => { + assert.equal(DEFAULT_RECALL_TIMEOUT_MS, 15_000); +}); + +test("recall timeoutMs 0 does not send deadline_ms", async () => { + let body = {}; + globalThis.fetch = async (url, init = {}) => { + const path = new URL(url).pathname; + if (path === "/version") return Response.json(VERSION_BODY); + if (path === "/api/recall") { + body = JSON.parse(init.body); + return Response.json({ results: [], total: 0 }); + } + throw new Error(path); + }; + const result = await client().recall({ query: "x", timeoutMs: 0 }); + assert.deepEqual(result, { results: [], total: 0 }); + assert.equal(body.deadline_ms, undefined); + const timeout = new Error("timed out"); + timeout.name = "MemWalRequestTimeout"; + assert.equal(isTimeoutError(timeout), true); +}); + +test("client recallTimeoutMs 0 does not send deadline_ms", async () => { + let body = {}; + globalThis.fetch = async (url, init = {}) => { + const path = new URL(url).pathname; + if (path === "/version") return Response.json(VERSION_BODY); + if (path === "/api/recall") { + body = JSON.parse(init.body); + return Response.json({ results: [], total: 0 }); + } + throw new Error(path); + }; + const result = await client({ recallTimeoutMs: 0 }).recall({ query: "x" }); + assert.deepEqual(result, { results: [], total: 0 }); + assert.equal(body.deadline_ms, undefined); +}); + +test("a short recall budget is sent whole instead of floored at 1", async () => { + let body = {}; + globalThis.fetch = async (url, init = {}) => { + const path = new URL(url).pathname; + if (path === "/version") return Response.json(VERSION_BODY); + if (path === "/api/recall") { + body = JSON.parse(init.body); + return Response.json({ results: [], total: 0 }); + } + throw new Error(path); + }; + await client().recall({ query: "x", timeoutMs: 400 }); + assert.equal(body.deadline_ms, 400); +}); + +test("RecallOptions and MemWalConfig declare the timeout knobs", () => { + // A .mjs test cannot catch a missing type; assert on the emitted .d.ts, + // which is what a TypeScript consumer actually resolves. + const dts = readFileSync( + fileURLToPath(new URL("../dist/types.d.ts", import.meta.url)), + "utf8", + ); + + assert.match(dts, /recallTimeoutMs\?: number;/); + assert.match(dts, /preflightTimeoutMs\?: number;/); +}); diff --git a/services/server/src/routes/admin.rs b/services/server/src/routes/admin.rs index b98b7bcb6..f1418dc0d 100644 --- a/services/server/src/routes/admin.rs +++ b/services/server/src/routes/admin.rs @@ -171,47 +171,95 @@ const DURABLE_WRITE_FAILURE_THRESHOLD: i64 = 3; /// `updated_at` index is the steady-state path; the 1s bound is a /// backstop, not the expected scan. const DURABLE_WRITE_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(30); +/// Cap on the Postgres write-readiness probe, unchanged from WALM-612. The +/// two probes run concurrently, so this is what bounds `/health`; a slower +/// query fails open at `warn` rather than reporting a false negative. const WRITE_READY_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1); + +/// Hard cap on the outbound sidecar probe. `/health` is a connectivity check, +/// so the handler body must stay bounded well under any client's request +/// budget (WALM-598 / GH #438). +const SIDECAR_PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(300); /// Neon refuses `smgrextend` once cluster size is at the cap; treat less /// than 1MB remaining as not writable so `/health` trips before the next /// page allocation fails. const POSTGRES_EXTEND_HEADROOM_BYTES: i64 = 1024 * 1024; -/// Sidecar liveness AND Postgres can accept writes. Cached together so -/// unsigned `/health` probes do not fan out on every load-balancer tick. -async fn write_ready(state: &std::sync::Arc) -> bool { - { - let cache = WRITE_READY_CACHE.lock().unwrap_or_else(|e| e.into_inner()); - if let Some((at, ready)) = *cache { - if at.elapsed() < WRITE_READY_CACHE_TTL { - return ready; - } +/// Memoized `write_ready` value plus the instant it was observed. +/// +/// `tokio::sync::RwLock`, not `std::sync::Mutex`, for two reasons: +/// +/// 1. The write guard now spans the probe (single-flight), and a +/// `std::sync::Mutex` must not be held across an `.await`. +/// 2. `std::sync::Mutex` poisons. The read side recovered with +/// `unwrap_or_else(|e| e.into_inner())` but the write side was a bare +/// `if let Ok(..)`, so a single panic while the lock was held froze the +/// cached timestamp forever: `elapsed()` stayed past the TTL and *every* +/// subsequent `/health` fanned out to the sidecar for the life of the +/// process. Tokio's locks have no poison flag, so an unwind through the +/// critical section just drops the guard. +type WriteReadyCache = tokio::sync::RwLock>; + +static WRITE_READY_CACHE: WriteReadyCache = tokio::sync::RwLock::const_new(None); + +/// Cache + single-flight wrapper around a write-readiness probe. +/// +/// The TTL used to be *checked* under the lock but not *held* across the +/// probe, so every request that arrived while a probe was in flight started +/// its own — a stampede at each TTL boundary, which CI's added `/health` +/// polling made routine. The write guard is now held for the probe, so +/// concurrent callers either read the fresh value or wait for the single +/// in-flight probe (bounded by the probe's own timeout). +/// +/// Generic over the probe so the cache semantics are testable without a live +/// `AppState` or sidecar. +async fn write_ready_cached( + cache: &WriteReadyCache, + ttl: std::time::Duration, + probe: F, +) -> bool +where + F: FnOnce() -> Fut, + Fut: std::future::Future, +{ + // Fast path: a fresh value is readable in parallel by every caller. + if let Some((at, ready)) = *cache.read().await { + if at.elapsed() < ttl { + return ready; } } - let (sidecar, postgres) = tokio::join!(sidecar_write_ready(state), postgres_write_ready(state)); - let ready = sidecar && postgres; - if let Ok(mut cache) = WRITE_READY_CACHE.lock() { - *cache = Some((std::time::Instant::now(), ready)); + let mut guard = cache.write().await; + // Re-check: a racing caller may have refreshed while we waited for the + // write lock. This is what collapses the stampede onto one probe. + if let Some((at, ready)) = *guard { + if at.elapsed() < ttl { + return ready; + } } + + let ready = probe().await; + *guard = Some((std::time::Instant::now(), ready)); ready } -async fn sidecar_write_ready(state: &std::sync::Arc) -> bool { - let url = format!("{}/health", state.config.sidecar_url.trim_end_matches('/')); - match state - .http_client - .get(&url) - .timeout(WRITE_READY_PROBE_TIMEOUT) - .send() - .await - { - Ok(resp) => resp.status().is_success(), - Err(err) => { - tracing::debug!(error = %err, "sidecar health probe failed"); - false - } - } +/// Sidecar liveness AND Postgres can accept writes. Cached together so +/// unsigned `/health` probes do not fan out on every load-balancer tick. +async fn write_ready(state: &std::sync::Arc) -> bool { + write_ready_cached(&WRITE_READY_CACHE, WRITE_READY_CACHE_TTL, || { + probe_write_ready(state) + }) + .await +} + +/// Both halves of write-readiness, run concurrently so `/health` costs one +/// probe window rather than two. +async fn probe_write_ready(state: &std::sync::Arc) -> bool { + let (sidecar, postgres) = tokio::join!( + probe_sidecar_write_ready(state), + postgres_write_ready(state) + ); + sidecar && postgres } /// Self-hosted Postgres without `neon.max_cluster_size` stays ready (sidecar @@ -329,8 +377,49 @@ fn postgres_can_accept_writes(used_bytes: i64, max_bytes: i64) -> bool { used_bytes.saturating_add(POSTGRES_EXTEND_HEADROOM_BYTES) < max_bytes } -static WRITE_READY_CACHE: std::sync::Mutex> = - std::sync::Mutex::new(None); +/// One outbound `GET {sidecar}/health`, instrumented like every other sidecar +/// call in this codebase (`storage/walrus.rs`, `storage/seal.rs`) so the +/// probe's latency and failure rate are measurable. +async fn probe_sidecar_write_ready(state: &std::sync::Arc) -> bool { + let url = format!("{}/health", state.config.sidecar_url.trim_end_matches('/')); + let started = std::time::Instant::now(); + match state + .http_client + .get(&url) + .timeout(SIDECAR_PROBE_TIMEOUT) + .send() + .await + { + Ok(resp) => { + let status = resp.status(); + crate::observability::observe_external( + "sidecar", + "health_probe", + status.as_u16().to_string().as_str(), + started.elapsed(), + ); + if !status.is_success() { + crate::observability::record_sidecar_failure("health_probe", "http_error"); + tracing::warn!( + status = %status, + "sidecar health probe returned non-success; reporting write_ready=false" + ); + } + status.is_success() + } + Err(err) => { + crate::observability::observe_external( + "sidecar", + "health_probe", + "transport_error", + started.elapsed(), + ); + crate::observability::record_sidecar_failure("health_probe", "transport_error"); + tracing::warn!(error = %err, "sidecar health probe failed"); + false + } + } +} static DURABLE_WRITE_CACHE: std::sync::Mutex> = std::sync::Mutex::new(None); @@ -1761,4 +1850,115 @@ mod tests { "non-empty namespace must pass the validation predicate" ); } + + // ── /health write-readiness cache (WALM-598) ───────────────── + + mod write_ready_cache { + use super::super::{write_ready_cached, WriteReadyCache}; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + use std::time::Duration; + + const TTL: Duration = Duration::from_secs(2); + + #[tokio::test] + async fn fresh_value_is_served_without_reprobing() { + let cache: WriteReadyCache = tokio::sync::RwLock::new(None); + let probes = AtomicUsize::new(0); + let probe = || async { + probes.fetch_add(1, Ordering::SeqCst); + true + }; + + assert!(write_ready_cached(&cache, TTL, probe).await); + assert!(write_ready_cached(&cache, TTL, probe).await); + assert_eq!( + probes.load(Ordering::SeqCst), + 1, + "second call must hit cache" + ); + } + + #[tokio::test] + async fn expired_value_reprobes() { + let cache: WriteReadyCache = tokio::sync::RwLock::new(None); + let probes = AtomicUsize::new(0); + let probe = || async { + probes.fetch_add(1, Ordering::SeqCst); + true + }; + + // A zero TTL makes every stored value instantly stale. + assert!(write_ready_cached(&cache, Duration::ZERO, probe).await); + assert!(write_ready_cached(&cache, Duration::ZERO, probe).await); + assert_eq!(probes.load(Ordering::SeqCst), 2); + } + + /// The stampede this replaced: the old code checked the TTL under the + /// lock but released it before probing, so N concurrent `/health` + /// requests arriving at a TTL boundary each fired their own outbound + /// probe. Holding the write guard across the probe collapses them. + #[tokio::test] + async fn concurrent_callers_share_one_probe() { + let cache: Arc = Arc::new(tokio::sync::RwLock::new(None)); + let probes = Arc::new(AtomicUsize::new(0)); + + let mut handles = Vec::new(); + for _ in 0..16 { + let cache = Arc::clone(&cache); + let probes = Arc::clone(&probes); + handles.push(tokio::spawn(async move { + write_ready_cached(&cache, TTL, || async move { + probes.fetch_add(1, Ordering::SeqCst); + // Yield across an await point so the other tasks are + // genuinely in flight while this probe runs. + tokio::time::sleep(Duration::from_millis(20)).await; + true + }) + .await + })); + } + + for handle in handles { + assert!(handle.await.unwrap()); + } + assert_eq!( + probes.load(Ordering::SeqCst), + 1, + "concurrent /health requests must not stampede the sidecar" + ); + } + + /// Regression for the poisoned-mutex bug: with `std::sync::Mutex` a + /// panic inside the critical section poisoned the lock, the bare + /// `if let Ok(mut cache)` write silently stopped refreshing, and the + /// frozen timestamp made every later request bypass the cache + /// forever. Tokio locks do not poison, so the cache must keep working. + #[tokio::test] + async fn panicking_probe_does_not_disable_the_cache() { + static PANIC_CACHE: WriteReadyCache = tokio::sync::RwLock::const_new(None); + + let panicked = tokio::spawn(async { + write_ready_cached(&PANIC_CACHE, TTL, || async { + panic!("probe blew up while holding the cache lock"); + }) + .await + }) + .await; + assert!(panicked.is_err(), "probe was expected to panic"); + + let probes = AtomicUsize::new(0); + let probe = || async { + probes.fetch_add(1, Ordering::SeqCst); + true + }; + assert!(write_ready_cached(&PANIC_CACHE, TTL, probe).await); + assert!(write_ready_cached(&PANIC_CACHE, TTL, probe).await); + assert_eq!( + probes.load(Ordering::SeqCst), + 1, + "cache must still memoize after a panicking probe" + ); + } + } } From 8c294914942a3ad2a75a821b39c5cebf8ce255b5 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Wed, 14 Oct 2026 20:47:15 +0700 Subject: [PATCH 44/70] fix(mcp): mint a fresh delegate key on forced login (WALM-645) (#946) * fix(mcp): mint a fresh delegate key on forced login (WALM-645) `loginFlow` adopted a still-valid pending record unconditionally, so a stranded login could never be abandoned. With `/api/whoami` absent from the default relayer, recovery lands in `unavailable` on every start and keeps the record for its full 24h TTL, while the browser's `add_delegate_key` aborts with `EDelegateKeyAlreadyExists`. `login` skips recovery but still reused the record, so it was not an escape hatch either. `login` now passes `freshKey`, which mints a new keypair and write-aheads it over the stuck one. The `unavailable` notice no longer says no action is needed. * fix(mcp): name the key a fresh login discards, and reach it from the tool (WALM-645) `freshKey` dropped the pending record and `savePendingLogin` overwrote it unconditionally, with no equivalent of `backupIfReplacingAnotherAccount`. That is the loss the write-ahead record exists to prevent: the discarded record was the only copy of a key the browser may already have paid to register, and recovery is skipped on this path, so nothing ever named it. `loginFlow` now reports the discarded record's public key and delegate address before the overwrite, so the registration can be found in the dashboard. `freshKey` was also wired only into the CLI. The `memwal_login` tool path goes through `startOrReuseLoginFlow` from `auth-required` and `bridge`, neither of which passed it, so a user inside an MCP host stayed stuck with no escape but a real terminal. Both now take it from an opt-in `freshKey` tool argument, defaulting to false so an ordinary login still reuses the pending key. The `unavailable` notice gave key collision as the reason to remove the key, which cannot happen to a freshly minted one, and pointed at `memwal-mcp login` rather than the documented npx form. `LOGIN_RETRY_INSTRUCTION` still offered the retry only to users who had not approved the wallet step, and the bridge's failure notice still promised the next start reclaims the key, which it cannot where the relayer has no `/api/whoami`. * fix(mcp): reset inflight login on freshKey so forced retry mints a new key * fix(mcp): close the login listener that freshKey replaces freshKey minted a new key but left the previous listener bound. Its timeout still reported a failure, and a late callback could save credentials or clear the write-ahead record the new login just wrote. Aborting that listener drops both. The tool result names the abandoned public key and address. WALM-645 * fix(mcp): drop a freshKey login aborted while it is still minting A second freshKey aborts the controller before the first flow subscribes. That flow now stops before it writes the pending key or publishes a URL, so the caller does not keep a link to a listener that is already closed. * fix(mcp): do not send a replaced login to the CLI fallback A superseded sign-in tells the caller to open the newer memwal_login result instead of running a second login that would mint over it. --------- Co-authored-by: Harry Phan Co-authored-by: ducnmm <165614309+ducnmm@users.noreply.github.com> --- docs/mcp/changelog.mdx | 6 + packages/mcp/CHANGELOG.md | 4 + packages/mcp/src/auth-required.ts | 45 ++++-- packages/mcp/src/bridge.ts | 38 ++++- packages/mcp/src/index.ts | 2 +- packages/mcp/src/login.ts | 131 ++++++++++++++++-- packages/mcp/src/messages.ts | 24 +++- packages/mcp/src/recovery.ts | 9 +- .../mcp/test/login-failure-notice.test.mjs | 24 ++-- .../mcp/test/login-prompt-unified.test.mjs | 14 ++ packages/mcp/test/login-singleflight.test.mjs | 93 +++++++++++++ packages/mcp/test/login-write-ahead.test.mjs | 90 ++++++++++++ 12 files changed, 428 insertions(+), 52 deletions(-) diff --git a/docs/mcp/changelog.mdx b/docs/mcp/changelog.mdx index 680a2f8f7..d0724282b 100644 --- a/docs/mcp/changelog.mdx +++ b/docs/mcp/changelog.mdx @@ -31,6 +31,12 @@ answer: >- The latest published MCP package on npm `latest` is 0.0.13. This branch's in-tree plugin launcher installs `@mysten-incubation/memwal-mcp@0.0.14-dev.6` because npm has no `0.0.14` yet — pinning the unpublished version would fail to resolve. 0.0.14 (unreleased) stops cold start from advertising tools an older relayer does not serve, bounds stalled relayer calls, honours `retry_after` on 429, and returns `memwal_remember` at accept by default (`MEMWAL_MCP_REMEMBER_WAIT_MS=0`; set `90000` to wait for `blob_id`). 0.0.13 stopped reporting a timed-out write as safe to retry, answers a rejected delegate key with `memwal_login` instead of a dropped-connection story, backs off on handshake 429, and pinned plugin launchers to `@0.0.13` so npx cannot keep a cached 0.0.5. --- +## Unreleased + +### Fixed + +- `memwal_login` with `freshKey: true` closes the listener already bound. The replaced listener cannot time out into a failure notice, and a late callback cannot save credentials or clear the new write-ahead record. The tool result names the abandoned public key and address so it can be removed from the dashboard. (WALM-645) + ## 0.0.14 Unreleased package version. The plugin's launcher installs `@mysten-incubation/memwal-mcp@0.0.14-dev.6` (the published `dev` dist-tag), named by `mcpPackageVersion` in `plugin/plugin.json`, until `0.0.14` exists on npm. Drop that key in the same release that publishes it, so the launcher goes back to the plugin's own version, or plugin installs stay on the dev build. diff --git a/packages/mcp/CHANGELOG.md b/packages/mcp/CHANGELOG.md index 842f4a3c5..70c2b8990 100644 --- a/packages/mcp/CHANGELOG.md +++ b/packages/mcp/CHANGELOG.md @@ -9,6 +9,10 @@ - `MEMWAL_CREDS_DIR` must now be an absolute path outside the current project, and a value that is not is refused with an error naming it rather than followed. The override decides the credentials before any approval is looked up, so it was a way around the gate above rather than through it: an empty value made the approval store the bare relative name `project-approvals.json`, which resolves against the working directory — a repository that committed that one file at its root approved its own credentials — and a relative value such as `.memwal` put both the credentials and the approval record inside the repository. Neither had to be typed by the user: an MCP client reads `.cursor/mcp.json`, `.vscode/mcp.json` or `.claude/settings.json` out of the checkout and passes on the `env` block it finds, and editors expand `${workspaceFolder}` there, so an absolute path inside the project is refused too. An empty value now means unset, matching every other reader of the variable. Tests and CI pointing at an absolute temp directory are unaffected. (WALM-639) - The login write-ahead record no longer goes into the repository. It sat beside whichever credentials file resolution chose, so once a project file was approved every sign-in wrote a plaintext 64-hex Ed25519 delegate seed to `/.memwal/login-pending.json` — in a directory an attacker who planted the credentials file had already created, so tracked rather than ignored, and staged by `git add -A`. It now lives in `~/.memwal/login-pending/`, keyed by project so a sign-in is still reclaimable only by the project that started it. The credentials file itself cannot move — it is the file the user approved — so `approve-project` and the pre-sign-in warning now state plainly that signing in from this project writes a delegate private key inside the repository, and suggest adding `.memwal/` to `.gitignore`. (WALM-639) +### Fixed + +- `memwal_login` with `freshKey: true` closes the listener already bound. The replaced listener cannot time out into a failure notice, and a late callback cannot save credentials or clear the new write-ahead record. The tool result names the abandoned public key and address so it can be removed from the dashboard. (WALM-645) + ## 0.0.14 Unreleased. The plugin's launcher installs `@mysten-incubation/memwal-mcp@0.0.14-dev.6`, named by `mcpPackageVersion` in `plugin/plugin.json`, until `0.0.14` is on npm; drop that key in that release. diff --git a/packages/mcp/src/auth-required.ts b/packages/mcp/src/auth-required.ts index fe2f5335e..17f780762 100644 --- a/packages/mcp/src/auth-required.ts +++ b/packages/mcp/src/auth-required.ts @@ -25,7 +25,7 @@ import { credsPath, loadCreds, type MemWalCredentials } from "./auth.js"; import { rememberInitializeClientInfo } from "./client-info.js"; import { loginFailureNotice, loginPrompt, loginSuccessNotification } from "./messages.js"; import { log } from "./logger.js"; -import { startOrReuseLoginFlow, resolveLoginTimeoutMs } from "./login.js"; +import { startOrReuseLoginFlow, resolveLoginTimeoutMs, isLoginSuperseded } from "./login.js"; import { AUTH_REQUIRED_INSTRUCTIONS } from "./instructions.js"; import { SECRET_EXCLUSION_RULES, @@ -210,7 +210,14 @@ function buildToolDefinitions(proactive: boolean) { "Sign this MCP client into your Walrus Memory account by opening a browser. Run once when the agent reports Walrus Memory is not signed in. Opens the dashboard in the default browser, waits for wallet approval, then writes credentials to ~/.memwal/credentials.json. Other memwal_* tools become usable on the next call after a successful login.", inputSchema: { type: "object", - properties: {}, + properties: { + freshKey: { + type: "boolean", + default: false, + description: + "Mint a new delegate key instead of reusing the one a previous unfinished sign-in left pending. Leave false for an ordinary login. Set it only after a normal sign-in has already failed on the pending key — it abandons that key, which must then be removed from the dashboard if it was approved on-chain, and the new one needs its own add_delegate_key transaction.", + }, + }, additionalProperties: false, }, }, @@ -291,13 +298,17 @@ const LOGIN_INSTRUCTION = [ /** Replaces {@link LOGIN_INSTRUCTION} after a failed attempt, which * {@link loginFailureNotice} has just described. The generic copy promises "no * client restart" and leads with `memwal_login`; for a key the user already - * approved, a restart is the only thing that recovers it and signing in again - * cannot register it twice. So the retry is offered only for the case it - * actually fixes. */ + * approved, a restart is the only thing that recovers it and a plain retry + * reuses it rather than registering a second one. So the retry names + * `freshKey` for the case the plain one cannot fix. */ const LOGIN_RETRY_INSTRUCTION = [ - "If you did not approve the wallet step, start a new sign-in: call the `memwal_login`", - "tool from this client, or run `npx -y @mysten-incubation/memwal-mcp login`. Open the", - "new link straight away and leave this client running through the wallet prompt.", + "Start a new sign-in: call the `memwal_login` tool from this client, or run", + "`npx -y @mysten-incubation/memwal-mcp login`. Open the new link straight away and", + "leave this client running through the wallet prompt.", + "", + "If that keeps failing on the same key, call `memwal_login` with `freshKey: true` to", + "mint a new one. That abandons the pending key: if you already approved it on-chain,", + "remove it from the dashboard afterwards.", ].join("\n"); /** Set when a background `memwal_login` ends without credentials. The tool call @@ -361,6 +372,7 @@ function sendLogMessage(level: "info" | "warning" | "error", text: string): void async function handleLoginToolCall( config: AuthRequiredConfig, _progressToken: unknown, + freshKey: boolean, ): Promise<{ text: string; isError: boolean }> { // Fire login but DO NOT await the wallet callback — it runs in the // background. openBrowser: false because (a) child-process spawning a @@ -372,6 +384,7 @@ async function handleLoginToolCall( // Concurrent memwal_login calls join this in-flight flow instead of // opening a second listener (that race hung later recall/remember). lastLoginFailure = null; + let discardedKey: { publicKey: string; address: string } | undefined; const session = startOrReuseLoginFlow( { relayerUrl: config.relayerUrl, @@ -379,6 +392,10 @@ async function handleLoginToolCall( label: config.label, timeoutMs: resolveLoginTimeoutMs(), openBrowser: false, + freshKey, + onDiscarded: (key) => { + discardedKey = key; + }, onUrl: (url) => { sendLogMessage("info", `Walrus Memory MCP login URL: ${url}`); }, @@ -424,6 +441,9 @@ async function handleLoginToolCall( try { url = await Promise.race([session.url, timeoutPromise]); } catch (err) { + if (isLoginSuperseded(err)) { + return { isError: true, text: "This sign-in was replaced by a newer memwal_login. Open that result instead of starting another login." }; + } const msg = err instanceof Error ? err.message : String(err); log.error("memwal_login.tool.url_not_ready", { msg }); return { @@ -449,6 +469,7 @@ async function handleLoginToolCall( url, credentialsPath: credsPath(), signedIn: loadCreds() !== null, + discardedKey, }), }; } @@ -535,7 +556,7 @@ function handleAuthLine( if (method === "tools/call") { const params = (req.params ?? {}) as { name?: string; - arguments?: unknown; + arguments?: { freshKey?: unknown }; _meta?: { progressToken?: unknown }; }; const toolName = params.name; @@ -545,7 +566,11 @@ function handleAuthLine( // Returns near-instantly with the click-able URL. The listener // stays alive in the background — see handleLoginToolCall for the // rationale on not blocking. - void handleLoginToolCall(config, progressToken).then((result) => { + void handleLoginToolCall( + config, + progressToken, + params.arguments?.freshKey === true, + ).then((result) => { writeStdoutMessage({ jsonrpc: "2.0", id, diff --git a/packages/mcp/src/bridge.ts b/packages/mcp/src/bridge.ts index c3883d38c..69cb6e2db 100644 --- a/packages/mcp/src/bridge.ts +++ b/packages/mcp/src/bridge.ts @@ -31,7 +31,7 @@ import { type HealthProbe, } from "./health-probe.js"; import { proactiveInstructions } from "./instructions.js"; -import { startOrReuseLoginFlow, resolveLoginTimeoutMs } from "./login.js"; +import { startOrReuseLoginFlow, resolveLoginTimeoutMs, isLoginSuperseded } from "./login.js"; import { log, note } from "./logger.js"; import { loginPrompt, @@ -154,7 +154,14 @@ const LOCAL_TOOL_DEFINITIONS = [ "Sign in (or re-sign in) to Walrus Memory by opening a browser. Use to switch wallets, refresh credentials, or sign in for the first time. Returns a click-able URL — the user must approve in their browser.", inputSchema: { type: "object", - properties: {}, + properties: { + freshKey: { + type: "boolean", + default: false, + description: + "Mint a new delegate key instead of reusing the one a previous unfinished sign-in left pending. Leave false for an ordinary login. Set it only after a normal sign-in has already failed on the pending key — it abandons that key, which must then be removed from the dashboard if it was approved on-chain, and the new one needs its own add_delegate_key transaction.", + }, + }, additionalProperties: false, }, }, @@ -919,7 +926,9 @@ function writeStdoutMessage(msg: RpcMessage): void { async function handleLocalLogin( config: BridgeConfig, onCredentials: (creds: MemWalCredentials) => Promise, + freshKey: boolean, ): Promise<{ text: string; isError: boolean }> { + let discardedKey: { publicKey: string; address: string } | undefined; const session = startOrReuseLoginFlow( { relayerUrl: config.relayerUrl, @@ -927,6 +936,10 @@ async function handleLocalLogin( label: config.label, timeoutMs: resolveLoginTimeoutMs(), openBrowser: false, + freshKey, + onDiscarded: (key) => { + discardedKey = key; + }, }, async (creds) => { await onCredentials(creds); @@ -972,8 +985,10 @@ async function handleLocalLogin( // dashboard cannot register it twice. data: `Walrus Memory sign-in did not complete: ${msg}. Existing credentials are ` + - `unchanged. If you approved the wallet step, the next start reclaims that ` + - `key; otherwise call memwal_login again to retry.`, + `unchanged. The key is kept on disk and a later start reclaims it only if ` + + `the relayer can confirm it was registered, which it cannot always do. ` + + `Call memwal_login again to retry with that key, or with freshKey: true to ` + + `mint a new one and abandon it.`, }, }); }, @@ -990,6 +1005,9 @@ async function handleLocalLogin( try { url = await Promise.race([session.url, timeoutPromise]); } catch (err) { + if (isLoginSuperseded(err)) { + return { isError: true, text: "This sign-in was replaced by a newer memwal_login. Open that result instead of starting another login." }; + } return { isError: true, text: `❌ Failed to start login: ${err instanceof Error ? err.message : String(err)}`, @@ -1006,6 +1024,7 @@ async function handleLocalLogin( url, credentialsPath: credsPath(), signedIn: loadCreds() !== null, + discardedKey, }), }; } @@ -2061,7 +2080,10 @@ export async function runBridge( // can call them any time to re-auth or sign out without // having to remove + re-add the MCP server. if (msg.method === "tools/call" && msg.id != null) { - const params = (msg.params ?? {}) as { name?: string }; + const params = (msg.params ?? {}) as { + name?: string; + arguments?: { freshKey?: unknown }; + }; // Tool NAME only, never `arguments` — memory text is the // user's private data and must not reach a log file. // Without this the only trace of a call is the host's own @@ -2074,7 +2096,11 @@ export async function runBridge( id: msg.id, }); if (params.name === "memwal_login") { - const result = await handleLocalLogin(config, adoptCredentials); + const result = await handleLocalLogin( + config, + adoptCredentials, + params.arguments?.freshKey === true, + ); writeStdoutMessage({ jsonrpc: "2.0", id: msg.id, diff --git a/packages/mcp/src/index.ts b/packages/mcp/src/index.ts index 90f80dd5c..686345031 100644 --- a/packages/mcp/src/index.ts +++ b/packages/mcp/src/index.ts @@ -524,7 +524,7 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise void; + freshKey?: boolean; + /** When set, `startOrReuseLoginFlow` aborts this listener so a newer + * sign-in can mint a key without the old timeout or callback writing + * over it. */ + signal?: AbortSignal; + /** Fired when `freshKey` drops a pending record. The public key is no + * longer in the write-ahead file after the replacement is saved. */ + onDiscarded?: (key: { publicKey: string; address: string }) => void; } -const DEFAULTS: Required> & { label: string } = { +function loginSupersededError(): Error { + const err = new Error("Sign-in was replaced by a newer login. This listener is closed."); + err.name = "MemWalLoginSuperseded"; + return err; +} + +export function isLoginSuperseded(err: unknown): boolean { + return err instanceof Error && err.name === "MemWalLoginSuperseded"; +} + +const DEFAULTS: Required> & { label: string } = { webUrl: process.env.MEMWAL_WEB_URL ?? "https://memory.walrus.xyz", relayerUrl: process.env.MEMWAL_SERVER_URL ?? "https://relayer.memory.walrus.xyz", label: process.env.MEMWAL_CLIENT_LABEL ?? "Walrus Memory MCP", @@ -217,18 +235,48 @@ export async function loginFlow(opts: LoginOptions = {}): Promise cfg.signal?.aborted === true; + if (superseded()) throw loginSupersededError(); // Adopt a still-valid record for this relayer rather than minting over it. // Recovery only runs at process start and is skipped for `--login`, so a // timed-out login followed by `memwal_login` in the same process would // otherwise replace the only copy of a key the browser may already have // paid to register. - const reusable = reusablePendingLogin(cfg.relayerUrl); + const onDisk = reusablePendingLogin(cfg.relayerUrl); + const discarded = opts.freshKey ? onDisk : null; + const reusable = discarded ? null : onDisk; if (reusable) { log.info("login.pending.reused", { publicKey: reusable.delegatePublicKeyHex, createdAt: reusable.createdAt, }); } + if (discarded) { + log.warn("login.pending.discarded", { + publicKey: discarded.delegatePublicKeyHex, + address: discarded.delegateAddress, + createdAt: discarded.createdAt, + }); + note( + `Minting a fresh delegate key and abandoning the one left by the last ` + + `unfinished sign-in:\n` + + ` public key: ${discarded.delegatePublicKeyHex}\n` + + ` address: ${discarded.delegateAddress}\n` + + `If you approved that key in your wallet, nothing will use it again — ` + + `remove it from the dashboard.`, + ); + try { + opts.onDiscarded?.({ + publicKey: discarded.delegatePublicKeyHex, + address: discarded.delegateAddress, + }); + } catch { + /* caller errors don't break the flow */ + } + } const keypair = reusable ? { privateKeyHex: reusable.delegatePrivateKey, @@ -236,6 +284,7 @@ export async function loginFlow(opts: LoginOptions = {}): Promise | undefined; + // One winner. The timeout, an abort from a newer `freshKey` login, and the + // callback all try to finish this listener. Claiming first closes the + // others out so a late callback cannot saveCreds over the replacement key. + const claim = (): boolean => { + if (retired) return false; + retired = true; + if (timer) clearTimeout(timer); + return true; + }; + const done = new Promise((resolve) => { - const timer = setTimeout(() => { + const retire = (err: Error) => { + if (!claim()) return; + error = err; + server.close(); + resolve(); + }; + timer = setTimeout(() => { if (!creds) { - error = new Error( - `Login timed out after ${cfg.timeoutMs}ms. If you already approved the wallet transaction, that delegate key is saved on disk: the next start reclaims it, and running login again cannot register it a second time. Only remove it from the dashboard if you mean to abandon it.`, - ); - server.close(); - resolve(); + retire(new Error( + `Login timed out after ${cfg.timeoutMs}ms. A restart retries the same key. ` + + `If the wallet keeps rejecting it, call memwal_login with freshKey: true. ` + + `That mints a new key and closes the sign-in already open; remove the old ` + + `one from the dashboard if you already approved it on-chain.`, + )); } - }, cfg.timeoutMs).unref?.(); - void timer; + }, cfg.timeoutMs); + timer.unref?.(); + + if (cfg.signal) { + const onAbort = () => retire(loginSupersededError()); + if (cfg.signal.aborted) onAbort(); + else cfg.signal.addEventListener("abort", onAbort, { once: true }); + } server.on("request", async (req: IncomingMessage, res: ServerResponse) => { + if (retired) { + res.writeHead(409, { "content-type": "text/html" }); + res.end(FAIL_HTML_TEMPLATE("This sign-in was replaced by a newer login.")); + return; + } // CORS — only the dashboard origin we control may POST. `*` would // let any web page on the internet talk to this localhost port. res.setHeader("access-control-allow-origin", expectedOrigin); @@ -462,6 +545,14 @@ export async function loginFlow(opts: LoginOptions = {}): Promise void, ): InflightLogin { - if (inflightLogin) return inflightLogin; + if (opts.freshKey) { + resetInflightLogin(); + } else if (inflightLogin) { + return inflightLogin; + } + + const abort = new AbortController(); + inflightAbort = abort; let resolveUrl!: (url: string) => void; let rejectUrl!: (err: unknown) => void; @@ -556,6 +655,7 @@ export function startOrReuseLoginFlow( const userOnUrl = opts.onUrl; const result = loginFlow({ ...opts, + signal: abort.signal, onUrl: (connectUrl) => { resolveUrl(connectUrl); try { @@ -572,6 +672,9 @@ export function startOrReuseLoginFlow( // failure twice for one flow. result.catch((err) => { rejectUrl(err); + // The replacement login is the one the caller was just handed. The + // old timeout and its onFailure must not talk over that. + if (isLoginSuperseded(err)) return; const msg = err instanceof Error ? err.message : String(err); // This used to be a `warn` and nothing else, which made a background // login failure invisible: `handleLocalLogin` has already returned the @@ -620,7 +723,11 @@ export function startOrReuseLoginFlow( return session; } -/** Drop the in-flight handle. Does not abort a listener already bound. */ +/** Drop the in-flight handle and close its listener. A late callback or + * timeout on that listener must not save credentials or clear the pending + * record a newer login just wrote. */ export function resetInflightLogin(): void { + inflightAbort?.abort(); + inflightAbort = null; inflightLogin = null; } diff --git a/packages/mcp/src/messages.ts b/packages/mcp/src/messages.ts index c9143a402..ecf09c85b 100644 --- a/packages/mcp/src/messages.ts +++ b/packages/mcp/src/messages.ts @@ -25,6 +25,9 @@ export interface LoginPromptInfo { credentialsPath: string; /** True when credentials already exist, so signing in REPLACES them. */ signedIn: boolean; + /** Set when `freshKey` dropped a pending delegate key. The write-ahead + * file no longer has it, so the prompt is the place that names it. */ + discardedKey?: { publicKey: string; address: string }; } /** @@ -61,6 +64,15 @@ export function loginPrompt(info: LoginPromptInfo): string { `**Note:** you are already signed in. Completing this replaces the stored delegate key at \`${info.credentialsPath}\` with the new wallet's.`, ] : []), + ...(info.discardedKey + ? [ + ``, + `**Abandoned key:** this sign-in replaced an unfinished delegate key.`, + `Public key: \`${info.discardedKey.publicKey}\``, + `Address: \`${info.discardedKey.address}\``, + `If you already approved that key on-chain, remove it from the dashboard.`, + ] + : []), ``, `_The login link stays valid for 5 minutes. If it expires, call \`memwal_login\` again to get a fresh URL._`, ].join("\n"); @@ -119,9 +131,10 @@ export function loginSuccessNotification(info: LoginSuccessInfo): string { * can prefix unconditionally. * * The advice splits on whether the user approved the wallet step. If they did, - * the key is on-chain and in the write-ahead record (WALM-332), so a restart - * reclaims it. Signing in again would reuse that same key, and the dashboard's - * `add_delegate_key` aborts on a key that is already registered. + * the key is on-chain and in the write-ahead record (WALM-332). A restart + * retries that same key. `freshKey: true` mints a new one and closes the + * sign-in that is already open. A key already approved on-chain has to be + * removed from the dashboard; nothing else will use it. */ export function loginFailureNotice(reason: string | null): string { if (!reason) return ""; @@ -131,8 +144,9 @@ export function loginFailureNotice(reason: string | null): string { `Reason: ${reason}`, "", "If you approved the wallet step, that key is registered and saved on this machine.", - "Restart the MCP client within 24 hours and it is reclaimed. Signing in again cannot", - "register the same key twice, and removing it from the dashboard abandons it.", + "A restart retries the same key. Call", + "`memwal_login` with `freshKey: true` to mint a new one, and remove it from the", + "dashboard if you already approved the old key on-chain.", "", "If you did not approve it, sign in again and open the new link straight away. A", "retry only helps once the MCP client is left running through the wallet prompt.", diff --git a/packages/mcp/src/recovery.ts b/packages/mcp/src/recovery.ts index f39deb8a8..8345ef067 100644 --- a/packages/mcp/src/recovery.ts +++ b/packages/mcp/src/recovery.ts @@ -204,8 +204,7 @@ export async function recoverPendingLogin(): Promise { // again, after removing the key from the dashboard if it was already // registered — is worth giving. During a transient upstream // failure that advice is worse than silence: the key is still good, and - // `unavailable` correctly says the next start retries it with no action - // from the user. + // `unavailable` correctly says the next start retries it. // // So only 401/403 is a denial. A 503 carrying // `x-auth-error: AUTH_UPSTREAM_UNAVAILABLE` is Sui RPC being down, and @@ -284,7 +283,11 @@ export function formatStrandedLoginNotice(result: RecoveryResult): string | null } else if (result.outcome === "unavailable") { lines.push( `The relayer could not be reached to check. This will be retried on the`, - `next start — no action needed yet.`, + `next start. If sign-in keeps failing, run`, + `\`npx -y @mysten-incubation/memwal-mcp login\`, or call \`memwal_login\` with`, + `\`freshKey: true\`: both mint a new key instead of waiting on this one.`, + `That abandons the key above, so if you approved it on-chain, remove it`, + `from the dashboard — nothing will use it again.`, ); } else { lines.push( diff --git a/packages/mcp/test/login-failure-notice.test.mjs b/packages/mcp/test/login-failure-notice.test.mjs index 07d1abcde..3f7c436a4 100644 --- a/packages/mcp/test/login-failure-notice.test.mjs +++ b/packages/mcp/test/login-failure-notice.test.mjs @@ -211,27 +211,21 @@ test("a sign-in that never completes is reported on the next tool call", async ( assertKeepsTheStrandedKey(text); // Still tells them how to sign in, rather than replacing the instruction. assert.match(text, /memwal_login/); - // ...but not by promising the opposite of the notice above it: an approved - // key is reclaimed by a restart, so the blob cannot also sell "no restart". assert.doesNotMatch(text, /no client restart/i); }); /** - * A failed attempt leaves its key in the write-ahead record (WALM-332), so the - * notice must send an approved key to a restart, which reclaims it. Signing in - * again reuses that key and the dashboard cannot register it twice, and - * removing it from the dashboard throws away the registration the user paid - * for. Both the wrapper and the timeout reason inside it are checked. + * A failed attempt leaves its key in the write-ahead record (WALM-332). A + * restart retries that same key. `freshKey: true` mints another one, and a + * key the user already approved has to be removed from the dashboard. */ function assertKeepsTheStrandedKey(text) { - assert.match(text, /Restart the MCP client/); - assert.match(text, /reclaim/); - assert.doesNotMatch(text, /revoke/i); - for (const sentence of text.split(/(?<=\.)\s+/)) { - if (/dashboard/.test(sentence)) { - assert.match(sentence, /abandon/, `dashboard advice must be limited to abandoning: "${sentence}"`); - } - } + assert.match(text, /freshKey: true/); + assert.match(text, /retries the same key/); + assert.match(text, /remove it/); + assert.match(text, /dashboard/); + assert.doesNotMatch(text, /cannot register/i); + assert.doesNotMatch(text, /no client restart/i); } test("the failure notice keeps a stranded key reclaimable", async () => { diff --git a/packages/mcp/test/login-prompt-unified.test.mjs b/packages/mcp/test/login-prompt-unified.test.mjs index 8b363a229..091ec4261 100644 --- a/packages/mcp/test/login-prompt-unified.test.mjs +++ b/packages/mcp/test/login-prompt-unified.test.mjs @@ -66,6 +66,20 @@ test("only the signed-in prompt warns that the stored key is replaced", () => { ); }); +test("a discarded key is named so it can be removed from the dashboard", () => { + const publicKey = "ab".repeat(32); + const address = "0x" + "cd".repeat(32); + const text = loginPrompt({ + url: URL_, + credentialsPath: CREDS, + signedIn: false, + discardedKey: { publicKey, address }, + }); + assert.ok(text.includes(publicKey)); + assert.ok(text.includes(address)); + assert.match(text, /remove it from the dashboard/); +}); + test("the two prompts differ ONLY in that warning", () => { // Drop the warning, then collapse the blank line that separated it — the // claim under test is that no other CONTENT differs. diff --git a/packages/mcp/test/login-singleflight.test.mjs b/packages/mcp/test/login-singleflight.test.mjs index a414d231c..d933ed068 100644 --- a/packages/mcp/test/login-singleflight.test.mjs +++ b/packages/mcp/test/login-singleflight.test.mjs @@ -75,3 +75,96 @@ test("onFailure runs once when a concurrent call joins the in-flight flow", asyn await Promise.allSettled([first.result, second.result]); assert.equal(failures, 1); }); + +test("freshKey: true bypasses the in-flight flow and starts a new one", async (t) => { + const home = mkdtempSync(join(tmpdir(), "memwal-test-")); + const prevHome = process.env.HOME; + const prevProfile = process.env.USERPROFILE; + process.env.HOME = home; + process.env.USERPROFILE = home; + resetInflightLogin(); + + t.after(() => { + if (prevHome === undefined) delete process.env.HOME; + else process.env.HOME = prevHome; + if (prevProfile === undefined) delete process.env.USERPROFILE; + else process.env.USERPROFILE = prevProfile; + resetInflightLogin(); + rmSync(home, { recursive: true, force: true }); + }); + + const opts = { + openBrowser: false, + timeoutMs: 1500, + webUrl: "http://127.0.0.1:9", + relayerUrl: "http://127.0.0.1:9", + label: "singleflight-fresh-test", + }; + const first = startOrReuseLoginFlow(opts); + const second = startOrReuseLoginFlow({ ...opts, freshKey: true }); + const [firstUrl, secondUrl] = await Promise.allSettled([first.url, second.url]); + assert.equal(firstUrl.status, "rejected"); + assert.equal(firstUrl.reason.name, "MemWalLoginSuperseded"); + assert.equal(secondUrl.status, "fulfilled"); + assert.match(secondUrl.value, /\/connect\/mcp\?/); + await Promise.allSettled([first.result, second.result]); +}); + +test("freshKey closes the previous listener so its timeout is not a failure", async (t) => { + const home = mkdtempSync(join(tmpdir(), "memwal-test-")); + const prevHome = process.env.HOME; + const prevProfile = process.env.USERPROFILE; + process.env.HOME = home; + process.env.USERPROFILE = home; + resetInflightLogin(); + + t.after(() => { + if (prevHome === undefined) delete process.env.HOME; + else process.env.HOME = prevHome; + if (prevProfile === undefined) delete process.env.USERPROFILE; + else process.env.USERPROFILE = prevProfile; + resetInflightLogin(); + rmSync(home, { recursive: true, force: true }); + }); + + const opts = { + openBrowser: false, + timeoutMs: 300, + webUrl: "http://127.0.0.1:9", + relayerUrl: "http://127.0.0.1:9", + label: "singleflight-supersede-test", + }; + let failures = 0; + const first = startOrReuseLoginFlow(opts, undefined, () => { + failures += 1; + }); + const urlA = await first.url; + const port = new URL(urlA).searchParams.get("port"); + const second = startOrReuseLoginFlow( + { ...opts, freshKey: true, timeoutMs: 10_000 }, + undefined, + () => { + failures += 1; + }, + ); + await second.url; + await new Promise((resolve) => setTimeout(resolve, 500)); + assert.equal(failures, 0); + await assert.rejects(first.result, (err) => { + assert.equal(err.name, "MemWalLoginSuperseded"); + return true; + }); + await assert.rejects( + fetch(`http://127.0.0.1:${port}/callback`, { + method: "POST", + headers: { + "content-type": "application/json", + origin: "http://127.0.0.1:9", + }, + body: "{}", + signal: AbortSignal.timeout(1000), + }), + ); + resetInflightLogin(); + await Promise.allSettled([second.result]); +}); diff --git a/packages/mcp/test/login-write-ahead.test.mjs b/packages/mcp/test/login-write-ahead.test.mjs index fa4667d51..abea23bac 100644 --- a/packages/mcp/test/login-write-ahead.test.mjs +++ b/packages/mcp/test/login-write-ahead.test.mjs @@ -179,6 +179,96 @@ test("a second login for the same relayer reuses the stranded keypair", async (t ); }); +/** + * WALM-645 — `memwal-mcp login` is the escape hatch when a stranded record + * can never be reclaimed (a relayer without `/api/whoami` leaves recovery in + * `unavailable` on every start, so the record survives its full 24h TTL while + * the wallet step keeps aborting with `EDelegateKeyAlreadyExists`). Forcing a + * login has to mint a new key rather than hand the browser the stuck one. + */ +test("a forced login mints a fresh key instead of reusing the pending record", async (t) => { + const home = freshHome(); + t.after(() => rmSync(home, { recursive: true, force: true })); + + mkdirSync(join(home, ".memwal"), { recursive: true }); + const stranded = { + delegatePrivateKey: "a".repeat(64), + delegatePublicKeyHex: "b".repeat(64), + delegateAddress: `0x${"c".repeat(64)}`, + relayerUrl: RELAYER, + label: "Stranded", + createdAt: new Date().toISOString(), + version: 1, + }; + writeFileSync(pendingPath(home), JSON.stringify(stranded), { mode: 0o600 }); + + const { flow, url } = await startLogin({ freshKey: true }); + flow.catch(() => {}); + + const minted = url.searchParams.get("publicKey")?.toLowerCase(); + assert.notEqual( + minted, + stranded.delegatePublicKeyHex, + "a forced login must not send the browser the key that is already stuck", + ); + + const after = JSON.parse(readFileSync(pendingPath(home), "utf8")); + assert.equal( + after.delegatePublicKeyHex.toLowerCase(), + minted, + "the write-ahead record must hold the key the browser was actually given", + ); + assert.notEqual(after.delegatePrivateKey, stranded.delegatePrivateKey); +}); + +/** + * WALM-645 — minting over the record destroys the only copy of a key the + * browser may already have paid to register. Nothing else will ever name it + * again: recovery reads that same record, and it is gone. So the flow has to + * say which registration it is orphaning, while it still can. + */ +test("a forced login names the pending key it is about to discard", async (t) => { + const home = freshHome(); + t.after(() => rmSync(home, { recursive: true, force: true })); + + mkdirSync(join(home, ".memwal"), { recursive: true }); + const stranded = { + delegatePrivateKey: "d".repeat(64), + delegatePublicKeyHex: "e".repeat(64), + delegateAddress: `0x${"f".repeat(64)}`, + relayerUrl: RELAYER, + label: "Stranded", + createdAt: new Date().toISOString(), + version: 1, + }; + writeFileSync(pendingPath(home), JSON.stringify(stranded), { mode: 0o600 }); + + const realWrite = process.stderr.write.bind(process.stderr); + const captured = []; + process.stderr.write = (chunk, ...rest) => { + captured.push(String(chunk)); + return realWrite(chunk, ...rest); + }; + t.after(() => { + process.stderr.write = realWrite; + }); + + const { flow } = await startLogin({ freshKey: true }); + flow.catch(() => {}); + + const stderr = captured.join(""); + assert.match( + stderr, + new RegExp(stranded.delegatePublicKeyHex), + "the discarded key's public half must be reported so it can be found in the dashboard", + ); + assert.match( + stderr, + new RegExp(stranded.delegateAddress), + "the discarded key's address must be reported too", + ); +}); + test("a login against a different relayer does not reuse the record", async (t) => { // A key registered against one relayer's account proves nothing to // another, and recovery must never repoint a record at a new relayer. From 56c807cff1d761105f061c0eef397b1cf9b22877 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Wed, 14 Oct 2026 20:47:23 +0700 Subject: [PATCH 45/70] fix(sdk): surface a failed auto-save from flush (#1209) * fix(sdk): surface a failed auto-save from flush Background analyze still does not fail the chat turn. flush() used to report success anyway. It now raises, and onSaveError / on_save_error fires when the save fails. Fixes #1117, #1150 * fix(sdk): isolate a throwing onSaveError from the host process The callback runs inside the background save. If it throws, log it and still report the original analyze failure from flush(). Fixes #1117, #1150 * fix(sdk): keep auto-save failures without pinning request frames The ledger stores a type and a one-line message, not the exception traceback. The callback runs before the record is stored, so a flush inside the callback cannot drop it. flush also records a task that failed or was cancelled. Fixes #1117, #1150 * fix(sdk): keep an auto-save failure when its callback flushes The failure is recorded before onSaveError / on_save_error. A flush from that callback does not clear it, and flush_sync does not join its own worker. * fix(sdk): return from on_save_error before joining workers A flush started inside the callback no longer drains, joins, or clears the ledger. The callback depth is updated under the lock so two overlapping workers cannot join each other or drop an increment. * fix(sdk): only the callback thread skips a failed auto-save A flush from on_save_error still returns without clearing the failure. A flush from any other thread waits and raises, which is the signal #1117 and #1150 ask for. --- .../python-sdk-memwal/memwal/middleware.py | 121 ++++++++++++-- .../tests/test_middleware.py | 154 ++++++++++++++++++ packages/sdk/src/ai/middleware.ts | 33 +++- packages/sdk/test/withMemWal-flush.test.mjs | 91 +++++++++++ 4 files changed, 382 insertions(+), 17 deletions(-) diff --git a/packages/python-sdk-memwal/memwal/middleware.py b/packages/python-sdk-memwal/memwal/middleware.py index 953d8523b..dd0d9edde 100644 --- a/packages/python-sdk-memwal/memwal/middleware.py +++ b/packages/python-sdk-memwal/memwal/middleware.py @@ -140,10 +140,67 @@ class _PendingSaves: run — this keeps a strong reference until each one is done. """ - def __init__(self) -> None: + def __init__( + self, + on_save_error: Optional[Callable[[BaseException], None]] = None, + ) -> None: self._tasks: "set[asyncio.Task[Any]]" = set() self._threads: List[threading.Thread] = [] + self._failures: List[tuple[str, str]] = [] + self._dropped = 0 + self._on_save_error = on_save_error self._lock = threading.Lock() + self._callback_threads: set[int] = set() + + def note_error(self, err: BaseException) -> None: + """Remember a background analyze failure so flush does not report success. + + The ledger is updated before the callback. A flush from that callback + returns without draining or clearing the failure. A flush from any + other thread still waits and raises. The ledger keeps only a type and + a one-line message, so a client that never flushes does not pin the + request traceback. + """ + type_name = type(err).__name__ + message = str(err).splitlines()[0][:200] if str(err) else "" + with self._lock: + if len(self._failures) < 8: + self._failures.append((type_name, message)) + else: + self._dropped += 1 + if self._on_save_error is None: + return + ident = threading.get_ident() + with self._lock: + self._callback_threads.add(ident) + try: + self._on_save_error(err) + except Exception: + logger.warning("Walrus Memory on_save_error raised", exc_info=True) + finally: + with self._lock: + self._callback_threads.discard(ident) + + def _flush_called_from_callback(self) -> bool: + with self._lock: + return threading.get_ident() in self._callback_threads + + def _note_result(self, result: Any) -> None: + if isinstance(result, BaseException): + self.note_error(result) + + def _raise_if_failed(self) -> None: + with self._lock: + failures, self._failures = self._failures, [] + dropped, self._dropped = self._dropped, 0 + total = len(failures) + dropped + if total == 0: + return + type_name, message = failures[0] if failures else ("Error", "") + detail = f"{type_name}: {message}" if message else type_name + raise RuntimeError( + f"Walrus Memory auto-save failed ({total} call(s)): {detail}" + ) def track_task(self, task: "asyncio.Task[Any]") -> None: self._tasks.add(task) @@ -169,25 +226,44 @@ def _run_and_untrack() -> None: thread.start() async def flush(self) -> None: - """Await every pending task, then join every pending thread.""" + """Await every pending task, then join every pending thread. + + Raises when any background auto-save failed. The chat call itself + still returned; this is the signal for a caller that waited. A call + made from ``on_save_error`` returns before draining, joining, or + clearing the failure being recorded. + """ + if self._flush_called_from_callback(): + return if self._tasks: - await asyncio.gather(*list(self._tasks), return_exceptions=True) + tasks = list(self._tasks) + results = await asyncio.gather(*tasks, return_exceptions=True) + for result in results: + self._note_result(result) self._join_threads() + self._raise_if_failed() def flush_sync(self) -> None: - """Join every pending thread. A pending Task belongs to the loop - that created it (e.g. an earlier `await llm.ainvoke(...)` call) and - cannot be awaited from a different one -- if any are still pending - here, the caller mixed sync and async entry points on the same - wrapped client, and this cleanup path can't safely drain them. Log - and continue draining threads rather than raise out of what's - usually shutdown code.""" + """Join every pending thread and raise when a background auto-save failed. + + A pending Task belongs to the loop that created it. When this thread + has no running loop, those tasks are awaited here. When it does, they + are logged and left for :meth:`flush`. A call made from + ``on_save_error`` returns before draining, joining, or clearing the + failure being recorded. + """ + if self._flush_called_from_callback(): + return if self._tasks: - async def _drain(tasks: "list[asyncio.Task[Any]]") -> None: - await asyncio.gather(*tasks, return_exceptions=True) + tasks = list(self._tasks) + + async def _drain(pending_tasks: "list[asyncio.Task[Any]]") -> None: + results = await asyncio.gather(*pending_tasks, return_exceptions=True) + for result in results: + self._note_result(result) try: - asyncio.run(_drain(list(self._tasks))) + asyncio.run(_drain(tasks)) except (RuntimeError, ValueError): logger.warning( "Walrus Memory flush_sync() could not await %d pending " @@ -197,10 +273,14 @@ async def _drain(tasks: "list[asyncio.Task[Any]]") -> None: len(self._tasks), ) self._join_threads() + self._raise_if_failed() def _join_threads(self) -> None: + current = threading.current_thread() with self._lock: threads, self._threads = self._threads, [] + self._threads.extend(thread for thread in threads if thread is current) + threads = [thread for thread in threads if thread is not current] for thread in threads: thread.join() @@ -291,6 +371,7 @@ def with_memwal_langchain( min_relevance: float = 0.3, debug: bool = False, env: Optional[str] = None, + on_save_error: Optional[Callable[[BaseException], None]] = None, ) -> "BaseChatModel": """Wrap a LangChain ``BaseChatModel`` with Walrus Memory management. @@ -313,6 +394,9 @@ def with_memwal_langchain( debug: Enable debug logging. env: Optional relayer preset (``"prod"`` / ``"dev"`` / ``"staging"`` / ``"local"``). Same precedence as :meth:`MemWal.create`. + on_save_error: Called with the original exception when a background + auto-save fails. ``flush`` / ``flush_sync`` from this callback does + not clear that failure. Returns: A wrapped ``BaseChatModel`` that automatically uses Walrus Memory. @@ -335,7 +419,7 @@ def with_memwal_langchain( ) log = logger.debug if not debug else logger.warning - pending = _PendingSaves() + pending = _PendingSaves(on_save_error) original_agenerate = llm._agenerate original_generate = llm._generate @@ -408,6 +492,7 @@ async def _post_analyze(messages: List[BaseMessage]) -> None: await memwal.analyze(user_text, namespace) except Exception as e: log(f"[Walrus Memory] Auto-save failed: {e}") + pending.note_error(e) async def patched_agenerate( messages: List[List[BaseMessage]], *args: Any, **kwargs: Any @@ -470,6 +555,7 @@ def with_memwal_openai( min_relevance: float = 0.3, debug: bool = False, env: Optional[str] = None, + on_save_error: Optional[Callable[[BaseException], None]] = None, ) -> Any: """Wrap an OpenAI client with Walrus Memory management. @@ -494,6 +580,9 @@ def with_memwal_openai( debug: Enable debug logging. env: Optional relayer preset (``"prod"`` / ``"dev"`` / ``"staging"`` / ``"local"``). Same precedence as :meth:`MemWal.create`. + on_save_error: Called with the original exception when a background + auto-save fails. ``flush`` / ``flush_sync`` from this callback does + not clear that failure. Returns: The same client, with ``chat.completions.create`` wrapped to use Walrus Memory. @@ -507,7 +596,7 @@ def with_memwal_openai( ) log = logger.debug if not debug else logger.warning - pending = _PendingSaves() + pending = _PendingSaves(on_save_error) is_async = hasattr(client, "_async_client") or type(client).__name__ == "AsyncOpenAI" @@ -572,6 +661,7 @@ async def _analyze() -> None: await memwal.analyze(user_text, namespace) except Exception as e: log(f"[Walrus Memory] Auto-save failed: {e}") + pending.note_error(e) _fire_and_forget(_analyze(), pending) @@ -633,6 +723,7 @@ def _analyze() -> None: _run_memwal(lambda: memwal.analyze(user_text, namespace)) except Exception as e: log(f"[Walrus Memory] Auto-save failed: {e}") + pending.note_error(e) pending.spawn_thread(_analyze) diff --git a/packages/python-sdk-memwal/tests/test_middleware.py b/packages/python-sdk-memwal/tests/test_middleware.py index c02fbf7bd..92d16fecc 100644 --- a/packages/python-sdk-memwal/tests/test_middleware.py +++ b/packages/python-sdk-memwal/tests/test_middleware.py @@ -1000,3 +1000,157 @@ def test_completed_threads_are_untracked_without_flushing(self) -> None: time.sleep(0.05) assert len(pending._threads) == 0 + + def test_flush_raises_when_a_background_save_failed(self) -> None: + seen: list[BaseException] = [] + pending = _PendingSaves(seen.append) + pending.note_error(RuntimeError("analyze down")) + + async def go() -> None: + await pending.flush() + + try: + asyncio.run(go()) + except RuntimeError as err: + assert "auto-save failed" in str(err) + else: + raise AssertionError("flush did not raise") + assert len(seen) == 1 + + def test_flush_sync_raises_when_a_background_save_failed(self) -> None: + pending = _PendingSaves() + pending.note_error(RuntimeError("analyze down")) + try: + pending.flush_sync() + except RuntimeError as err: + assert "auto-save failed" in str(err) + else: + raise AssertionError("flush_sync did not raise") + + def test_flush_sees_a_failing_task_without_note_error(self) -> None: + pending = _PendingSaves() + + async def fail() -> None: + raise RuntimeError("analyze down") + + async def go() -> None: + task = asyncio.create_task(fail()) + pending.track_task(task) + await pending.flush() + + try: + asyncio.run(go()) + except RuntimeError as err: + assert "analyze down" in str(err) + else: + raise AssertionError("flush did not raise") + + def test_callback_flush_does_not_swallow_the_failure(self) -> None: + pending = _PendingSaves() + + def on_error(_err: BaseException) -> None: + pending.flush_sync() + + pending._on_save_error = on_error + pending.note_error(RuntimeError("analyze down")) + + async def go() -> None: + await pending.flush() + + try: + asyncio.run(go()) + except RuntimeError as err: + assert "analyze down" in str(err) + else: + raise AssertionError("flush did not raise") + + def test_worker_callback_flush_sync_keeps_the_failure(self) -> None: + pending = _PendingSaves() + callback_error: list[str] = [] + saw_self = [] + + def on_error(_err: BaseException) -> None: + try: + pending.flush_sync() + except Exception as err: + callback_error.append(repr(err)) + with pending._lock: + saw_self.append(threading.current_thread() in pending._threads) + + pending._on_save_error = on_error + done = threading.Event() + + def fail() -> None: + pending.note_error(RuntimeError("analyze down")) + done.set() + + pending.spawn_thread(fail) + assert done.wait(2) + assert callback_error == [] + assert saw_self == [True] + try: + pending.flush_sync() + except RuntimeError as err: + assert "analyze down" in str(err) + else: + raise AssertionError("flush_sync did not raise") + + def test_two_worker_callbacks_do_not_join_each_other(self) -> None: + """Overlapping on_save_error flushes must not join the other worker. + + Each callback returns before drain, join, and raise. The later caller + flush still reports both failures. + """ + pending = _PendingSaves() + entered = threading.Barrier(2) + callback_error: list[str] = [] + done = threading.Barrier(3) + + def on_error(_err: BaseException) -> None: + entered.wait(2) + try: + pending.flush_sync() + except Exception as err: + callback_error.append(repr(err)) + + pending._on_save_error = on_error + + def fail(message: str): + def run() -> None: + pending.note_error(RuntimeError(message)) + done.wait(2) + return run + + pending.spawn_thread(fail("analyze down")) + pending.spawn_thread(fail("analyze also down")) + done.wait(2) + assert callback_error == [] + try: + pending.flush_sync() + except RuntimeError as err: + assert "2 call" in str(err) + assert "analyze down" in str(err) + else: + raise AssertionError("flush_sync did not raise") + + def test_flush_from_another_thread_still_raises_during_callback(self) -> None: + """The callback thread bails out. Any other flush still reports the failure.""" + import time + + pending = _PendingSaves() + started = threading.Event() + + def on_error(_err: BaseException) -> None: + started.set() + time.sleep(0.2) + + pending._on_save_error = on_error + pending.spawn_thread(lambda: pending.note_error(RuntimeError("analyze down"))) + assert started.wait(2) + try: + pending.flush_sync() + except RuntimeError as err: + assert "analyze down" in str(err) + else: + raise AssertionError("flush_sync did not raise") + diff --git a/packages/sdk/src/ai/middleware.ts b/packages/sdk/src/ai/middleware.ts index 09e225ea3..76fde61b2 100644 --- a/packages/sdk/src/ai/middleware.ts +++ b/packages/sdk/src/ai/middleware.ts @@ -54,6 +54,9 @@ export interface WithMemWalOptions extends MemWalConfig { debug?: boolean; /** Optional callback invoked when memory recall fails during transformParams */ onRecallError?: (error: unknown) => void; + /** Called when a background auto-save fails. The chat turn still returns. + * `flush()` also rejects, so a caller waiting on the write can see it. */ + onSaveError?: (error: unknown) => void; } // ============================================================ @@ -90,11 +93,28 @@ export function withMemWal( // callers (CLI scripts, serverless handlers) can await them via // flush() before the process exits, instead of losing them silently. const pendingSaves = new Set>(); + const saveFailures: string[] = []; + let notingFailure = 0; function saveInBackground(userMessage: string): void { const savePromise = memwal .analyze(userMessage) - .catch((err: unknown) => log("Auto-save failed:", err)); + .catch((err: unknown) => { + log("Auto-save failed:", err); + if (saveFailures.length < 8) { + const name = err instanceof Error ? err.name : "Error"; + const message = err instanceof Error ? err.message.split("\n")[0] : ""; + saveFailures.push(message ? `${name}: ${message}` : name); + } + notingFailure += 1; + try { + options.onSaveError?.(err); + } catch (callbackError) { + console.warn("[Walrus Memory] onSaveError threw:", callbackError); + } finally { + notingFailure -= 1; + } + }); pendingSaves.add(savePromise); savePromise.finally(() => pendingSaves.delete(savePromise)); } @@ -177,7 +197,16 @@ export function withMemWal( // Lets short-lived callers await outstanding auto-save writes before // exiting, e.g. `await model.flush()` right before `process.exit()`. wrapped.flush = async (): Promise => { - await Promise.allSettled([...pendingSaves]); + if (notingFailure === 0) { + await Promise.allSettled([...pendingSaves]); + } + if (notingFailure > 0 || saveFailures.length === 0) return; + const errors = saveFailures.splice(0, saveFailures.length); + const failure = new Error( + `MemWal auto-save failed (${errors.length} call(s))`, + ); + (failure as Error & { cause?: unknown }).cause = errors[0]; + throw failure; }; return wrapped; diff --git a/packages/sdk/test/withMemWal-flush.test.mjs b/packages/sdk/test/withMemWal-flush.test.mjs index 114817f00..16bf16906 100644 --- a/packages/sdk/test/withMemWal-flush.test.mjs +++ b/packages/sdk/test/withMemWal-flush.test.mjs @@ -86,3 +86,94 @@ test("a wrapped v2 model still reports its finish reason", async () => { MemWal.prototype.analyze = originalAnalyze; } }); + +test("flush() rejects when auto-save analyze() fails, and onSaveError runs", async () => { + const originalRecall = MemWal.prototype.recall; + const originalAnalyze = MemWal.prototype.analyze; + const seen = []; + + MemWal.prototype.recall = async () => ({ results: [] }); + MemWal.prototype.analyze = async () => { + throw new Error("analyze down"); + }; + + try { + const model = withMemWal(fakeLanguageModel(), { + key: TEST_KEY, + accountId: TEST_ACCOUNT_ID, + onSaveError(err) { + seen.push(err); + }, + }); + + const result = await model.doGenerate({ + prompt: [{ role: "user", content: [{ type: "text", text: "remember this" }] }], + }); + assert.equal(result.finishReason, "stop"); + + await assert.rejects( + () => model.flush(), + /auto-save failed/, + ); + assert.equal(seen.length, 1); + assert.match(String(seen[0]), /analyze down/); + } finally { + MemWal.prototype.recall = originalRecall; + MemWal.prototype.analyze = originalAnalyze; + } +}); + +test("a throwing onSaveError does not reject the model call", async () => { + const originalRecall = MemWal.prototype.recall; + const originalAnalyze = MemWal.prototype.analyze; + MemWal.prototype.recall = async () => ({ results: [] }); + MemWal.prototype.analyze = async () => { + throw new Error("analyze down"); + }; + try { + const model = withMemWal(fakeLanguageModel(), { + key: TEST_KEY, + accountId: TEST_ACCOUNT_ID, + onSaveError() { + throw new Error("telemetry down"); + }, + }); + const result = await model.doGenerate({ + prompt: [{ role: "user", content: [{ type: "text", text: "remember this" }] }], + }); + assert.equal(result.finishReason, "stop"); + await assert.rejects(() => model.flush(), /auto-save failed/); + } finally { + MemWal.prototype.recall = originalRecall; + MemWal.prototype.analyze = originalAnalyze; + } +}); + +test("flush() from onSaveError does not swallow the failure", async () => { + const originalRecall = MemWal.prototype.recall; + const originalAnalyze = MemWal.prototype.analyze; + MemWal.prototype.recall = async () => ({ results: [] }); + MemWal.prototype.analyze = async () => { + throw new Error("analyze down"); + }; + try { + let model; + model = withMemWal(fakeLanguageModel(), { + key: TEST_KEY, + accountId: TEST_ACCOUNT_ID, + onSaveError() { + void model.flush(); + }, + }); + const result = await model.doGenerate({ + prompt: [{ role: "user", content: [{ type: "text", text: "remember this" }] }], + }); + assert.equal(result.finishReason, "stop"); + await new Promise((resolve) => setImmediate(resolve)); + await assert.rejects(() => model.flush(), /auto-save failed/); + } finally { + MemWal.prototype.recall = originalRecall; + MemWal.prototype.analyze = originalAnalyze; + } +}); + From 52180323c6c6ab1583df0665712ba0991316fc87 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Wed, 14 Oct 2026 20:47:29 +0700 Subject: [PATCH 46/70] fix(sdk): preserve matching total under recall maxDistance filter (Fixes #1066, WALM-840) (#1198) * fix(sdk): preserve matching total under recall maxDistance filter (Fixes #1066, WALM-840) - In TypeScript SDK (memwal.ts), avoid overwriting total with filtered.length when maxDistance is applied, preserving the relayer's matching hit count - In Python SDK (client.py), preserve total from response data instead of overwriting with len(memories) after max_distance filtering - Update test suites to assert that total reflects matching memories while results carries post-filter hits, allowing callers to distinguish between an empty namespace and results trimmed by distance cutoff * test(sdk): expect the relayer total after a maxDistance filter The live e2e still asserted total equalled the filtered length. A cutoff that drops rows keeps the page total, including when every row is dropped. Fixes #1066 * fix(sdk): keep the mock recall total when maxDistance drops the page The offline mocks now match the HTTP clients: total is the page length before the client-side cutoff. maxTokens still replaces total with the budgeted length. maxDistance itself is unchanged. --- packages/python-sdk-memwal/memwal/client.py | 6 ++--- packages/python-sdk-memwal/memwal/mock.py | 9 ++++--- .../python-sdk-memwal/tests/test_client.py | 26 ++++++++++++++++++- packages/python-sdk-memwal/tests/test_mock.py | 5 ++++ packages/sdk/src/memwal.ts | 3 ++- packages/sdk/src/mock.ts | 25 +++++++++--------- packages/sdk/test/e2e/live.e2e.mjs | 10 ++++--- packages/sdk/test/mock.test.mjs | 12 +++++++-- .../sdk/test/recall-max-distance.test.mjs | 15 ++++++++--- 9 files changed, 79 insertions(+), 32 deletions(-) diff --git a/packages/python-sdk-memwal/memwal/client.py b/packages/python-sdk-memwal/memwal/client.py index a0662612e..940b4d559 100644 --- a/packages/python-sdk-memwal/memwal/client.py +++ b/packages/python-sdk-memwal/memwal/client.py @@ -1063,14 +1063,12 @@ async def recall( dropped = int(raw_dropped) except (TypeError, ValueError): dropped = 0 + total_count = data.get("total", len(memories)) if max_distance is not None: memories = [m for m in memories if m.distance < max_distance] - return RecallResult( - results=memories, total=len(memories), dropped_count=dropped - ) return RecallResult( results=memories, - total=data.get("total", len(memories)), + total=total_count, dropped_count=dropped, ) diff --git a/packages/python-sdk-memwal/memwal/mock.py b/packages/python-sdk-memwal/memwal/mock.py index 31737c024..1b5996ae0 100644 --- a/packages/python-sdk-memwal/memwal/mock.py +++ b/packages/python-sdk-memwal/memwal/mock.py @@ -278,14 +278,15 @@ async def recall( for memory in self._memories if memory.namespace == resolved_namespace ] - if max_distance is not None: - ranked = [item for item in ranked if item[1] < max_distance] ranked.sort(key=lambda item: (item[1], item[0].sequence)) + page = ranked[:limit] + if max_distance is not None: + page = [item for item in page if item[1] < max_distance] results = [ RecallMemory(blob_id=memory.blob_id, text=memory.text, distance=score) - for memory, score in ranked[:limit] + for memory, score in page ] - return RecallResult(results=results, total=len(results)) + return RecallResult(results=results, total=len(ranked[:limit])) async def analyze( self, diff --git a/packages/python-sdk-memwal/tests/test_client.py b/packages/python-sdk-memwal/tests/test_client.py index 1be9dee39..064b333ee 100644 --- a/packages/python-sdk-memwal/tests/test_client.py +++ b/packages/python-sdk-memwal/tests/test_client.py @@ -935,9 +935,33 @@ async def test_max_distance_filters_results(self, memwal_client: MemWal) -> None body = json.loads(route.calls[0].request.content) assert body["limit"] == 10 assert len(result.results) == 1 - assert result.total == 1 + assert result.total == 2 assert result.results[0].blob_id == "b1" + @respx.mock + async def test_max_distance_filters_all_preserves_total( + self, memwal_client: MemWal + ) -> None: + """When all matches exceed max_distance, results is empty but total + preserves matches (#1066).""" + mock_seal_session_prereqs() + respx.post(f"{_TEST_SERVER}/api/recall").mock( + return_value=httpx.Response( + 200, + json={ + "results": [ + {"blob_id": "b1", "text": "I love coffee", "distance": 0.8}, + {"blob_id": "b2", "text": "I live in Tokyo", "distance": 0.9}, + ], + "total": 2, + }, + ) + ) + + result = await memwal_client.recall("coffee", limit=10, max_distance=0.5) + assert len(result.results) == 0 + assert result.total == 2 + @respx.mock async def test_get_signed_request_uses_empty_body_hash_and_no_wire_body( self, memwal_client: MemWal diff --git a/packages/python-sdk-memwal/tests/test_mock.py b/packages/python-sdk-memwal/tests/test_mock.py index 9a3e006fc..f094528ce 100644 --- a/packages/python-sdk-memwal/tests/test_mock.py +++ b/packages/python-sdk-memwal/tests/test_mock.py @@ -57,6 +57,11 @@ async def test_mock_isolates_namespaces_and_honors_max_distance(): assert [memory.text for memory in alice.results] == ["Alice likes ramen"] assert [memory.text for memory in bob.results] == ["Bob likes tacos"] assert empty.total == 0 + dropped = await mock.recall( + RecallParams(query="likes", namespace="user-a", max_distance=0) + ) + assert dropped.results == [] + assert dropped.total == 1 @pytest.mark.asyncio diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index f32750192..d1e09ce2e 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -1178,10 +1178,11 @@ export class MemWal { const filtered = result.results.filter( (memory) => memory.distance < options.maxDistance!, ); + // Keep the relayer's matching total so callers can distinguish + // "no memories exist" from "matches exist but were outside maxDistance" (#1066). processed = { ...processed, results: filtered, - total: filtered.length, }; } diff --git a/packages/sdk/src/mock.ts b/packages/sdk/src/mock.ts index 79b4e6879..311284198 100644 --- a/packages/sdk/src/mock.ts +++ b/packages/sdk/src/mock.ts @@ -406,11 +406,6 @@ export class MemWalMock { memory, distance: distance(queryTokens, memory.text), })) - .filter((entry) => - options.maxDistance === undefined - ? true - : entry.distance < options.maxDistance - ) .sort( (left, right) => left.distance - right.distance || @@ -428,13 +423,17 @@ export class MemWalMock { left.distance - right.distance, ) : semantic; - const ranked = ordered - .slice(0, resolvedLimit) - .map(({ memory, distance: memoryDistance }) => ({ - blob_id: memory.blobId, - text: memory.text, - distance: memoryDistance, - })); + const page = ordered.slice(0, resolvedLimit); + const maxDistance = options.maxDistance; + const visible = + maxDistance === undefined + ? page + : page.filter((entry) => entry.distance < maxDistance); + const ranked = visible.map(({ memory, distance: memoryDistance }) => ({ + blob_id: memory.blobId, + text: memory.text, + distance: memoryDistance, + })); if (options.maxTokens !== undefined) { const { results, meta } = applyTokenBudget( @@ -446,7 +445,7 @@ export class MemWalMock { return { results, total: ranked.length, meta }; } - return { results: ranked, total: ranked.length }; + return { results: ranked, total: page.length }; } async analyze( diff --git a/packages/sdk/test/e2e/live.e2e.mjs b/packages/sdk/test/e2e/live.e2e.mjs index 0960ac7bf..5d8b3cbc1 100644 --- a/packages/sdk/test/e2e/live.e2e.mjs +++ b/packages/sdk/test/e2e/live.e2e.mjs @@ -267,16 +267,18 @@ describe("authenticated", { skip: requiresKey, concurrency: 1 }, () => { assert.ok(near.text.includes("lisinopril"), `closest hit should be the medication: ${near.text}`); assert.ok(far.distance > near.distance, "the two hits must be at different distances"); - // Cutoff between the two: only the close hit survives. + // Cutoff between the two: only the close hit survives. `total` stays + // the relayer page size, so a filtered page is not `total: 0`. const cutoff = (near.distance + far.distance) / 2; const filtered = await mw.recall({ query, limit: 5, maxDistance: cutoff }); assert.deepEqual(filtered.results.map((r) => r.blob_id), [near.blob_id]); - assert.equal(filtered.total, 1); + assert.equal(filtered.total, all.total); - // Cutoff at the closest distance: `>=` drops it too. + // Cutoff at the closest distance: `>=` drops it too, but the page + // was not empty. const none = await mw.recall({ query, limit: 5, maxDistance: near.distance }); assert.deepEqual(none.results, []); - assert.equal(none.total, 0); + assert.equal(none.total, all.total); }); test("listMemories counts a fresh namespace exactly", async () => { diff --git a/packages/sdk/test/mock.test.mjs b/packages/sdk/test/mock.test.mjs index 9f7971ccd..2663e262e 100644 --- a/packages/sdk/test/mock.test.mjs +++ b/packages/sdk/test/mock.test.mjs @@ -60,6 +60,13 @@ test("MemWalMock isolates namespaces and honors maxDistance", async () => { ["Bob likes tacos"] ); assert.equal(empty.total, 0); + const dropped = await mock.recall({ + query: "likes", + namespace: "user-a", + maxDistance: 0, + }); + assert.equal(dropped.results.length, 0); + assert.equal(dropped.total, 1); }); test("MemWalMock matches production recall overloads and topK precedence", async () => { @@ -369,7 +376,7 @@ test("MemWalMock collapses a repeated idempotency key onto the original job", as ); assert.equal( (await mock.recall({ query: "other", namespace: "ns", maxDistance: 0.5 })).total, - 0, + 1, ); // WALM-781: Idempotency keys are scoped per namespace. // The same key used in another namespace must succeed and not collide. @@ -437,7 +444,8 @@ test("MemWalMock collapses a repeated idempotency key onto the original job", as namespace: "ns", maxDistance: 0.5, }); - assert.equal(after.total, 1); + assert.equal(after.results.length, 1); + assert.ok(after.total > after.results.length); assert.equal(after.results[0].blob_id, rewritten.blob_id); const cleared = MemWalMock.create(); diff --git a/packages/sdk/test/recall-max-distance.test.mjs b/packages/sdk/test/recall-max-distance.test.mjs index 494b7e3ce..60644d2c8 100644 --- a/packages/sdk/test/recall-max-distance.test.mjs +++ b/packages/sdk/test/recall-max-distance.test.mjs @@ -55,20 +55,21 @@ test("recall with maxDistance keeps hits inside the cutoff and drops the rest", stubRecall(hits); const result = await client().recall({ query: "meds", maxDistance: 0.7 }); assert.deepEqual(result.results.map((h) => h.blob_id), ["meds"]); - assert.equal(result.total, 1); + assert.equal(result.total, 3, "total preserves relayer count"); }); test("recall with maxDistance drops a hit exactly at the cutoff", async () => { stubRecall(hits); const result = await client().recall({ query: "meds", maxDistance: 0.75 }); assert.deepEqual(result.results.map((h) => h.blob_id), ["meds"]); + assert.equal(result.total, 3, "total preserves relayer count"); }); -test("recall with maxDistance can return nothing", async () => { +test("recall with maxDistance can return nothing while preserving total", async () => { stubRecall(hits); const result = await client().recall({ query: "weather?", maxDistance: 0.5 }); assert.deepEqual(result.results, []); - assert.equal(result.total, 0); + assert.equal(result.total, 3, "total preserves relayer count so caller distinguishes from empty namespace"); }); test("recall without maxDistance returns every hit, including distant ones", async () => { @@ -81,4 +82,12 @@ test("positional recall(query, { maxDistance }) applies the same cutoff", async stubRecall(hits); const result = await client().recall("meds", { maxDistance: 0.8 }); assert.deepEqual(result.results.map((h) => h.blob_id), ["meds", "pharmacy"]); + assert.equal(result.total, 3, "total preserves relayer count"); +}); + +test("recall on genuinely empty namespace returns 0 total and empty results", async () => { + stubRecall([]); + const result = await client().recall({ query: "anything", maxDistance: 0.5 }); + assert.deepEqual(result.results, []); + assert.equal(result.total, 0, "genuinely empty namespace reports total 0"); }); From dd00818c41056204ae8c52342d6308e9b3f40325 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Wed, 14 Oct 2026 20:47:36 +0700 Subject: [PATCH 47/70] fix(mcp): derive remember idempotency key from resolved namespace (Fixes #1093, WALM-831) (#1195) --- .../remember-idempotency-key.test.ts | 125 ++++++++++++++++++ .../server/scripts/mcp/tools/remember-wait.ts | 3 +- services/server/scripts/mcp/tools/remember.ts | 3 +- 3 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 services/server/scripts/mcp/__tests__/remember-idempotency-key.test.ts diff --git a/services/server/scripts/mcp/__tests__/remember-idempotency-key.test.ts b/services/server/scripts/mcp/__tests__/remember-idempotency-key.test.ts new file mode 100644 index 000000000..dad2343b6 --- /dev/null +++ b/services/server/scripts/mcp/__tests__/remember-idempotency-key.test.ts @@ -0,0 +1,125 @@ +process.env.MEMWAL_MCP_REMEMBER_WAIT_MS = "0"; + +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import test, { type TestContext } from "node:test"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import type { MemWalSession } from "../auth.js"; + +const { createMcpServer } = await import("../server.js"); +const { derivedIdempotencyKey } = await import("../tools/remember-wait.js"); + +interface RememberCall { + text: string; + namespace?: string; + idempotencyKey: string; +} + +function sessionWithCalls( + calls: RememberCall[], + sessionNamespace?: string, +): MemWalSession { + return { + oauthScope: "memwal:read memwal:write", + namespace: sessionNamespace, + memwal: { + async rememberAsync( + text: string, + namespace?: string, + opts?: { idempotencyKey?: string }, + ) { + calls.push({ + text, + namespace, + idempotencyKey: opts?.idempotencyKey ?? "", + }); + return { job_id: "job-1", status: "running" }; + }, + }, + } as unknown as MemWalSession; +} + +async function clientFor(session: MemWalSession, t: TestContext): Promise { + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + const server = createMcpServer(session); + const client = new Client({ name: "remember-idempotency-test", version: "1.0.0" }); + t.after(async () => { + await client.close(); + await server.close(); + }); + await server.connect(serverTransport); + await client.connect(clientTransport); + return client; +} + +test("derivedIdempotencyKey defaults undefined or empty namespace to 'default'", () => { + const fact = "the user prefers pour-over coffee"; + const keyUndefined = derivedIdempotencyKey(undefined, fact); + const keyEmpty = derivedIdempotencyKey("", fact); + const keyDefault = derivedIdempotencyKey("default", fact); + + assert.equal(keyUndefined, keyDefault, "omitted namespace must match explicit default"); + assert.equal(keyEmpty, keyDefault, "empty namespace must match explicit default"); + + // Must match the SDK format: r1- + const bucket = Math.floor(Date.now() / (30 * 60 * 1000)); + const expectedDigest = createHash("sha256") + .update(`${bucket}\0default\0${fact}`) + .digest("hex"); + assert.equal(keyDefault, `r1-${expectedDigest}`); +}); + +test("memwal_remember with omitted namespace sends idempotency key scoped to 'default' (GH #1093, WALM-831)", async (t) => { + const calls: RememberCall[] = []; + const client = await clientFor(sessionWithCalls(calls), t); + const fact = "user prefers tabs over spaces"; + + await client.callTool({ + name: "memwal_remember", + arguments: { text: fact }, + }); + + await client.callTool({ + name: "memwal_remember", + arguments: { text: fact, namespace: "default" }, + }); + + assert.equal(calls.length, 2); + // When omitted, tool passes undefined as namespace argument to SDK rememberAsync + assert.equal(calls[0].namespace, undefined); + assert.equal(calls[1].namespace, "default"); + + // But both must have derived the exact same idempotencyKey under 'default' + assert.equal(calls[0].idempotencyKey, calls[1].idempotencyKey); + assert.equal(calls[0].idempotencyKey, derivedIdempotencyKey("default", fact)); +}); + +test("memwal_remember with omitted namespace respects session.namespace if configured", async (t) => { + const calls: RememberCall[] = []; + const client = await clientFor(sessionWithCalls(calls, "project-team-a"), t); + const fact = "repo build uses rust edition 2024"; + + await client.callTool({ + name: "memwal_remember", + arguments: { text: fact }, + }); + + assert.equal(calls.length, 1); + assert.equal(calls[0].idempotencyKey, derivedIdempotencyKey("project-team-a", fact)); +}); + +test("memwal_remember with explicit namespace uses that namespace for idempotency key", async (t) => { + const calls: RememberCall[] = []; + const client = await clientFor(sessionWithCalls(calls, "default"), t); + const fact = "server runs on port 8080"; + + await client.callTool({ + name: "memwal_remember", + arguments: { text: fact, namespace: "custom-zone" }, + }); + + assert.equal(calls.length, 1); + assert.equal(calls[0].namespace, "custom-zone"); + assert.equal(calls[0].idempotencyKey, derivedIdempotencyKey("custom-zone", fact)); +}); diff --git a/services/server/scripts/mcp/tools/remember-wait.ts b/services/server/scripts/mcp/tools/remember-wait.ts index 8483a2b6b..e1c9b7763 100644 --- a/services/server/scripts/mcp/tools/remember-wait.ts +++ b/services/server/scripts/mcp/tools/remember-wait.ts @@ -43,9 +43,10 @@ const IDEMPOTENCY_BUCKET_MS = 30 * 60 * 1000; * `idempotent: false` instead of calling this. */ export function derivedIdempotencyKey(namespace: string | undefined, text: string): string { + const resolvedNamespace = namespace || "default"; const bucket = Math.floor(Date.now() / IDEMPOTENCY_BUCKET_MS); const digest = createHash("sha256") - .update(`${bucket}\0${namespace ?? ""}\0${text}`) + .update(`${bucket}\0${resolvedNamespace}\0${text}`) .digest("hex"); return `r1-${digest}`; } diff --git a/services/server/scripts/mcp/tools/remember.ts b/services/server/scripts/mcp/tools/remember.ts index 91979ba68..25aa17b90 100644 --- a/services/server/scripts/mcp/tools/remember.ts +++ b/services/server/scripts/mcp/tools/remember.ts @@ -80,6 +80,7 @@ export function registerRememberTool( // Two steps rather than `rememberAndWait`, because the accept and // the wait need separate budgets: acceptance is the part that // must succeed, the wait is a courtesy we cut short. + const resolvedNamespace = namespace ?? session.namespace ?? "default"; const accepted = await withAcceptDeadline( withRelayerRetry( () => @@ -89,7 +90,7 @@ export function registerRememberTool( // accept-timeout message's retry promise true. // Keyed on the REDACTED text, so a retry of the // same fact derives the same key. - idempotencyKey: derivedIdempotencyKey(namespace, safeText), + idempotencyKey: derivedIdempotencyKey(resolvedNamespace, safeText), }), "save this fact", ), From 6bd33106c8ca12a702aaa737d425dbe9605ad902 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Wed, 14 Oct 2026 20:47:49 +0700 Subject: [PATCH 48/70] fix(mcp): validate hex input in hexToBytes and support uppercase 0X (Fixes #1162) (#1174) --- packages/mcp/src/crypto.ts | 16 +++++++++++++- .../mcp/test/login-recovery-signing.test.mjs | 21 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/packages/mcp/src/crypto.ts b/packages/mcp/src/crypto.ts index fbc3aa53e..16fae101e 100644 --- a/packages/mcp/src/crypto.ts +++ b/packages/mcp/src/crypto.ts @@ -11,7 +11,21 @@ function hex(bytes: Uint8Array): string { } function fromHex(s: string): Uint8Array { - const clean = s.startsWith("0x") ? s.slice(2) : s; + if (typeof s !== "string") { + throw new TypeError("hexToBytes: expected string input"); + } + const clean = s.startsWith("0x") || s.startsWith("0X") ? s.slice(2) : s; + if (clean.length === 0) { + throw new Error("hexToBytes: empty hex string"); + } + if (clean.length % 2 !== 0) { + throw new Error( + `hexToBytes: odd-length hex string (length=${clean.length}); hex must have an even number of digits`, + ); + } + if (!/^[0-9a-fA-F]+$/.test(clean)) { + throw new Error("hexToBytes: input contains non-hex characters"); + } const out = new Uint8Array(clean.length / 2); for (let i = 0; i < out.length; i++) { out[i] = parseInt(clean.slice(i * 2, i * 2 + 2), 16); diff --git a/packages/mcp/test/login-recovery-signing.test.mjs b/packages/mcp/test/login-recovery-signing.test.mjs index 8234c7974..6d04d575d 100644 --- a/packages/mcp/test/login-recovery-signing.test.mjs +++ b/packages/mcp/test/login-recovery-signing.test.mjs @@ -93,3 +93,24 @@ test("the signature the client produces verifies against its public key", async ); assert.equal(tampered, false, "a different path must not verify"); }); + +test("hexToBytes round-trips even hex with 0x and 0X prefixes", () => { + assert.deepEqual(hexToBytes("deadbeef"), Uint8Array.from([0xde, 0xad, 0xbe, 0xef])); + assert.deepEqual(hexToBytes("0xdeadbeef"), Uint8Array.from([0xde, 0xad, 0xbe, 0xef])); + assert.deepEqual(hexToBytes("0XDEADBEEF"), Uint8Array.from([0xde, 0xad, 0xbe, 0xef])); +}); + +test("hexToBytes rejects empty, odd-length, non-hex, and non-string inputs", () => { + assert.throws(() => hexToBytes(null), /expected string input/); + assert.throws(() => hexToBytes(123), /expected string input/); + assert.throws(() => hexToBytes(""), /empty hex string/); + assert.throws(() => hexToBytes("0x"), /empty hex string/); + assert.throws(() => hexToBytes("0X"), /empty hex string/); + assert.throws(() => hexToBytes("abc"), /odd-length/); + assert.throws(() => hexToBytes("0xabc"), /odd-length/); + assert.throws(() => hexToBytes("zz"), /non-hex/); + assert.throws(() => hexToBytes("0Xzz"), /non-hex/); + assert.throws(() => hexToBytes("deadbeefzz"), /non-hex/); + assert.throws(() => hexToBytes("ab cd"), /non-hex/); +}); + From 5d94673c4ff45a15b1c0205219a3c810235b1bca Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Wed, 14 Oct 2026 20:47:55 +0700 Subject: [PATCH 49/70] fix(python-sdk): redact IPv6, 0.0.0.0, and localhost subdomains in error messages (Fixes #1161) (#1173) --- packages/python-sdk-memwal/memwal/client.py | 2 +- .../python-sdk-memwal/tests/test_client.py | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/packages/python-sdk-memwal/memwal/client.py b/packages/python-sdk-memwal/memwal/client.py index 940b4d559..df910d5d2 100644 --- a/packages/python-sdk-memwal/memwal/client.py +++ b/packages/python-sdk-memwal/memwal/client.py @@ -1815,7 +1815,7 @@ class MemWalClockDriftError(MemWalError): def _redact_internal_urls(text: str) -> str: return re.sub( - r"https?://(?:localhost|127\.0\.0\.1)(?::\d+)?[^\s)\]>'\"]*", + r"https?://(?:(?:[\w-]+\.)*localhost|127\.\d{1,3}\.\d{1,3}\.\d{1,3}|\[::1\]|0\.0\.0\.0)(?::\d+)?[^\s)\]>'\"]*", "[internal]", text, flags=re.IGNORECASE, diff --git a/packages/python-sdk-memwal/tests/test_client.py b/packages/python-sdk-memwal/tests/test_client.py index 064b333ee..f4f9420a0 100644 --- a/packages/python-sdk-memwal/tests/test_client.py +++ b/packages/python-sdk-memwal/tests/test_client.py @@ -1607,3 +1607,25 @@ async def test_builds_cached_session_envelope(self, memwal_client: MemWal) -> No assert exported["ttlMin"] == 5 assert exported["sessionKey"].startswith("suiprivkey") assert exported["personalMessageSignature"] + + +class TestRedactInternalUrls: + @pytest.mark.parametrize( + "raw, expected", + [ + ("Error at http://localhost:8080/path", "Error at [internal]"), + ("Error at http://127.0.0.1:3000/api", "Error at [internal]"), + ("Error at http://127.0.0.2:8000/status", "Error at [internal]"), + ("Error at http://127.128.0.1:9000/info", "Error at [internal]"), + ("Error at http://[::1]:9000/seal/encrypt", "Error at [internal]"), + ("Error at http://0.0.0.0:8000/health", "Error at [internal]"), + ("Error at http://sidecar.localhost:8080/v1", "Error at [internal]"), + ("Error at http://sub.domain.localhost:3000/test", "Error at [internal]"), + ("Error at https://api.wal.app/v1/remember", "Error at https://api.wal.app/v1/remember"), + ], + ) + def test_redact_internal_urls(self, raw: str, expected: str) -> None: + from memwal.client import _redact_internal_urls + + assert _redact_internal_urls(raw) == expected + From 2bccdbb5ddade84fdea050bd1bc34b6517657f96 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Wed, 14 Oct 2026 20:48:03 +0700 Subject: [PATCH 50/70] fix(python-sdk): support uppercase 0X prefix in normalize_private_key (#1172) Allow private keys with uppercase 0X prefix to be normalized without failing during hex decoding. Matches TypeScript SDK behavior. Fixes #1163 --- packages/python-sdk-memwal/memwal/utils.py | 4 +++- packages/python-sdk-memwal/tests/test_signing.py | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/python-sdk-memwal/memwal/utils.py b/packages/python-sdk-memwal/memwal/utils.py index 4bd1abe5f..4c686ddec 100644 --- a/packages/python-sdk-memwal/memwal/utils.py +++ b/packages/python-sdk-memwal/memwal/utils.py @@ -299,7 +299,9 @@ def normalize_private_key(key: str) -> str: candidate = key.strip() if candidate.lower().startswith("suiprivkey1"): return bytes_to_hex(decode_sui_private_key(candidate)) - return candidate[2:] if candidate.startswith("0x") else candidate + if candidate.startswith(("0x", "0X")): + return candidate[2:] + return candidate def encode_sui_private_key(seed_bytes: bytes) -> str: diff --git a/packages/python-sdk-memwal/tests/test_signing.py b/packages/python-sdk-memwal/tests/test_signing.py index d07a94f13..7ada6df36 100644 --- a/packages/python-sdk-memwal/tests/test_signing.py +++ b/packages/python-sdk-memwal/tests/test_signing.py @@ -324,11 +324,20 @@ def test_normalize_accepts_every_accepted_form(self) -> None: for form in ( self._SEED_HEX, f"0x{self._SEED_HEX}", + f"0X{self._SEED_HEX}", + f" 0X{self._SEED_HEX} ", encode_sui_private_key(seed), f" {encode_sui_private_key(seed)} ", ): assert normalize_private_key(form) == self._SEED_HEX + def test_uppercase_0x_prefix_key_produces_valid_client(self) -> None: + from memwal.client import MemWal + + upper_0x = f"0X{self._SEED_HEX}" + client = MemWal.create(key=upper_0x, account_id="0xtest") + assert client._private_key_hex == self._SEED_HEX + def test_bech32_key_produces_the_same_client_identity(self) -> None: from memwal.client import MemWal from memwal.utils import encode_sui_private_key From 4a5199041ad712041c464369c88bc06dcd3c3d90 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Wed, 14 Oct 2026 20:50:40 +0700 Subject: [PATCH 51/70] fix(sdk): validate accountId in MemWal.create and MemWalManual.create (Fixes #1155) (#1175) --- packages/sdk/src/manual.ts | 5 ++ packages/sdk/src/memwal.ts | 5 ++ .../create-account-id-validation.test.mjs | 65 +++++++++++++++++++ 3 files changed, 75 insertions(+) create mode 100644 packages/sdk/test/create-account-id-validation.test.mjs diff --git a/packages/sdk/src/manual.ts b/packages/sdk/src/manual.ts index 413b6223c..d17b3ac69 100644 --- a/packages/sdk/src/manual.ts +++ b/packages/sdk/src/manual.ts @@ -191,6 +191,11 @@ export class MemWalManual { // against non-localhost hosts. this.serverUrl = normalizeServerUrl(config.serverUrl ?? "https://relayer.memory.walrus.xyz"); this.walletSigner = config.walletSigner ?? null; + if (typeof config.accountId !== "string" || config.accountId.trim().length === 0) { + throw new TypeError( + `MemWalManual.create: "accountId" is required (Walrus Memory object ID, e.g. 0x...); got ${config.accountId}`, + ); + } this.config = config; if (config.namespace !== undefined && !config.namespace) { throw new Error("namespace cannot be empty"); diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index d1e09ce2e..5ba1cc26e 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -490,6 +490,11 @@ export class MemWal { : typeof this.privateKey; throw new Error(`Ed25519 seed must be exactly 32 bytes, got ${got}`); } + if (typeof config.accountId !== "string" || config.accountId.trim().length === 0) { + throw new TypeError( + `MemWal.create: "accountId" is required (Walrus Memory object ID, e.g. 0x...); got ${config.accountId}`, + ); + } this.accountId = config.accountId; // LOW-22: default to HTTPS for production usage; normalizeServerUrl // warns (does not throw) if a user passes plain http:// for a diff --git a/packages/sdk/test/create-account-id-validation.test.mjs b/packages/sdk/test/create-account-id-validation.test.mjs new file mode 100644 index 000000000..86eedbf4b --- /dev/null +++ b/packages/sdk/test/create-account-id-validation.test.mjs @@ -0,0 +1,65 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { MemWal } from "../dist/memwal.js"; +import { MemWalManual } from "../dist/manual.js"; + +const VALID_KEY = "01".repeat(32); +const VALID_ACCOUNT_ID = `0x${"02".repeat(32)}`; + +test("MemWal.create throws TypeError when accountId is missing or undefined", () => { + assert.throws( + () => MemWal.create({ key: VALID_KEY }), + (err) => err instanceof TypeError && err.message.includes('"accountId" is required'), + ); + + assert.throws( + () => MemWal.create({ key: VALID_KEY, accountId: undefined }), + (err) => err instanceof TypeError && err.message.includes('"accountId" is required'), + ); +}); + +test("MemWal.create throws TypeError when accountId is empty, whitespace-only, or not a string", () => { + assert.throws( + () => MemWal.create({ key: VALID_KEY, accountId: "" }), + (err) => err instanceof TypeError && err.message.includes('"accountId" is required'), + ); + + assert.throws( + () => MemWal.create({ key: VALID_KEY, accountId: " " }), + (err) => err instanceof TypeError && err.message.includes('"accountId" is required'), + ); + + assert.throws( + () => MemWal.create({ key: VALID_KEY, accountId: 12345 }), + (err) => err instanceof TypeError && err.message.includes('"accountId" is required'), + ); +}); + +test("MemWal.create succeeds when valid accountId is provided", () => { + const client = MemWal.create({ + key: VALID_KEY, + accountId: VALID_ACCOUNT_ID, + }); + assert.ok(client); +}); + +test("MemWalManual.create throws TypeError when accountId is missing or empty", () => { + const manualConfigBase = { + key: VALID_KEY, + suiPrivateKey: "suiprivkey1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq0g24m4", + embeddingApiKey: "test-openai-key", + packageId: `0x${"03".repeat(32)}`, + registryId: `0x${"04".repeat(32)}`, + }; + + assert.throws( + () => MemWalManual.create({ ...manualConfigBase, accountId: undefined }), + (err) => err instanceof TypeError && err.message.includes('"accountId" is required'), + ); + + assert.throws( + () => MemWalManual.create({ ...manualConfigBase, accountId: "" }), + (err) => err instanceof TypeError && err.message.includes('"accountId" is required'), + ); +}); From d86535400148031ade2602a07d9ed03f927b884b Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Wed, 14 Oct 2026 20:51:39 +0700 Subject: [PATCH 52/70] fix(sdk): validate limit is a positive integer in recall, recallManual, and restore (#1180) * fix(sdk): validate limit is a positive integer in recall, recallManual, and restore - Validate limit is a positive integer before signing requests in recall(), recallManual(), and restore() - Reject limit <= 0, floats, NaN, and invalid types with TypeError - Document relayer cap of 100 on recall and [1, 100] clamp on restore in types JSDoc and api-reference.md - Add comprehensive test coverage in limit-validation.test.mjs Fixes #1086 * fix(sdk): validate limit in MemWalManual.restore recall, recallManual, and MemWal.restore already reject a non-positive or non-integer limit. MemWalManual.restore still forwarded it. Fixes #1086 --------- Co-authored-by: harrymove-ctrl <267878750+harrymove-ctrl@users.noreply.github.com> --- docs/sdk/api-reference.md | 4 +- packages/sdk/src/manual.ts | 12 +- packages/sdk/src/memwal.ts | 21 ++- packages/sdk/src/types.ts | 4 +- packages/sdk/test/limit-validation.test.mjs | 173 ++++++++++++++++++++ 5 files changed, 207 insertions(+), 7 deletions(-) create mode 100644 packages/sdk/test/limit-validation.test.mjs diff --git a/docs/sdk/api-reference.md b/docs/sdk/api-reference.md index 875e0660f..888f81507 100644 --- a/docs/sdk/api-reference.md +++ b/docs/sdk/api-reference.md @@ -122,7 +122,7 @@ Submit a bulk remember request and wait until every job reaches a terminal state Search for memories matching a natural language query, scoped to `owner + namespace`. - Preferred form: `recall({ query, limit?, topK?, namespace?, maxDistance?, sort?, scoringWeights? })` -- `limit` defaults to `10`; `topK` is an alias and wins when both are set +- `limit` defaults to `10` (relayer caps at `100`); `topK` is an alias and wins when both are set - Legacy positional forms still work: `recall(query)`, `recall(query, limit)`, `recall(query, limit, namespace)`, and `recall(query, options)`. `recall(query, namespace)` throws `TypeError`; namespace is not a valid second argument. - `maxDistance` filters weak matches client-side by dropping results where `distance >= maxDistance` - `sort` picks the ordering: `"relevance"` (default) or `"recent"` for newest-among-matches @@ -272,7 +272,7 @@ Use `analyzeAndWait(text, namespace?, opts?)` to wait for every extracted fact j Rebuild missing indexed entries for one namespace from Walrus. Incremental — only re-indexes blobs that aren't already in the local database. -- `limit` defaults to `10` +- `limit` defaults to `10` (relayer clamps to `[1, 100]`) **Returns:** diff --git a/packages/sdk/src/manual.ts b/packages/sdk/src/manual.ts index d17b3ac69..48ef4ec87 100644 --- a/packages/sdk/src/manual.ts +++ b/packages/sdk/src/manual.ts @@ -398,7 +398,12 @@ export class MemWalManual { limitOrOptions, namespace, ); - const limit = options.limit ?? 10; + const limit = options.limit !== undefined ? options.limit : 10; + if (typeof limit !== "number" || !Number.isInteger(limit) || limit <= 0) { + throw new TypeError( + `limit must be a positive integer, got ${limit}`, + ); + } const ns = options.namespace ?? this.namespace; // Step 1: Embed query @@ -942,6 +947,11 @@ export class MemWalManual { * @returns RestoreResult with count of restored entries */ async restore(namespace: string, limit: number = 10): Promise { + if (typeof limit !== "number" || !Number.isInteger(limit) || limit <= 0) { + throw new TypeError( + `limit must be a positive integer, got ${limit}`, + ); + } const result = await this.signedRequest("POST", "/api/restore", { namespace, limit, diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index 5ba1cc26e..16e46ceaf 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -1149,7 +1149,13 @@ export class MemWal { namespace, ); validateMaxTokens(options.maxTokens); - const limit = options.topK ?? options.limit ?? 10; + const rawLimit = options.topK !== undefined ? options.topK : options.limit; + const limit = rawLimit !== undefined ? rawLimit : 10; + if (typeof limit !== "number" || !Number.isInteger(limit) || limit <= 0) { + throw new TypeError( + `limit must be a positive integer, got ${limit}`, + ); + } const resolvedNamespace = options.namespace ?? this.namespace; { @@ -1283,12 +1289,18 @@ export class MemWal { * ``` */ async recallManual(opts: RecallManualOptions): Promise { + const limit = opts.limit !== undefined ? opts.limit : 10; + if (typeof limit !== "number" || !Number.isInteger(limit) || limit <= 0) { + throw new TypeError( + `limit must be a positive integer, got ${limit}`, + ); + } return this.signedRequest( "POST", "/api/recall/manual", { vector: opts.vector, - limit: opts.limit ?? 10, + limit, namespace: opts.namespace ?? this.namespace, scoring_weights: scoringWeightsToWire(opts.scoringWeights), }, @@ -1410,6 +1422,11 @@ export class MemWal { * ``` */ async restore(namespace: string, limit: number = 10): Promise { + if (typeof limit !== "number" || !Number.isInteger(limit) || limit <= 0) { + throw new TypeError( + `limit must be a positive integer, got ${limit}`, + ); + } const result = await this.signedRequest( "POST", "/api/restore", diff --git a/packages/sdk/src/types.ts b/packages/sdk/src/types.ts index 4b182227f..4f5cc5e3c 100644 --- a/packages/sdk/src/types.ts +++ b/packages/sdk/src/types.ts @@ -150,7 +150,7 @@ export type TruncationStrategy = "high-relevance-only" | "per-hit-cap" | "drop-t /** Options for recall(). */ export interface RecallOptions { - /** Max number of nearest memories to request from the relayer (default: 10). */ + /** Max number of nearest memories to request from the relayer (default: 10, relayer caps at 100). Must be a positive integer. */ limit?: number; /** Alias for limit, useful when describing top-K search behaviour. */ topK?: number; @@ -445,7 +445,7 @@ export interface RememberManualResult { export interface RecallManualOptions { /** Pre-computed query embedding vector */ vector: number[]; - /** Max number of results (default: 10) */ + /** Max number of results (default: 10, relayer caps at 100). Must be a positive integer. */ limit?: number; /** Namespace (default: config namespace or "default") */ namespace?: string; diff --git a/packages/sdk/test/limit-validation.test.mjs b/packages/sdk/test/limit-validation.test.mjs new file mode 100644 index 000000000..8a9047924 --- /dev/null +++ b/packages/sdk/test/limit-validation.test.mjs @@ -0,0 +1,173 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { MemWal } from "../dist/memwal.js"; +import { MemWalManual } from "../dist/manual.js"; + +function client() { + return MemWal.create({ + key: new Uint8Array(32).fill(1), + accountId: "0x1", + serverUrl: "https://relayer.example", + }); +} + +function manualClient() { + return MemWalManual.create({ + key: new Uint8Array(32).fill(1), + walletSigner: { + address: "0x2", + signAndExecuteTransaction: async () => ({ digest: "0xtx" }), + signPersonalMessage: async () => ({ signature: "sig" }), + }, + suiClient: {}, + embeddingApiKey: "test", + packageId: `0x${"33".repeat(32)}`, + accountId: "0x1", + registryId: "0xregistry", + sealServerConfigs: [{ objectId: "0xserver", weight: 1 }], + }); +} + +test("MemWal.recall() rejects non-positive or non-integer limit before signing", async () => { + const memwal = client(); + let requests = 0; + memwal.signedRequest = async () => { + requests += 1; + return { results: [], total: 0 }; + }; + + const invalidLimits = [0, -1, 2.5, NaN, -0, "5", null]; + for (const bad of invalidLimits) { + await assert.rejects( + () => memwal.recall({ query: "user", limit: bad }), + { + name: "TypeError", + message: /limit must be a positive integer/, + }, + ); + await assert.rejects( + () => memwal.recall({ query: "user", topK: bad }), + { + name: "TypeError", + message: /limit must be a positive integer/, + }, + ); + } + + const invalidPositionalLimits = [0, -1, 2.5, NaN, -0]; + for (const bad of invalidPositionalLimits) { + await assert.rejects( + () => memwal.recall("user", bad), + { + name: "TypeError", + message: /limit must be a positive integer/, + }, + ); + } + assert.equal(requests, 0); +}); + +test("MemWal.recallManual() rejects non-positive or non-integer limit before signing", async () => { + const memwal = client(); + let requests = 0; + memwal.signedRequest = async () => { + requests += 1; + return { results: [], total: 0 }; + }; + + const invalidLimits = [0, -1, 2.5, NaN, "10", null]; + for (const bad of invalidLimits) { + await assert.rejects( + () => memwal.recallManual({ vector: [0.1, 0.2], limit: bad }), + { + name: "TypeError", + message: /limit must be a positive integer/, + }, + ); + } + assert.equal(requests, 0); +}); + +test("MemWal.restore() rejects non-positive or non-integer limit before signing", async () => { + const memwal = client(); + let requests = 0; + memwal.signedRequest = async () => { + requests += 1; + return { restored: 0, skipped: 0, total: 0, failed: 0, namespace: "demo", owner: "0x1" }; + }; + + const invalidLimits = [0, -1, 2.5, NaN, "10", null]; + for (const bad of invalidLimits) { + await assert.rejects( + () => memwal.restore("demo", bad), + { + name: "TypeError", + message: /limit must be a positive integer/, + }, + ); + } + assert.equal(requests, 0); +}); + +test("MemWalManual.recallManual() rejects non-positive or non-integer limit before signing", async () => { + const manual = manualClient(); + let requests = 0; + manual.signedRequest = async () => { + requests += 1; + return { results: [], total: 0 }; + }; + manual.embed = async () => [0.1, 0.2]; + + const invalidLimits = [0, -1, 2.5, NaN, "5", null]; + for (const bad of invalidLimits) { + await assert.rejects( + () => manual.recallManual("query", { limit: bad }), + { + name: "TypeError", + message: /limit must be a positive integer/, + }, + ); + } + + const invalidPositionalLimits = [0, -1, 2.5, NaN]; + for (const bad of invalidPositionalLimits) { + await assert.rejects( + () => manual.recallManual("query", bad), + { + name: "TypeError", + message: /limit must be a positive integer/, + }, + ); + } + assert.equal(requests, 0); +}); + +test("MemWalManual.restore() rejects non-positive or non-integer limit before signing", async () => { + const manual = manualClient(); + let requests = 0; + manual.signedRequest = async () => { + requests += 1; + return { + restored: 0, + skipped: 0, + failed: 0, + total: 0, + namespace: "default", + owner: "0x1", + truncated: false, + }; + }; + + const invalidLimits = [0, -1, 2.5, NaN, "10", null]; + for (const bad of invalidLimits) { + await assert.rejects( + () => manual.restore("default", bad), + { + name: "TypeError", + message: /limit must be a positive integer/, + }, + ); + } + assert.equal(requests, 0); +}); From aa582ee17ca735abdbeb2a911398cb4a49d2e508 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Wed, 14 Oct 2026 20:54:39 +0700 Subject: [PATCH 53/70] fix(sdk): attach SEAL session and timeout in MemWalManual.restore (Fixes #1137) (#1190) * fix(sdk): attach SEAL session and timeout in MemWalManual.restore (Fixes #1137) * test(sdk): drop the unused restore timeout import The tests name the constant in their titles but never reference the binding, which CodeQL flagged on the restore PR. Fixes #1137 * fix(sdk): keep the manual request deadline armed across the body read fetch settles when the headers arrive. Disposing the timer before res.text() or res.json() let a stalled body hang. The deadline now covers that read and is cleared afterwards. Fixes #1137 * fix(sdk): sign manual restore sessions with the delegate key An owner wallet session prompts on every refresh and is broader than the delegate this client already holds. restore() now builds the SEAL session from that delegate key. The timeout test drives restore() and stalls the body after headers. --------- Co-authored-by: harrymove-ctrl <267878750+harrymove-ctrl@users.noreply.github.com> --- packages/sdk/src/manual.ts | 208 +++++++++++++----- packages/sdk/src/memwal.ts | 6 +- packages/sdk/src/utils.ts | 57 +++++ packages/sdk/test/manual-restore.test.mjs | 116 ++++++++++ .../sdk/test/private-key-formats.test.mjs | 1 + 5 files changed, 333 insertions(+), 55 deletions(-) create mode 100644 packages/sdk/test/manual-restore.test.mjs diff --git a/packages/sdk/src/manual.ts b/packages/sdk/src/manual.ts index 48ef4ec87..506baabaf 100644 --- a/packages/sdk/src/manual.ts +++ b/packages/sdk/src/manual.ts @@ -9,7 +9,7 @@ * * @example * ```typescript - * import { MemWalManual } from "@mysten-incubation/memwal" + * import { MemWalManual } from "@mysten-incubation/memwal/manual" * * const memwal = MemWalManual.create({ * key: process.env.MEMWAL_DELEGATE_KEY!, // Ed25519 delegate key @@ -52,6 +52,9 @@ import { scoringWeightsToWire, fetchWithDeadline, DEFAULT_PREFLIGHT_TIMEOUT_MS, + RESTORE_REQUEST_TIMEOUT_MS, + deadlineSignal, + requestTimeoutError, } from "./utils.js"; import { assertCompatibleRelayer, compatibilityErrorFromStatus } from "./compatibility.js"; import { resolveLimitOrOptions } from "./recall-args.js"; @@ -152,6 +155,9 @@ function bytesToBase64(bytes: Uint8Array): string { // MemWalManual Client // ============================================================ +const SEAL_SESSION_TTL_MIN = 5; +const SEAL_SESSION_SAFETY_MARGIN_MS = 30_000; + export class MemWalManual { private delegatePrivateKey: Uint8Array; private delegatePublicKey: Uint8Array | null = null; @@ -163,6 +169,8 @@ export class MemWalManual { private compatibilityPromise: Promise | null = null; /** Per-round-trip budget for the compatibility preflight (WALM-598). */ private preflightTimeoutMs: number; + private sessionCache: { bytes: string; expiresAt: number } | null = null; + private sessionBuildPromise: Promise | null = null; // Lazily initialized heavy clients (typed as any to avoid peer dep compile errors) private _suiClient: any = null; @@ -235,6 +243,74 @@ export class MemWalManual { } this.relayerVersionMetadata = null; this.compatibilityPromise = null; + this.sessionCache = null; + this.sessionBuildPromise = null; + } + + /** + * Build an ephemeral SEAL SessionKey, sign its personal message, and export + * it as a base64-encoded JSON envelope suitable for `x-seal-session`. + * Memoised until 30s before expiration (5m TTL). + */ + async buildSealSession(): Promise { + // Fast path: cached session still fresh. + if (this.sessionCache && Date.now() < this.sessionCache.expiresAt) { + return this.sessionCache.bytes; + } + // Single-flight: concurrent requests share one build. + if (this.sessionBuildPromise) return this.sessionBuildPromise; + + this.sessionBuildPromise = this.buildSealSessionInner().finally(() => { + this.sessionBuildPromise = null; + }); + return this.sessionBuildPromise; + } + + private async buildSealSessionInner(): Promise { + // Same key MemWal uses. An owner wallet session prompts on every + // refresh and is broader than the delegate this client already holds. + // Dynamic imports: @mysten/sui and @mysten/seal are optional peers. + // Server-mode MemWal must still load when they are not installed. + const { Ed25519Keypair } = await import("@mysten/sui/keypairs/ed25519"); + const keypair = Ed25519Keypair.fromSecretKey(this.delegatePrivateKey); + const suiClient = await this.getSuiClient(); + const { SessionKey } = await import("@mysten/seal"); + + const session = await SessionKey.create({ + address: keypair.getPublicKey().toSuiAddress(), + packageId: this.config.packageId, + ttlMin: SEAL_SESSION_TTL_MIN, + signer: keypair, + suiClient, + }); + + const personalMessage = session.getPersonalMessage(); + const signResult = await keypair.signPersonalMessage(personalMessage); + await session.setPersonalMessageSignature(signResult.signature); + + const exported = session.export(); + const jsonStr = JSON.stringify({ + address: exported.address, + packageId: exported.packageId, + mvrName: exported.mvrName, + creationTimeMs: exported.creationTimeMs, + ttlMin: exported.ttlMin, + personalMessageSignature: exported.personalMessageSignature, + sessionKey: exported.sessionKey, + }); + const bytes = + typeof btoa === "function" + ? btoa(jsonStr) + : Buffer.from(jsonStr, "utf8").toString("base64"); + + this.sessionCache = { + bytes, + expiresAt: + Date.now() + + SEAL_SESSION_TTL_MIN * 60_000 - + SEAL_SESSION_SAFETY_MARGIN_MS, + }; + return bytes; } /** @@ -867,7 +943,12 @@ export class MemWalManual { * * Headers sent: x-public-key, x-signature, x-timestamp, x-nonce, x-account-id. */ - private async signedRequest(method: string, path: string, body: object): Promise { + private async signedRequest( + method: string, + path: string, + body: object, + options: { timeoutMs?: number; signal?: AbortSignal; includeSealSession?: boolean } = {}, + ): Promise { await this.ensureCompatibleRelayer(); const ed = await import("@noble/ed25519"); @@ -886,53 +967,68 @@ export class MemWalManual { const signature = await ed.signAsync(msgBytes, this.delegatePrivateKey); const publicKey = await this.getDelegatePublicKey(); + const headers: Record = { + "Content-Type": "application/json", + "x-public-key": bytesToHex(publicKey), + "x-signature": bytesToHex(signature), + "x-timestamp": timestamp, + "x-nonce": nonce, + "x-account-id": this.config.accountId, + }; + + if (options.includeSealSession) { + headers["x-seal-session"] = await this.buildSealSession(); + } + const url = `${this.serverUrl}${path}`; - const res = await fetch(url, { - method, - headers: { - "Content-Type": "application/json", - "x-public-key": bytesToHex(publicKey), - "x-signature": bytesToHex(signature), - "x-timestamp": timestamp, - "x-nonce": nonce, - "x-account-id": this.config.accountId, - }, - body: bodyStr, - }); + const deadline = options.timeoutMs ? deadlineSignal(options.timeoutMs, options.signal) : undefined; + try { + const res = await fetch(url, { + method, + headers, + body: bodyStr, + signal: deadline?.signal ?? options.signal, + }); - if (!res.ok) { - // LOW-26: sanitize server error bodies before re-throwing. - const raw = await res.text(); - const compatibilityError = compatibilityErrorFromStatus(res.status, raw); - if (compatibilityError) throw compatibilityError; - - // Surface a stale-timestamp rejection (401 + x-auth-error) as an - // actionable clock-drift error rather than an opaque server error. - const clockDriftError = clockDriftErrorFromResponse(res); - if (clockDriftError) throw clockDriftError; - - const { message: sanitized, serverCode } = sanitizeServerError( - res.status, - raw, - res.headers.get("x-auth-error"), - ); - const err = new Error(sanitized) as Error & { - status?: number; - serverCode?: string; - retryAfterSeconds?: number; - cause?: string; - }; - err.status = res.status; - if (serverCode) err.serverCode = serverCode; - const retryAfter = Number(res.headers.get("retry-after")); - if (Number.isFinite(retryAfter) && retryAfter > 0) { - err.retryAfterSeconds = retryAfter; + if (!res.ok) { + const raw = await res.text(); + const compatibilityError = compatibilityErrorFromStatus(res.status, raw); + if (compatibilityError) throw compatibilityError; + + const clockDriftError = clockDriftErrorFromResponse(res); + if (clockDriftError) throw clockDriftError; + + const { message: sanitized, serverCode } = sanitizeServerError( + res.status, + raw, + res.headers.get("x-auth-error"), + ); + const err = new Error(sanitized) as Error & { + status?: number; + serverCode?: string; + retryAfterSeconds?: number; + cause?: string; + }; + err.status = res.status; + if (serverCode) err.serverCode = serverCode; + const retryAfter = Number(res.headers.get("retry-after")); + if (Number.isFinite(retryAfter) && retryAfter > 0) { + err.retryAfterSeconds = retryAfter; + } + err.cause = raw; + throw err; + } + + return await res.json() as T; + } catch (err) { + const status = err && typeof err === "object" && "status" in err ? err.status : undefined; + if (deadline?.timedOut() && typeof status !== "number") { + throw requestTimeoutError(method, url, options.timeoutMs!); } - err.cause = raw; throw err; + } finally { + deadline?.dispose(); } - - return res.json() as Promise; } // ============================================================ @@ -940,8 +1036,12 @@ export class MemWalManual { // ============================================================ /** - * Restore a namespace — server downloads all blobs from Walrus, - * decrypts with delegate key, re-embeds, and re-indexes. + * Restore a namespace. The server downloads the blobs, decrypts them with + * the delegate key, and re-indexes them. + * + * Sends `x-seal-session` built from the delegate key, not the owner + * wallet. That session lets the relayer decrypt for this restore only, + * for the usual short TTL. * * @param namespace - Namespace to restore * @returns RestoreResult with count of restored entries @@ -952,10 +1052,18 @@ export class MemWalManual { `limit must be a positive integer, got ${limit}`, ); } - const result = await this.signedRequest("POST", "/api/restore", { - namespace, - limit, - }); + const result = await this.signedRequest( + "POST", + "/api/restore", + { + namespace, + limit, + }, + { + includeSealSession: true, + timeoutMs: RESTORE_REQUEST_TIMEOUT_MS, + }, + ); // Relayers older than WALM-319 omit `truncated` entirely — treat // "not present" as "not known to be truncated" rather than drop // the field or require every relayer version to send it. diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index 16e46ceaf..59225fa2a 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -76,6 +76,7 @@ import { derivedBulkIdempotencyKey, DEFAULT_PREFLIGHT_TIMEOUT_MS, DEFAULT_RECALL_TIMEOUT_MS, + RESTORE_REQUEST_TIMEOUT_MS, } from "./utils.js"; import { assertCompatibleRelayer, @@ -213,11 +214,6 @@ function destroyedClientError(): Error { ); } -/** `POST /api/restore` bounds itself at 55s server-side and answers with an - * error rather than going quiet, so the client must outlast that or it would - * abandon a response already on its way. */ -const RESTORE_REQUEST_TIMEOUT_MS = 60_000; - /** `POST /api/analyze` runs the extractor LLM inline before it accepts, so it * is the one write that legitimately outruns the default. */ const ANALYZE_REQUEST_TIMEOUT_MS = 60_000; diff --git a/packages/sdk/src/utils.ts b/packages/sdk/src/utils.ts index 636258423..8b7f9dd57 100644 --- a/packages/sdk/src/utils.ts +++ b/packages/sdk/src/utils.ts @@ -609,3 +609,60 @@ export async function delegateKeyToPublicKey(privateKeyHex: string): Promise boolean; dispose: () => void } { + const controller = new AbortController(); + let expired = false; + + const onCallerAbort = () => controller.abort(caller?.reason); + if (caller) { + if (caller.aborted) controller.abort(caller.reason); + else caller.addEventListener("abort", onCallerAbort, { once: true }); + } + + const timer = setTimeout(() => { + expired = true; + controller.abort(); + }, ms); + + return { + signal: controller.signal, + timedOut: () => expired, + dispose: () => { + clearTimeout(timer); + caller?.removeEventListener("abort", onCallerAbort); + }, + }; +} + +/** + * The error a request deadline produces. + */ +export function requestTimeoutError(method: string, path: string, ms: number): Error { + const err = new Error( + `Walrus Memory request timed out after ${ms}ms (${method} ${path}). The relayer ` + + `accepted the connection but did not answer in time.`, + ); + err.name = "MemWalRequestTimeout"; + (err as Error & { status?: number }).status = 504; + return err; +} + diff --git a/packages/sdk/test/manual-restore.test.mjs b/packages/sdk/test/manual-restore.test.mjs new file mode 100644 index 000000000..6ad16e534 --- /dev/null +++ b/packages/sdk/test/manual-restore.test.mjs @@ -0,0 +1,116 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { MemWalManual } from "../dist/manual.js"; +import { RESTORE_REQUEST_TIMEOUT_MS } from "../dist/utils.js"; + + +const originalFetch = globalThis.fetch; + +test.afterEach(() => { + globalThis.fetch = originalFetch; +}); + +const ACCOUNT_ID = `0x${"aa".repeat(32)}`; +const PACKAGE_ID = `0x${"33".repeat(32)}`; +const REGISTRY_ID = `0x${"bb".repeat(32)}`; +const DELEGATE_SIGNER = `0x${"22".repeat(32)}`; + +function createManualClient() { + return MemWalManual.create({ + key: new Uint8Array(32).fill(1), + walletSigner: { + address: DELEGATE_SIGNER, + signAndExecuteTransaction: async () => ({ digest: "0xtx" }), + signPersonalMessage: async () => ({ signature: "test-sig" }), + }, + embeddingApiKey: "test-key", + packageId: PACKAGE_ID, + accountId: ACCOUNT_ID, + registryId: REGISTRY_ID, + serverUrl: "https://relayer.example", + }); +} + +test("MemWalManual.restore() attaches x-seal-session header and applies RESTORE_REQUEST_TIMEOUT_MS", async () => { + let capturedHeaders = null; + let hadAbortSignal = false; + + globalThis.fetch = async (url, init = {}) => { + const path = new URL(url).pathname; + if (path === "/version") { + return Response.json({ + apiVersion: "1.0.0", + relayerVersion: "0.1.8", + minSupportedSdk: { typescript: "0.0.4" }, + }); + } + if (path === "/api/restore" && init.method === "POST") { + capturedHeaders = init.headers; + hadAbortSignal = Boolean(init.signal); + return Response.json({ + restored: 2, + skipped: 0, + failed: 0, + total: 2, + namespace: "default", + owner: "0x1", + truncated: false, + }); + } + throw new Error(`Unexpected request: ${path}`); + }; + + const client = createManualClient(); + client.buildSealSession = async () => "mock-seal-session-token"; + + const result = await client.restore("default", 10); + + assert.equal(result.restored, 2); + assert.ok(capturedHeaders, "Headers must be captured"); + assert.equal(capturedHeaders["x-seal-session"], "mock-seal-session-token"); + assert.ok(hadAbortSignal, "Must pass an abort signal to bound the request"); +}); + +test("restore() times out when the body stalls after headers", async (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }); + globalThis.fetch = async (url, init = {}) => { + const path = new URL(url).pathname; + if (path === "/version") { + return Response.json({ + apiVersion: "1.0.0", + relayerVersion: "0.1.8", + minSupportedSdk: { typescript: "0.0.4" }, + }); + } + if (path === "/api/restore") { + started(); + return { + ok: true, + status: 200, + headers: new Headers(), + json: () => new Promise((_, reject) => { + const abort = () => { + const err = new Error("The operation was aborted"); + err.name = "AbortError"; + reject(err); + }; + if (init.signal?.aborted) abort(); + else init.signal?.addEventListener("abort", abort, { once: true }); + }), + }; + } + throw new Error(`Unexpected request: ${path}`); + }; + + const client = createManualClient(); + client.buildSealSession = async () => "mock-seal-session-token"; + let started = () => {}; + const bodyStalled = new Promise((resolve) => { started = resolve; }); + const pending = client.restore("default", 10).catch((err) => err); + await bodyStalled; + t.mock.timers.tick(RESTORE_REQUEST_TIMEOUT_MS); + const err = await pending; + assert.equal(err.name, "MemWalRequestTimeout"); + assert.equal(err.status, 504); +}); diff --git a/packages/sdk/test/private-key-formats.test.mjs b/packages/sdk/test/private-key-formats.test.mjs index 85baaa50a..7ea11e94f 100644 --- a/packages/sdk/test/private-key-formats.test.mjs +++ b/packages/sdk/test/private-key-formats.test.mjs @@ -82,6 +82,7 @@ test("MemWalManual signs with the same delegate key whether given hex or bech32" serverUrl: "https://relayer.example", sealServerConfigs: [{ objectId: "0xserver", weight: 1 }], }); + manual.buildSealSession = async () => "test-session"; let captured = null; const originalFetch = globalThis.fetch; From 6f20ea83ce4b89c75fd6b1b86bcf058f3a6bef68 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Wed, 14 Oct 2026 20:56:11 +0700 Subject: [PATCH 54/70] fix(python-sdk): expose created_at and score in RecallMemory and RecallManualHit (#1178) - Add score and created_at fields to RecallMemory dataclass - Add created_at and importance fields to RecallManualHit dataclass - Forward fields from API response in MemWal.recall and MemWal.recall_manual - Populate score and created_at in MemWalMock.recall - Add regression tests in test_client.py and test_mock.py Fixes #1063 Fixes #1160 --- packages/python-sdk-memwal/memwal/client.py | 9 ++- packages/python-sdk-memwal/memwal/mock.py | 8 ++- packages/python-sdk-memwal/memwal/types.py | 4 ++ .../python-sdk-memwal/tests/test_client.py | 69 +++++++++++++++++++ packages/python-sdk-memwal/tests/test_mock.py | 4 ++ 5 files changed, 92 insertions(+), 2 deletions(-) diff --git a/packages/python-sdk-memwal/memwal/client.py b/packages/python-sdk-memwal/memwal/client.py index df910d5d2..4675395cf 100644 --- a/packages/python-sdk-memwal/memwal/client.py +++ b/packages/python-sdk-memwal/memwal/client.py @@ -1055,6 +1055,8 @@ async def recall( blob_id=m["blob_id"], text=m["text"], distance=m["distance"], + score=m.get("score"), + created_at=m.get("created_at"), ) for m in data.get("results", []) ] @@ -1453,7 +1455,12 @@ async def recall_manual(self, opts: RecallManualOptions) -> RecallManualResult: include_seal_session=False, ) hits = [ - RecallManualHit(blob_id=h["blob_id"], distance=h["distance"]) + RecallManualHit( + blob_id=h["blob_id"], + distance=h["distance"], + created_at=h.get("created_at"), + importance=h.get("importance"), + ) for h in data.get("results", []) ] return RecallManualResult(results=hits, total=data.get("total", len(hits))) diff --git a/packages/python-sdk-memwal/memwal/mock.py b/packages/python-sdk-memwal/memwal/mock.py index 1b5996ae0..2f7aeed53 100644 --- a/packages/python-sdk-memwal/memwal/mock.py +++ b/packages/python-sdk-memwal/memwal/mock.py @@ -283,7 +283,13 @@ async def recall( if max_distance is not None: page = [item for item in page if item[1] < max_distance] results = [ - RecallMemory(blob_id=memory.blob_id, text=memory.text, distance=score) + RecallMemory( + blob_id=memory.blob_id, + text=memory.text, + distance=score, + score=1.0 - score, + created_at=_mock_timestamp(memory.sequence), + ) for memory, score in page ] return RecallResult(results=results, total=len(ranked[:limit])) diff --git a/packages/python-sdk-memwal/memwal/types.py b/packages/python-sdk-memwal/memwal/types.py index cadaed9be..b9813c8fa 100644 --- a/packages/python-sdk-memwal/memwal/types.py +++ b/packages/python-sdk-memwal/memwal/types.py @@ -88,6 +88,8 @@ class RecallMemory: blob_id: str text: str distance: float + score: Optional[float] = None + created_at: Optional[str] = None @dataclass @@ -328,6 +330,8 @@ class RecallManualHit: blob_id: str distance: float + created_at: Optional[str] = None + importance: Optional[float] = None @dataclass diff --git a/packages/python-sdk-memwal/tests/test_client.py b/packages/python-sdk-memwal/tests/test_client.py index f4f9420a0..4044697b7 100644 --- a/packages/python-sdk-memwal/tests/test_client.py +++ b/packages/python-sdk-memwal/tests/test_client.py @@ -841,6 +841,40 @@ async def test_recall_surfaces_dropped_count(self, memwal_client: MemWal) -> Non assert result.results == [] assert result.dropped_count == 3 + @respx.mock + async def test_recall_preserves_score_and_created_at( + self, memwal_client: MemWal + ) -> None: + mock_seal_session_prereqs() + respx.post(f"{_TEST_SERVER}/api/recall").mock( + return_value=httpx.Response( + 200, + json={ + "results": [ + { + "blob_id": "b1", + "text": "Fact with metadata", + "distance": 0.1, + "score": 0.95, + "created_at": "2026-03-30T12:00:00Z", + }, + { + "blob_id": "b2", + "text": "Legacy fact", + "distance": 0.3, + }, + ], + "total": 2, + }, + ) + ) + result = await memwal_client.recall("test") + assert len(result.results) == 2 + assert result.results[0].score == 0.95 + assert result.results[0].created_at == "2026-03-30T12:00:00Z" + assert result.results[1].score is None + assert result.results[1].created_at is None + @respx.mock async def test_accepts_recall_params_object( self, memwal_client: MemWal @@ -1475,6 +1509,41 @@ async def test_recall_manual(self, memwal_client: MemWal) -> None: assert len(result.results) == 1 assert result.results[0].blob_id == "b1" + @respx.mock + async def test_recall_manual_preserves_created_at_and_importance( + self, memwal_client: MemWal + ) -> None: + _mock_version() + respx.post(f"{_TEST_SERVER}/api/recall/manual").mock( + return_value=httpx.Response( + 200, + json={ + "results": [ + { + "blob_id": "b1", + "distance": 0.15, + "created_at": "2026-03-30T12:00:00Z", + "importance": 0.85, + }, + { + "blob_id": "b2", + "distance": 0.4, + }, + ], + "total": 2, + }, + ) + ) + + opts = RecallManualOptions(vector=[0.1, 0.2, 0.3], limit=5) + result = await memwal_client.recall_manual(opts) + + assert len(result.results) == 2 + assert result.results[0].created_at == "2026-03-30T12:00:00Z" + assert result.results[0].importance == 0.85 + assert result.results[1].created_at is None + assert result.results[1].importance is None + @respx.mock async def test_recall_manual_forwards_scoring_weights(self, memwal_client: MemWal) -> None: _mock_version() diff --git a/packages/python-sdk-memwal/tests/test_mock.py b/packages/python-sdk-memwal/tests/test_mock.py index f094528ce..a01e2b256 100644 --- a/packages/python-sdk-memwal/tests/test_mock.py +++ b/packages/python-sdk-memwal/tests/test_mock.py @@ -39,7 +39,11 @@ def reject_network(*args, **kwargs): "I drink tea at night", ] assert recalled.results[0].distance == 0 + assert recalled.results[0].score == 1.0 + assert recalled.results[0].created_at == "2026-01-01T00:00:01.000Z" assert recalled.results[1].distance == 1 + assert recalled.results[1].score == 0.0 + assert recalled.results[1].created_at == "2026-01-01T00:00:02.000Z" @pytest.mark.asyncio From de6d6b75bd3eb736c45d9af05e16d43e35286ce7 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Wed, 14 Oct 2026 20:56:28 +0700 Subject: [PATCH 55/70] fix(sdk): suppress recall and auto-save on tool execution steps (Fixes #1079) (#1184) * fix(sdk): suppress recall and auto-save on tool execution steps (Fixes #1079) * fix(sdk): keep recalled memory on tool steps without a second recall Auto-save still runs only on a user turn. A later tool step re-injects the cached recall for that same user message, so the answer step does not lose the memory context. Fixes #1079 * fix(sdk): recall when a tool step has no cached memory A matching cache is still reused and does not call recall again. A miss, including the first call on a fresh withMemWal, recalls once and stores that result. Fixes #1079 * fix(sdk): cache an empty recall so tool steps do not call again A user turn with no matching memories used to clear the cache. The next tool step then called /api/recall again. An empty result is now remembered for that turn. Fixes #1079 * fix(sdk): recall again on the next user turn Tool steps still reuse a cached recall, including an empty one. A later user message recalls even when the text matches the previous turn. --- packages/sdk/src/ai/middleware.ts | 68 ++++-- .../sdk/test/withMemWal-tool-steps.test.mjs | 229 ++++++++++++++++++ 2 files changed, 283 insertions(+), 14 deletions(-) create mode 100644 packages/sdk/test/withMemWal-tool-steps.test.mjs diff --git a/packages/sdk/src/ai/middleware.ts b/packages/sdk/src/ai/middleware.ts index 76fde61b2..805fefa4e 100644 --- a/packages/sdk/src/ai/middleware.ts +++ b/packages/sdk/src/ai/middleware.ts @@ -95,6 +95,9 @@ export function withMemWal( const pendingSaves = new Set>(); const saveFailures: string[] = []; let notingFailure = 0; + // Tool steps rebuild the prompt without the memory injected on step 1. + // Keep that recall and put it back, instead of calling recall again. + let cachedRecall: { key: string; context: string } | null = null; function saveInBackground(userMessage: string): void { const savePromise = memwal @@ -126,29 +129,45 @@ export function withMemWal( // ============================================================ // BEFORE: Search memories + inject into prompt // ============================================================ - transformParams: async ({ params }: any) => { + transformParams: async ({ params }: { params: { prompt?: unknown } }) => { try { + const turnKey = lastUserTurnKey(params.prompt); + if (!turnKey) return params; + + // Tool steps reuse this turn, including a recall that + // matched nothing. A user message always recalls again, so + // a later fact is not stuck behind the first result. + if ( + cachedRecall && + cachedRecall.key === turnKey && + !isPromptEndingWithUser(params.prompt) + ) { + if (!cachedRecall.context) return params; + return { + ...params, + prompt: injectMemoryContext(params.prompt, cachedRecall.context), + }; + } + const lastUserMessage = findLastUserMessage(params.prompt); if (!lastUserMessage) return params; const recallResult = await memwal.recall(lastUserMessage, maxMemories); - - // Filter by minimum relevance (distance < 1 - minRelevance) const relevant = recallResult.results.filter( (m: RecallMemory) => (1 - m.distance) >= minRelevance ); - - if (relevant.length === 0) return params; + if (relevant.length === 0) { + cachedRecall = { key: turnKey, context: "" }; + return params; + } const memoryContext = formatUntrustedMemories(relevant); - const enrichedPrompt = injectMemoryContext( - params.prompt, - memoryContext - ); - + cachedRecall = { key: turnKey, context: memoryContext }; log(`🔍 Found ${relevant.length} relevant memories`); - - return { ...params, prompt: enrichedPrompt }; + return { + ...params, + prompt: injectMemoryContext(params.prompt, memoryContext), + }; } catch (error) { console.warn("[Walrus Memory] Recall failed during transformParams:", error); try { @@ -166,7 +185,9 @@ export function withMemWal( wrapGenerate: async ({ doGenerate, params }: any) => { const result = await doGenerate(); - if (autoSave) { + // Only auto-save on the first step of a user turn, not on + // intermediate tool steps where the last message is tool/assistant. + if (autoSave && isPromptEndingWithUser(params.prompt)) { const userMessage = findLastUserMessage(params.prompt); if (userMessage) { saveInBackground(userMessage); @@ -180,7 +201,9 @@ export function withMemWal( wrapStream: async ({ doStream, params }: any) => { const result = await doStream(); - if (autoSave) { + // Only auto-save on the first step of a user turn, not on + // intermediate tool steps where the last message is tool/assistant. + if (autoSave && isPromptEndingWithUser(params.prompt)) { const userMessage = findLastUserMessage(params.prompt); if (userMessage) { saveInBackground(userMessage); @@ -236,6 +259,23 @@ function findLastUserMessage( return null; } +function isPromptEndingWithUser(prompt: unknown): boolean { + if (!Array.isArray(prompt) || prompt.length === 0) return false; + const lastMsg = prompt[prompt.length - 1] as { role?: string; content?: unknown }; + return lastMsg?.role === "user"; +} + +function lastUserTurnKey(prompt: unknown): string | null { + if (!Array.isArray(prompt)) return null; + for (let i = prompt.length - 1; i >= 0; i--) { + const msg = prompt[i] as { role?: string }; + if (msg?.role !== "user") continue; + const text = findLastUserMessage(prompt); + return text ? `${i}\0${text}` : null; + } + return null; +} + export function injectMemoryContext( prompt: unknown, memoryContext: string diff --git a/packages/sdk/test/withMemWal-tool-steps.test.mjs b/packages/sdk/test/withMemWal-tool-steps.test.mjs new file mode 100644 index 000000000..eb7238c2b --- /dev/null +++ b/packages/sdk/test/withMemWal-tool-steps.test.mjs @@ -0,0 +1,229 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { MemWal } from "../dist/memwal.js"; +import { withMemWal } from "../dist/ai/middleware.js"; + +const TEST_KEY = "11".repeat(32); +const TEST_ACCOUNT_ID = "0x1"; + +const generatedPrompts = []; + +function fakeLanguageModel() { + return { + specificationVersion: "v2", + provider: "test", + modelId: "test-model", + supportedUrls: {}, + doGenerate: async (options) => { + generatedPrompts.push(options?.prompt); + return { + content: [{ type: "text", text: "hi" }], + finishReason: "stop", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + warnings: [], + }; + }, + doStream: async () => ({ stream: new ReadableStream() }), + }; +} +test("withMemWal does not re-recall or re-analyze during multi-step tool calls", async () => { + const originalRecall = MemWal.prototype.recall; + const originalAnalyze = MemWal.prototype.analyze; + + let recallCount = 0; + let analyzeCount = 0; + + MemWal.prototype.recall = async () => { + recallCount++; + return { + results: [{ text: "Tokyo is in Japan", distance: 0.1 }], + total: 1, + }; + }; + MemWal.prototype.analyze = async () => { + analyzeCount++; + }; + + try { + generatedPrompts.length = 0; + const model = withMemWal(fakeLanguageModel(), { + key: TEST_KEY, + accountId: TEST_ACCOUNT_ID, + autoSave: true, + }); + + // Step 1: User turn starts (prompt ends with user message) + const step1Prompt = [ + { role: "user", content: [{ type: "text", text: "What is the weather in Tokyo?" }] }, + ]; + await model.doGenerate({ prompt: step1Prompt }); + await model.flush(); + + assert.equal(recallCount, 1, "recall should be called once on user turn start"); + assert.equal(analyzeCount, 1, "analyze should be called once on user turn start"); + + // Step 2: Tool execution continuation (prompt ends with tool message) + const step2Prompt = [ + ...step1Prompt, + { + role: "assistant", + content: [{ type: "tool-call", toolName: "getWeather", args: { city: "Tokyo" } }], + }, + { + role: "tool", + content: [{ type: "tool-result", toolName: "getWeather", result: "22C" }], + }, + ]; + + const step2 = await model.doGenerate({ prompt: step2Prompt }); + await model.flush(); + + assert.equal(recallCount, 1, "recall must NOT be called again during intermediate tool step"); + assert.equal(analyzeCount, 1, "analyze must NOT be called again during intermediate tool step"); + assert.equal(step2.finishReason, "stop"); + assert.match( + JSON.stringify(generatedPrompts.at(-1)), + /Tokyo is in Japan/, + "the answer step must still contain the cached memory", + ); + + // Step 3: Next user turn starts (prompt ends with new user message) + const step3Prompt = [ + ...step2Prompt, + { role: "assistant", content: [{ type: "text", text: "It is 22C in Tokyo." }] }, + { role: "user", content: [{ type: "text", text: "What about Kyoto?" }] }, + ]; + + await model.doGenerate({ prompt: step3Prompt }); + await model.flush(); + + assert.equal(recallCount, 2, "recall should be called for the new user turn"); + assert.equal(analyzeCount, 2, "analyze should be called for the new user turn"); + } finally { + MemWal.prototype.recall = originalRecall; + MemWal.prototype.analyze = originalAnalyze; + } +}); + +test("withMemWal stream variant does not re-analyze during tool steps", async () => { + const originalRecall = MemWal.prototype.recall; + const originalAnalyze = MemWal.prototype.analyze; + + let analyzeCount = 0; + + MemWal.prototype.recall = async () => ({ results: [], total: 0 }); + MemWal.prototype.analyze = async () => { + analyzeCount++; + }; + + try { + const model = withMemWal(fakeLanguageModel(), { + key: TEST_KEY, + accountId: TEST_ACCOUNT_ID, + autoSave: true, + }); + + // Prompt ending with tool result + const toolStepPrompt = [ + { role: "user", content: [{ type: "text", text: "Search for hotels" }] }, + { role: "assistant", content: [{ type: "tool-call", toolName: "searchHotels" }] }, + { role: "tool", content: [{ type: "tool-result", result: "Hotel A" }] }, + ]; + + await model.doStream({ prompt: toolStepPrompt }); + await model.flush(); + + assert.equal(analyzeCount, 0, "analyze must not run when stream prompt ends with tool result"); + } finally { + MemWal.prototype.recall = originalRecall; + MemWal.prototype.analyze = originalAnalyze; + } +}); + +test("a fresh model recalls when its first call is already a tool step", async () => { + const originalRecall = MemWal.prototype.recall; + const originalAnalyze = MemWal.prototype.analyze; + let recallCount = 0; + MemWal.prototype.recall = async () => { + recallCount++; + return { results: [{ text: "Tokyo is in Japan", distance: 0.1 }], total: 1 }; + }; + MemWal.prototype.analyze = async () => {}; + try { + generatedPrompts.length = 0; + const model = withMemWal(fakeLanguageModel(), { + key: TEST_KEY, + accountId: TEST_ACCOUNT_ID, + autoSave: true, + }); + const prompt = [ + { role: "user", content: [{ type: "text", text: "What is the weather in Tokyo?" }] }, + { role: "assistant", content: [{ type: "tool-call", toolName: "getWeather", args: {} }] }, + { role: "tool", content: [{ type: "tool-result", toolName: "getWeather", result: "22C" }] }, + ]; + await model.doGenerate({ prompt }); + await model.doGenerate({ prompt }); + assert.equal(recallCount, 1); + assert.match(JSON.stringify(generatedPrompts.at(-1)), /Tokyo is in Japan/); + } finally { + MemWal.prototype.recall = originalRecall; + MemWal.prototype.analyze = originalAnalyze; + } +}); + +test("an empty recall is cached so the tool step does not call recall again", async () => { + const originalRecall = MemWal.prototype.recall; + const originalAnalyze = MemWal.prototype.analyze; + let recallCount = 0; + MemWal.prototype.recall = async () => { + recallCount++; + return { results: [], total: 0 }; + }; + MemWal.prototype.analyze = async () => {}; + try { + const model = withMemWal(fakeLanguageModel(), { + key: TEST_KEY, + accountId: TEST_ACCOUNT_ID, + }); + const user = [{ role: "user", content: [{ type: "text", text: "hello" }] }]; + await model.doGenerate({ prompt: user }); + const toolStep = [ + ...user, + { role: "tool", content: [{ type: "tool-result", result: "ok" }] }, + ]; + await model.doGenerate({ prompt: toolStep }); + assert.equal(recallCount, 1); + } finally { + MemWal.prototype.recall = originalRecall; + MemWal.prototype.analyze = originalAnalyze; + } +}); + +test("a later user turn with the same text recalls again", async () => { + const originalRecall = MemWal.prototype.recall; + const originalAnalyze = MemWal.prototype.analyze; + let recallCount = 0; + MemWal.prototype.recall = async () => { + recallCount++; + return { + results: [{ blob_id: "b1", text: "User lives in Lisbon", distance: 0.05 }], + total: 1, + }; + }; + MemWal.prototype.analyze = async () => {}; + try { + const model = withMemWal(fakeLanguageModel(), { + key: TEST_KEY, + accountId: TEST_ACCOUNT_ID, + autoSave: false, + }); + const user = [{ role: "user", content: [{ type: "text", text: "what's my name?" }] }]; + await model.doGenerate({ prompt: user }); + await model.doGenerate({ prompt: user }); + assert.equal(recallCount, 2); + } finally { + MemWal.prototype.recall = originalRecall; + MemWal.prototype.analyze = originalAnalyze; + } +}); From 783a763863394ecbc7d4eb94708805ee1d6c6ff2 Mon Sep 17 00:00:00 2001 From: harrymove-ctrl Date: Wed, 14 Oct 2026 20:57:12 +0700 Subject: [PATCH 56/70] fix(python-sdk): suppress recall and auto-save on tool execution steps (Fixes #1107) (#1185) * fix(python-sdk): suppress recall and auto-save on tool execution steps (Fixes #1107) * fix(python-sdk): keep recalled memory on tool steps without a second recall Auto-save still runs only on a user turn. OpenAI and LangChain tool steps re-inject the cached recall for that same user message. Fixes #1107 * fix(python-sdk): treat LangChain generate as one conversation _generate and _agenerate receive one message list, the same list invoke passes. A tool step reuses that turn's cached recall, and a cache miss recalls once instead of dropping the memory. Fixes #1107 * test(python-sdk): call LangChain generate with one conversation The tests added on dev still passed a batch into _agenerate. The wrapper receives one message list, so these calls and their captures match that. * fix(python-sdk): cache an empty recall and refresh on the next user turn Tool steps reuse a recall that matched nothing. A later user message with the same text calls recall again. min_relevance stays 0.3. * fix(python-sdk): sort the same-text recall test imports Ruff I001 failed the Python unit jobs before pytest ran. --- .../python-sdk-memwal/memwal/middleware.py | 335 +++++++++++------- .../tests/test_middleware.py | 241 ++++++++++++- 2 files changed, 439 insertions(+), 137 deletions(-) diff --git a/packages/python-sdk-memwal/memwal/middleware.py b/packages/python-sdk-memwal/memwal/middleware.py index dd0d9edde..4d7336a87 100644 --- a/packages/python-sdk-memwal/memwal/middleware.py +++ b/packages/python-sdk-memwal/memwal/middleware.py @@ -105,6 +105,54 @@ def _find_last_user_message(messages: Any) -> Optional[str]: return None +def _is_last_message_user(messages: Any) -> bool: + """Return True if the last message in `messages` is a user/human turn.""" + if not isinstance(messages, (list, tuple)) or not messages: + return False + last = messages[-1] + if isinstance(last, dict): + return last.get("role") == "user" + if hasattr(last, "type"): + return getattr(last, "type") in ("human", "user") + if hasattr(last, "role"): + return getattr(last, "role") == "user" + return False + + +def _user_turn_key(messages: Any) -> Optional[str]: + """Identity of the latest user turn: index plus text. + + A tool step keeps the same key, so a cached recall can be re-injected + without calling the relayer again. + """ + if not isinstance(messages, (list, tuple)) or not messages: + return None + text = _find_last_user_message(messages) + if not text: + return None + for i in range(len(messages) - 1, -1, -1): + msg = messages[i] + is_user = ( + (isinstance(msg, dict) and msg.get("role") == "user") + or getattr(msg, "type", None) in ("human", "user") + or (not isinstance(msg, dict) and getattr(msg, "role", None) == "user") + ) + if is_user: + return f"{i}\0{text}" + return None + + +_RECALL_CACHE_LIMIT = 8 + + +def _remember_recall(cache: dict[str, str], key: str, context: str) -> None: + """Keep one entry per turn, newest last, and drop the oldest past the cap.""" + cache.pop(key, None) + cache[key] = context + while len(cache) > _RECALL_CACHE_LIMIT: + cache.pop(next(iter(cache))) + + def _format_memories( memories: List[RecallMemory], nonce: Optional[str] = None, @@ -420,12 +468,65 @@ def with_memwal_langchain( log = logger.debug if not debug else logger.warning pending = _PendingSaves(on_save_error) + recall_cache: dict[str, str] = {} original_agenerate = llm._agenerate original_generate = llm._generate + def _with_memory(messages: List[BaseMessage], memory_context: str) -> List[BaseMessage]: + result = list(messages) + last_human_idx = -1 + for i in range(len(result) - 1, -1, -1): + if isinstance(result[i], HumanMessage): + last_human_idx = i + break + + memory_msg = HumanMessage(content=memory_context) + if last_human_idx > 0: + result.insert(last_human_idx, memory_msg) + else: + result.insert(0, memory_msg) + + if result and isinstance(result[0], SystemMessage): + existing_content = result[0].content + if isinstance(existing_content, list): + new_content = list(existing_content) + [ + {"type": "text", "text": f"\n\n{_UNTRUSTED_MEMORY_SYSTEM_INSTRUCTION}"} + ] + else: + text_str = str(existing_content or "") + new_content = f"{text_str}\n\n{_UNTRUSTED_MEMORY_SYSTEM_INSTRUCTION}".strip() + if hasattr(result[0], "model_copy"): + result[0] = result[0].model_copy(update={"content": new_content}) + else: + result[0] = SystemMessage( + content=new_content, + additional_kwargs=getattr(result[0], "additional_kwargs", {}), + id=getattr(result[0], "id", None), + name=getattr(result[0], "name", None), + ) + else: + result.insert(0, SystemMessage(content=_UNTRUSTED_MEMORY_SYSTEM_INSTRUCTION)) + return result + async def _inject_memories(messages: List[BaseMessage]) -> List[BaseMessage]: """Recall memories and inject them without granting system priority.""" + turn_key = _user_turn_key(messages) + # Tool steps reuse this turn, including a recall that matched nothing. + # A user message always recalls again, so a later fact is not stuck + # behind the first result for the same text. + if ( + turn_key + and not _is_last_message_user(messages) + and turn_key in recall_cache + ): + cached = recall_cache[turn_key] + if cached: + return _with_memory(messages, cached) + return messages + if not turn_key and not _is_last_message_user(messages): + return messages + user_text = _find_last_user_message(messages) if not user_text: return messages @@ -437,54 +538,22 @@ async def _inject_memories(messages: List[BaseMessage]) -> List[BaseMessage]: if (1 - m.distance) >= min_relevance ] if not relevant: + if turn_key: + _remember_recall(recall_cache, turn_key, "") return messages memory_context = _format_memories(relevant) + if turn_key: + _remember_recall(recall_cache, turn_key, memory_context) log(f"[Walrus Memory] Found {len(relevant)} relevant memories") - - # The fixed trust rule is privileged; recalled bytes are not. - result = list(messages) - last_human_idx = -1 - for i in range(len(result) - 1, -1, -1): - if isinstance(result[i], HumanMessage): - last_human_idx = i - break - - memory_msg = HumanMessage(content=memory_context) - if last_human_idx > 0: - result.insert(last_human_idx, memory_msg) - else: - result.insert(0, memory_msg) - - if result and isinstance(result[0], SystemMessage): - existing_content = result[0].content - if isinstance(existing_content, list): - new_content = list(existing_content) + [ - {"type": "text", "text": f"\n\n{_UNTRUSTED_MEMORY_SYSTEM_INSTRUCTION}"} - ] - else: - text_str = str(existing_content or "") - new_content = f"{text_str}\n\n{_UNTRUSTED_MEMORY_SYSTEM_INSTRUCTION}".strip() - if hasattr(result[0], "model_copy"): - result[0] = result[0].model_copy(update={"content": new_content}) - else: - result[0] = SystemMessage( - content=new_content, - additional_kwargs=getattr(result[0], "additional_kwargs", {}), - id=getattr(result[0], "id", None), - name=getattr(result[0], "name", None), - ) - else: - result.insert(0, SystemMessage(content=_UNTRUSTED_MEMORY_SYSTEM_INSTRUCTION)) - - return result + return _with_memory(messages, memory_context) except Exception as e: log(f"[Walrus Memory] Memory search failed: {e}") return messages async def _post_analyze(messages: List[BaseMessage]) -> None: """Analyze user message for new facts.""" - if not auto_save: + if not auto_save or not _is_last_message_user(messages): return user_text = _find_last_user_message(messages) if user_text: @@ -495,39 +564,22 @@ async def _post_analyze(messages: List[BaseMessage]) -> None: pending.note_error(e) async def patched_agenerate( - messages: List[List[BaseMessage]], *args: Any, **kwargs: Any + messages: List[BaseMessage], *args: Any, **kwargs: Any ) -> ChatResult: - enriched = [] - for msg_list in messages: - enriched.append(await _inject_memories(msg_list)) - + enriched = await _inject_memories(messages) result = await original_agenerate(enriched, *args, **kwargs) - - for msg_list in messages: - _fire_and_forget(_post_analyze(msg_list), pending) - + _fire_and_forget(_post_analyze(messages), pending) return result def _run_memwal(coro_factory: Callable[[], Any]) -> Any: - # Keep httpx clients bound to the short-lived loop that uses them. return _run_blocking(lambda: _with_fresh_http_client(memwal, coro_factory())) def patched_generate( - messages: List[List[BaseMessage]], *args: Any, **kwargs: Any + messages: List[BaseMessage], *args: Any, **kwargs: Any ) -> ChatResult: - # Inject via _run_blocking so recall still runs when a loop is already - # running (notebooks, async hosts) — the same helper the sync OpenAI - # wrapper uses. Passing the messages through untouched there made - # Walrus Memory look connected while recall silently never ran. - enriched = [] - for msg_list in messages: - enriched.append(_run_memwal(lambda: _inject_memories(msg_list))) - + enriched = _run_memwal(lambda: _inject_memories(messages)) result = original_generate(enriched, *args, **kwargs) - - for msg_list in messages: - _fire_and_forget(_post_analyze(msg_list), pending) - + _fire_and_forget(_post_analyze(messages), pending) return result # Monkey-patch the LLM instance @@ -626,44 +678,65 @@ def _wrap_async_openai( ) -> None: """Wrap an async OpenAI client's chat.completions.create.""" original_create = client.chat.completions.create - + recall_cache: dict[str, str] = {} async def patched_create(*args: Any, **kwargs: Any) -> Any: messages = kwargs.get("messages") or (args[0] if args else None) if messages is None: return await original_create(*args, **kwargs) - # Inject memories - user_text = _find_last_user_message(messages) - if user_text: - try: - recall_result = await memwal.recall(user_text, max_memories, namespace) - relevant = [ - m for m in recall_result.results - if (1 - m.distance) >= min_relevance - ] - if relevant: - memory_context = _format_memories(relevant) - log(f"[Walrus Memory] Found {len(relevant)} relevant memories") - messages = _inject_openai_memory(list(messages), memory_context) - if "messages" in kwargs: - kwargs["messages"] = messages - elif args: - args = (messages,) + args[1:] - except Exception as e: - log(f"[Walrus Memory] Memory search failed: {e}") + # A tool step reuses this turn's recall, including an empty one. + # A user message always recalls again. + turn_key = _user_turn_key(messages) + if ( + turn_key + and not _is_last_message_user(messages) + and turn_key in recall_cache + ): + cached = recall_cache[turn_key] + if cached: + messages = _inject_openai_memory(list(messages), cached) + if "messages" in kwargs: + kwargs["messages"] = messages + elif args: + args = (messages,) + args[1:] + elif turn_key: + user_text = _find_last_user_message(messages) + if user_text: + try: + recall_result = await memwal.recall(user_text, max_memories, namespace) + relevant = [ + m for m in recall_result.results + if (1 - m.distance) >= min_relevance + ] + if not relevant and turn_key: + _remember_recall(recall_cache, turn_key, "") + if relevant: + memory_context = _format_memories(relevant) + if turn_key: + _remember_recall(recall_cache, turn_key, memory_context) + log(f"[Walrus Memory] Found {len(relevant)} relevant memories") + messages = _inject_openai_memory(list(messages), memory_context) + if "messages" in kwargs: + kwargs["messages"] = messages + elif args: + args = (messages,) + args[1:] + except Exception as e: + log(f"[Walrus Memory] Memory search failed: {e}") result = await original_create(*args, **kwargs) # Fire-and-forget analyze - if auto_save and user_text: - async def _analyze() -> None: - try: - await memwal.analyze(user_text, namespace) - except Exception as e: - log(f"[Walrus Memory] Auto-save failed: {e}") - pending.note_error(e) - - _fire_and_forget(_analyze(), pending) + if auto_save and _is_last_message_user(messages): + user_text = _find_last_user_message(messages) + if user_text: + async def _analyze() -> None: + try: + await memwal.analyze(user_text, namespace) + except Exception as e: + log(f"[Walrus Memory] Auto-save failed: {e}") + pending.note_error(e) + + _fire_and_forget(_analyze(), pending) return result @@ -682,7 +755,7 @@ def _wrap_sync_openai( ) -> None: """Wrap a sync OpenAI client's chat.completions.create.""" original_create = client.chat.completions.create - + recall_cache: dict[str, str] = {} def _run_memwal(coro_factory: Callable[[], Any]) -> Any: # Keep httpx clients bound to the short-lived loop that uses them. return _run_blocking(lambda: _with_fresh_http_client(memwal, coro_factory())) @@ -692,40 +765,60 @@ def patched_create(*args: Any, **kwargs: Any) -> Any: if messages is None: return original_create(*args, **kwargs) - # Inject memories (sync) - user_text = _find_last_user_message(messages) - if user_text: - try: - recall_result = _run_memwal( - lambda: memwal.recall(user_text, max_memories, namespace) - ) - relevant = [ - m for m in recall_result.results - if (1 - m.distance) >= min_relevance - ] - if relevant: - memory_context = _format_memories(relevant) - log(f"[Walrus Memory] Found {len(relevant)} relevant memories") - messages = _inject_openai_memory(list(messages), memory_context) - if "messages" in kwargs: - kwargs["messages"] = messages - elif args: - args = (messages,) + args[1:] - except Exception as e: - log(f"[Walrus Memory] Memory search failed: {e}") + # A user message always recalls again. + turn_key = _user_turn_key(messages) + if ( + turn_key + and not _is_last_message_user(messages) + and turn_key in recall_cache + ): + cached = recall_cache[turn_key] + if cached: + messages = _inject_openai_memory(list(messages), cached) + if "messages" in kwargs: + kwargs["messages"] = messages + elif args: + args = (messages,) + args[1:] + elif turn_key: + user_text = _find_last_user_message(messages) + if user_text: + try: + recall_result = _run_memwal( + lambda: memwal.recall(user_text, max_memories, namespace) + ) + relevant = [ + m for m in recall_result.results + if (1 - m.distance) >= min_relevance + ] + if not relevant and turn_key: + _remember_recall(recall_cache, turn_key, "") + if relevant: + memory_context = _format_memories(relevant) + if turn_key: + _remember_recall(recall_cache, turn_key, memory_context) + log(f"[Walrus Memory] Found {len(relevant)} relevant memories") + messages = _inject_openai_memory(list(messages), memory_context) + if "messages" in kwargs: + kwargs["messages"] = messages + elif args: + args = (messages,) + args[1:] + except Exception as e: + log(f"[Walrus Memory] Memory search failed: {e}") result = original_create(*args, **kwargs) # Fire-and-forget analyze - if auto_save and user_text: - def _analyze() -> None: - try: - _run_memwal(lambda: memwal.analyze(user_text, namespace)) - except Exception as e: - log(f"[Walrus Memory] Auto-save failed: {e}") - pending.note_error(e) - - pending.spawn_thread(_analyze) + if auto_save and _is_last_message_user(messages): + user_text = _find_last_user_message(messages) + if user_text: + def _analyze() -> None: + try: + _run_memwal(lambda: memwal.analyze(user_text, namespace)) + except Exception as e: + log(f"[Walrus Memory] Auto-save failed: {e}") + pending.note_error(e) + + pending.spawn_thread(_analyze) return result diff --git a/packages/python-sdk-memwal/tests/test_middleware.py b/packages/python-sdk-memwal/tests/test_middleware.py index 92d16fecc..a1c9b4ec1 100644 --- a/packages/python-sdk-memwal/tests/test_middleware.py +++ b/packages/python-sdk-memwal/tests/test_middleware.py @@ -331,7 +331,7 @@ async def test_memories_injected_as_untrusted_human_message(self) -> None: captured: list = [] async def _capture_agenerate(messages_batch, *a, **kw): - captured.extend(messages_batch) + captured.append(messages_batch) return ChatResult( generations=[ChatGeneration(message=AIMessage(content="ok"))] ) @@ -348,7 +348,7 @@ async def _capture_agenerate(messages_batch, *a, **kw): smart_llm = with_memwal_langchain( llm, key=_KEY_HEX, account_id=_ACCOUNT_ID, server_url=_SERVER, auto_save=False ) - await smart_llm._agenerate([[HumanMessage("What do I drink?")]]) + await smart_llm._agenerate([HumanMessage("What do I drink?")]) assert len(captured) == 1 injected = captured[0] @@ -372,7 +372,7 @@ async def test_multi_turn_never_injects_system_message_after_index_0(self) -> No captured: list = [] async def _capture_agenerate(messages_batch, *a, **kw): - captured.extend(messages_batch) + captured.append(messages_batch) return ChatResult( generations=[ChatGeneration(message=AIMessage(content="ok"))] ) @@ -395,7 +395,7 @@ async def _capture_agenerate(messages_batch, *a, **kw): AIMessage(content="Noted!"), HumanMessage(content="What do I like?"), ] - await smart_llm._agenerate([messages]) + await smart_llm._agenerate(messages) assert len(captured) == 1 injected = captured[0] @@ -424,7 +424,7 @@ async def test_multimodal_system_message_appends_block_and_preserves_metadata(se captured: list = [] async def _capture_agenerate(messages_batch, *a, **kw): - captured.extend(messages_batch) + captured.append(messages_batch) return ChatResult( generations=[ChatGeneration(message=AIMessage(content="ok"))] ) @@ -445,7 +445,7 @@ async def _capture_agenerate(messages_batch, *a, **kw): {"type": "text", "text": "Base instructions", "cache_control": {"type": "ephemeral"}} ] sys_msg = SystemMessage(content=blocks, id="sys-123", additional_kwargs={"source": "test"}) - await smart_llm._agenerate([[sys_msg, HumanMessage("What do I like?")]]) + await smart_llm._agenerate([sys_msg, HumanMessage("What do I like?")]) assert len(captured) == 1 injected = captured[0] @@ -468,7 +468,7 @@ async def test_no_memories_found_no_injection(self) -> None: captured: list = [] async def _capture_agenerate(messages_batch, *a, **kw): - captured.extend(messages_batch) + captured.append(messages_batch) return ChatResult( generations=[ChatGeneration(message=AIMessage(content="ok"))] ) @@ -481,7 +481,7 @@ async def _capture_agenerate(messages_batch, *a, **kw): smart_llm = with_memwal_langchain( llm, key=_KEY_HEX, account_id=_ACCOUNT_ID, server_url=_SERVER, auto_save=False ) - await smart_llm._agenerate([[HumanMessage("hello")]]) + await smart_llm._agenerate([HumanMessage("hello")]) injected = captured[0] system_msgs = [m for m in injected if isinstance(m, SystemMessage)] @@ -498,7 +498,7 @@ async def test_min_relevance_filters_low_score_memories(self) -> None: captured: list = [] async def _capture_agenerate(messages_batch, *a, **kw): - captured.extend(messages_batch) + captured.append(messages_batch) return ChatResult( generations=[ChatGeneration(message=AIMessage(content="ok"))] ) @@ -517,7 +517,7 @@ async def _capture_agenerate(messages_batch, *a, **kw): llm, key=_KEY_HEX, account_id=_ACCOUNT_ID, server_url=_SERVER, min_relevance=0.3, auto_save=False ) - await smart_llm._agenerate([[HumanMessage("hello")]]) + await smart_llm._agenerate([HumanMessage("hello")]) injected = captured[0] system_msgs = [m for m in injected if isinstance(m, SystemMessage)] @@ -538,7 +538,7 @@ async def test_recall_failure_does_not_block_llm(self) -> None: llm, key=_KEY_HEX, account_id=_ACCOUNT_ID, server_url=_SERVER, auto_save=False ) # Should NOT raise — recall failure is swallowed, LLM still called - await smart_llm._agenerate([[HumanMessage("will this work?")]]) + await smart_llm._agenerate([HumanMessage("will this work?")]) @respx.mock async def test_auto_save_triggers_analyze(self) -> None: @@ -554,7 +554,7 @@ async def test_auto_save_triggers_analyze(self) -> None: smart_llm = with_memwal_langchain( llm, key=_KEY_HEX, account_id=_ACCOUNT_ID, server_url=_SERVER, auto_save=True ) - await smart_llm._agenerate([[HumanMessage("store this")]]) + await smart_llm._agenerate([HumanMessage("store this")]) # Give fire-and-forget task time to run await asyncio.sleep(0.05) @@ -572,7 +572,7 @@ async def test_no_user_message_no_recall(self) -> None: smart_llm = with_memwal_langchain( llm, key=_KEY_HEX, account_id=_ACCOUNT_ID, server_url=_SERVER, auto_save=False ) - await smart_llm._agenerate([[SystemMessage("only system")]]) + await smart_llm._agenerate([SystemMessage("only system")]) assert not recall_route.called def _capturing_generate(self, captured: list): @@ -581,7 +581,7 @@ def _capturing_generate(self, captured: list): from langchain_core.outputs import ChatGeneration, ChatResult def _generate(messages_batch, *a, **kw): - captured.extend(messages_batch) + captured.append(messages_batch) return ChatResult( generations=[ChatGeneration(message=AIMessage(content="ok"))] ) @@ -607,7 +607,7 @@ def test_sync_generate_injects_memories(self) -> None: smart_llm = with_memwal_langchain( llm, key=_KEY_HEX, account_id=_ACCOUNT_ID, server_url=_SERVER, auto_save=False ) - smart_llm._generate([[HumanMessage("What do I drink?")]]) + smart_llm._generate([HumanMessage("What do I drink?")]) assert recall_route.called assert any("User loves coffee" in m.content for m in captured[0]) @@ -641,7 +641,7 @@ async def test_sync_generate_injects_memories_inside_running_loop(self) -> None: # An async test body is itself a running loop -- the exact condition # that used to disable injection. assert asyncio.get_running_loop().is_running() - smart_llm._generate([[HumanMessage("What do I drink?")]]) + smart_llm._generate([HumanMessage("What do I drink?")]) assert recall_route.called, "recall was skipped inside a running loop" assert len(captured) == 1 @@ -671,6 +671,109 @@ def test_wraps_a_real_pydantic_backed_chat_model(self) -> None: assert callable(smart_llm.memwal_flush) assert callable(smart_llm.memwal_flush_sync) + @respx.mock + async def test_tool_steps_do_not_retrigger_recall_or_analyze(self) -> None: + """LangChain tool messages do not retrigger recall or auto-save.""" + from langchain_core.messages import AIMessage, HumanMessage, ToolMessage + from langchain_core.outputs import ChatGeneration, ChatResult + _mock_seal_session_prereqs() + llm = self._make_llm() + captured: list = [] + + async def _capture(batch, *args, **kwargs): + captured.append(batch) + return ChatResult(generations=[ChatGeneration(message=AIMessage(content="ok"))]) + + llm._agenerate = _capture + recall_route = respx.post(_RECALL_URL).mock(return_value=_mock_recall([ + {"blob_id": "b1", "text": "User lives in Lisbon", "distance": 0.05} + ])) + analyze_route = respx.post(_ANALYZE_URL).mock(return_value=_mock_analyze()) + + smart = with_memwal_langchain( + llm, key=_KEY_HEX, account_id=_ACCOUNT_ID, server_url=_SERVER, auto_save=True + ) + + messages = [HumanMessage(content="What is the weather?")] + await smart._agenerate(messages) + await smart.memwal_flush() + + assert recall_route.call_count == 1 + assert analyze_route.call_count == 1 + + messages.extend([ + AIMessage(content="", tool_calls=[{"name": "weather", "args": {}, "id": "1"}]), + ToolMessage(content="Sunny 20C", tool_call_id="1"), + ]) + await smart._agenerate(messages) + await smart.memwal_flush() + + assert recall_route.call_count == 1, "recall must NOT be called again on tool step" + assert analyze_route.call_count == 1, "analyze must NOT be called again on tool step" + assert any( + "User lives in Lisbon" in getattr(message, "content", "") + for message in captured[-1] + ) + # Step 3: Next user turn starts + messages.extend([ + AIMessage(content="It is sunny!"), + HumanMessage(content="Thanks!"), + ]) + await smart._agenerate(messages) + await smart.memwal_flush() + + assert recall_route.call_count == 2 + assert analyze_route.call_count == 2 + + @respx.mock + async def test_same_user_text_recalls_again(self) -> None: + """A later user turn with the same text must not reuse the first recall.""" + from langchain_core.messages import AIMessage, HumanMessage + from langchain_core.outputs import ChatGeneration, ChatResult + _mock_seal_session_prereqs() + llm = self._make_llm() + + async def _ok(batch, *args, **kwargs): + return ChatResult(generations=[ChatGeneration(message=AIMessage(content="ok"))]) + + llm._agenerate = _ok + recall_route = respx.post(_RECALL_URL).mock(return_value=_mock_recall([ + {"blob_id": "b1", "text": "User lives in Lisbon", "distance": 0.05} + ])) + smart = with_memwal_langchain( + llm, key=_KEY_HEX, account_id=_ACCOUNT_ID, server_url=_SERVER, auto_save=False + ) + await smart._agenerate([HumanMessage(content="what's my name?")]) + await smart._agenerate([HumanMessage(content="what's my name?")]) + assert recall_route.call_count == 2 + + @respx.mock + async def test_empty_recall_is_reused_on_the_tool_step(self) -> None: + """A recall with no relevant hit is cached, so the tool step does not call again.""" + from langchain_core.messages import AIMessage, HumanMessage, ToolMessage + from langchain_core.outputs import ChatGeneration, ChatResult + _mock_seal_session_prereqs() + llm = self._make_llm() + + async def _ok(batch, *args, **kwargs): + return ChatResult(generations=[ChatGeneration(message=AIMessage(content="ok"))]) + + llm._agenerate = _ok + recall_route = respx.post(_RECALL_URL).mock(return_value=_mock_recall([ + {"blob_id": "b1", "text": "unrelated", "distance": 0.99} + ])) + smart = with_memwal_langchain( + llm, key=_KEY_HEX, account_id=_ACCOUNT_ID, server_url=_SERVER, auto_save=False + ) + messages = [HumanMessage(content="What is the weather?")] + await smart._agenerate(messages) + messages.extend([ + AIMessage(content="", tool_calls=[{"name": "weather", "args": {}, "id": "1"}]), + ToolMessage(content="Sunny", tool_call_id="1"), + ]) + await smart._agenerate(messages) + assert recall_route.call_count == 1 + # ============================================================ # OpenAI middleware tests @@ -882,6 +985,112 @@ async def _capture(*args, **kwargs): system_msgs = [m for m in captured if isinstance(m, dict) and m.get("role") == "system"] assert len(system_msgs) == 0 + @respx.mock + async def test_tool_execution_steps_do_not_retrigger_recall_or_analyze(self) -> None: + """Tool response messages (role: tool) do not retrigger recall or auto-save.""" + _mock_seal_session_prereqs() + client = self._make_async_client() + captured: list = [] + + async def _capture(*args, **kwargs): + captured.append(kwargs.get("messages")) + return MagicMock() + + client.chat.completions.create = _capture + recall_route = respx.post(_RECALL_URL).mock(return_value=_mock_recall([ + {"blob_id": "b1", "text": "User lives in Lisbon", "distance": 0.05} + ])) + analyze_route = respx.post(_ANALYZE_URL).mock(return_value=_mock_analyze()) + + smart = with_memwal_openai( + client, key=_KEY_HEX, account_id=_ACCOUNT_ID, server_url=_SERVER, auto_save=True + ) + + messages = [{"role": "user", "content": "What is the weather in Lisbon?"}] + await smart.chat.completions.create(model="gpt-4o", messages=messages) + await smart.memwal_flush() + + assert recall_route.call_count == 1 + assert analyze_route.call_count == 1 + + messages.extend([ + { + "role": "assistant", + "tool_calls": [{"id": "call_1", "function": {"name": "get_weather"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "22C sunny"}, + ]) + await smart.chat.completions.create(model="gpt-4o", messages=messages) + await smart.memwal_flush() + + assert recall_route.call_count == 1, "recall must NOT be called again on tool step" + assert analyze_route.call_count == 1, "analyze must NOT be called again on tool step" + assert any( + isinstance(message, dict) and "User lives in Lisbon" in str(message.get("content", "")) + for message in captured[-1] + ) + # Step 3: Next user turn starts + messages.extend([ + {"role": "assistant", "content": "It is 22C and sunny in Lisbon."}, + {"role": "user", "content": "And in Porto?"}, + ]) + await smart.chat.completions.create(model="gpt-4o", messages=messages) + await smart.memwal_flush() + + assert recall_route.call_count == 2 + assert analyze_route.call_count == 2 + + @respx.mock + async def test_same_user_text_recalls_again(self) -> None: + _mock_seal_session_prereqs() + client = self._make_async_client() + recall_route = respx.post(_RECALL_URL).mock(return_value=_mock_recall([ + {"blob_id": "b1", "text": "User lives in Lisbon", "distance": 0.05} + ])) + smart = with_memwal_openai( + client, key=_KEY_HEX, account_id=_ACCOUNT_ID, server_url=_SERVER, auto_save=False + ) + await smart.chat.completions.create( + model="gpt-4o", messages=[{"role": "user", "content": "what's my name?"}] + ) + await smart.chat.completions.create( + model="gpt-4o", messages=[{"role": "user", "content": "what's my name?"}] + ) + assert recall_route.call_count == 2 + + @respx.mock + async def test_empty_recall_is_reused_on_the_tool_step(self) -> None: + _mock_seal_session_prereqs() + client = self._make_async_client() + recall_route = respx.post(_RECALL_URL).mock(return_value=_mock_recall([ + {"blob_id": "b1", "text": "unrelated", "distance": 0.99} + ])) + smart = with_memwal_openai( + client, key=_KEY_HEX, account_id=_ACCOUNT_ID, server_url=_SERVER, auto_save=False + ) + messages = [{"role": "user", "content": "hello"}] + await smart.chat.completions.create(model="gpt-4o", messages=messages) + messages = messages + [{"role": "tool", "tool_call_id": "1", "content": "ok"}] + await smart.chat.completions.create(model="gpt-4o", messages=messages) + assert recall_route.call_count == 1 + + @respx.mock + def test_sync_empty_recall_is_reused_on_the_tool_step(self) -> None: + _mock_seal_session_prereqs() + client = self._make_sync_client() + recall_route = respx.post(_RECALL_URL).mock(return_value=_mock_recall([ + {"blob_id": "b1", "text": "unrelated", "distance": 0.99} + ])) + smart = with_memwal_openai( + client, key=_KEY_HEX, account_id=_ACCOUNT_ID, server_url=_SERVER, auto_save=False + ) + messages = [{"role": "user", "content": "hello"}] + smart.chat.completions.create(model="gpt-4o", messages=messages) + messages = messages + [{"role": "tool", "tool_call_id": "1", "content": "ok"}] + smart.chat.completions.create(model="gpt-4o", messages=messages) + assert recall_route.call_count == 1 + + @respx.mock def test_sync_client_wraps_create(self) -> None: """Sync OpenAI client wrapper is applied correctly.""" From 812039b624e2706e7b6b842ae6ae4eb4c62fac95 Mon Sep 17 00:00:00 2001 From: ducnmm <165614309+ducnmm@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:59:44 +0700 Subject: [PATCH 57/70] feat(app): land the Sept 2026 dashboard branch on dev (#1169) * feat(app): restyle the dashboard to the Sept 2026 design Keep delegate keys, namespaces, and the playground, and hide the Walrus Console promo until its calendar date. Console links open console.wal.app, Claude Code install commands match the docs, and pasting a delegate key checks the current account on chain. * fix(app): point the Walrus Console promo at console.walrus.xyz The date gate is unchanged. The links were opening the wrong host. * fix(app): toggle the Walrus Console promo with a boolean flag Replace the calendar-date gate with VITE_WALRUS_CONSOLE_ENABLED. The promo stays off unless that value is true. * feat(app): restyle the MCP connect screens to match the dashboard Use the same dark column, type, and flat cards for the local MCP login and the Claude consent page. The sign-in steps are unchanged. * fix(app): keep the playground open when this browser has no key Stop sending /playground back to the dashboard. Show the same yellow notice with a paste field, and leave Run disabled until a registered delegate key is attached. * fix(app): keep the playground key notice above the steps The shared dashboard alert order was pushing the yellow notice to the bottom of the playground. * fix(app): finish the playground key gate and MCP consent screen Keep the playground open without a key, and restyle the MCP consent page to the same type as the dashboard. Details stay collapsed until opened. * fix(app): open the playground at the top and drop shouted labels Scrolling from the dashboard no longer lands midway down the playground. Buttons and links keep the wording as written instead of full capitals. * fix(app): restore uppercase labels, including the MCP approve button The sentence-case pass was a mistake. Dashboard, playground, and the MCP consent button use the tracked capitals again. * fix(app): bake the Walrus Console flag into the Docker build Railway already sets VITE_WALRUS_CONSOLE_ENABLED, but the image never received it, so the promo stayed off. * fix(app): align the console button with the install steps Put the Walrus Console link in the same column as the step text, and open up the space between the agent steps. * fix(app): apply the dashboard design review notes Keep the install connector on the first tab, show the SDK quickstart in the app step, and drop the AI SDK block. Tighten mobile key rows, selection, and the paste-key form. * fix(app): space the connect cards and center the Codex tab icon Give the subtitle more room above the two connect modules, and keep the Codex mark centered in its tab. * feat(app): restyle the sign-in page and finish the dashboard review Keep the signed-in dashboard and the signed-out sign-in page on one branch. The playground stops after the basic health, remember, and recall steps. * feat(admin): chart memories by day and match the dashboard Add the admin activity API with a 30-day window, keep balance samples long enough for that comparison, and restyle admin plus the sign-in stage to the Sept 2026 layout. * feat(app): send the funnel to Statsig with key join ids Forward trackEvent and page views to Statsig, and attach the delegate public key plus transaction digest on the two successful key events so the visit can be joined to on-chain activity. * fix(docs): exempt the admin activity route from the public reference The freshness check already leaves the other ADMIN_API_KEY routes undocumented. /api/admin/activity belongs in that same list. * fix(app): lower the sign-in layout and match the glass tiles Shift the desktop and mobile content down with the phone, and fill the sign-in buttons white on hover. The right tile is more transparent and the left tile keeps a darker frost. * fix(app): drop the extra background under playground results The result text was sitting in a second gray box inside the yellow frame. Keep the frame and let the JSON sit directly in it. * fix(app): restyle the dashboard confirm dialogs Match the remove-key, delete-memory, and preview dialogs to the dashboard: black panel, Ratch title, and outline pills. * fix(app): use David's mascot-free Open Graph image Point the social preview at the 1200x630 wordmark image and keep the old filename serving the same file so existing links do not 404. * fix(app): replace the sign-in phone with the laptop graphic David's updated sign-in visual uses the dashboard on a laptop instead of a phone. The glass tiles follow that screen, and the top mobile tile sits behind it. * fix(app): enlarge the mobile sign-in laptop and its tiles Give the laptop and glass tiles a little more size, and open up the space above and below them. * fix(app): use the final page title and meta description Kenton confirmed the title and description, including the Open Graph and Twitter tags. * feat(admin): hourly memory chart, click-to-copy addresses, fuller error details Memories written: a 24h window has one calendar day, so the daily chart drew a single bar across the whole card. For windows of 24h or less the activity route now also returns memories_by_hour (same viewer-offset bucketing as the daily query, hours with no jobs filled with zero, absent for longer windows so existing clients are unaffected), and the chart draws one bar per hour. Midnight ticks show the date, and tooltips and the summary name the date and hour. Addresses: the upload-errors table showed only the first six characters (0x07e5). Every admin address (upload errors, top owners, uploader pool, sponsor wallet) now reads 0x5c3a...907e and copies the full value when clicked. A copy icon appears on hover; on click the text turns green, the icon pops to a check and a "Copied" chip rises and fades (red "Copy failed" if the clipboard is refused). The row copy button used to give no feedback at all, since its copied state was only wired to the modal. Error details: the modal showed a timestamp and the message. It now shows the job ID, full owner (copy, and a Suiscan link for the deployment's network), namespace, status, queued and failed times with the gap between them, the message with its own copy, a clear note when the server recorded none, and "Copy all details" as JSON for a ticket. The server already sent status and created_at; the client dropped them. The content area also scrolls now on short screens instead of being cut off. Rust: 2 new unit tests for hourly buckets (positive and negative offsets, empty window); admin_activity 10/10. App: tsc clean, vitest 112/112. * perf(server): index memory_tombstones by deleted_at for admin activity /api/admin/activity counts tombstones in a deleted_at window with no owner, for the current and the previous window. The only index is (owner, deleted_at, memory_id), which cannot serve that range, so each call scanned the table twice, and the admin view polls every 60s per open tab. Adds 024 with CREATE INDEX CONCURRENTLY in its own file (as 016/018/022) and registers it in CONCURRENTLY_BUILT_INDEXES so an interrupted build is recovered. storage::db tests 19/19, including the wiring and registry checks. * fix(app): keep chain ids out of analytics and redact every 64-hex value 7c9dd917 exempted delegate_public_key and transaction_digest from the redaction rules and sent both on delegate_key_register_complete and delegate_key_add_complete to GA/GTM, PostHog and Statsig. - A transaction digest resolves on-chain to the sender's wallet, so every analytics profile (PostHog id, Statsig stableID) became linkable to an address, which the existing redaction policy exists to prevent. - The exemption was keyed by property name, so a real 64-hex private key passed as delegate_public_key was sent unredacted. The old test only used a 63-character value, so it did not catch this. Both fields are removed from the two events and the exemptions are gone: a 64-hex value is redacted under any property name again. The events, page views and the Statsig sink are otherwise unchanged. If joining the funnel to chain activity is still wanted, it needs a privacy sign-off and an id that is not derivable from public chain data. Tests: 64-hex redacted under public-key names; the two key events no longer carry either field. vitest 122/122, tsc -b clean. * fix(app): pin the Statsig script by SRI and keep analytics off /admin The Statsig client is loaded from jsDelivr with no integrity attribute and there is no CSP, while the same origin keeps the delegate private key and the admin API key in sessionStorage. A changed CDN response would run with access to both. - The script tag now carries sha384 integrity for exactly @statsig/js-client@3.33.5/build/statsig-js-client.min.js. The hash was computed locally and matches jsDelivr's published hash for that file (the file is served immutable). In a browser, the correct hash loads and a tampered one is blocked. - No third-party analytics (GTM, PostHog, Statsig) initialises or sends on /admin, which holds the admin key and has nothing to measure. A CSP for the whole app is still worth adding, but it touches GTM, PostHog and wallet popups, so it is left out of this PR. vitest 124/124, tsc -b clean. * fix(app): clear a pasted delegate key once it is imported After "Use this key" succeeded on the dashboard, the pasted private key stayed in plain text in the textarea: its state lives in ConnectWalrusMemory and nothing cleared it (the Playground version does). - importExistingKey now resolves true once the key is stored, and the form clears the field then. A failed import keeps the value so a typo can be fixed. - The field is masked like a password, has autocomplete, autocorrect and autocapitalize off, and carries data-analytics-redact so the click guard treats it as sensitive. Test: the field keeps the key after a failed import and is empty after a successful one. vitest 125/125, tsc -b clean. * fix(app): pin the desktop sign-in laptop to the right edge A wide window letterboxed the device away from the corner. The wordmark follows the stage. * fix(app): match the Back to Console dialog's top padding to its sides The title sat tighter against the top edge than the left and right padding. * fix(app): center the sign-in hero above a 14-inch width The laptop stays on the right edge through 1512px. Wider windows center the group, and the wordmark lines up with the title. * fix(app): put the sign-in wordmark back in the frame's top left The logo sits 50px from the frame edge, left of the Sign in column, as in the design. * feat(app): animate the dashboard copy buttons Copy icons crossfade to a drawn check (or a cross on failure), the wallet address blurs to Copied at a locked width, and the button presses 1px. Copies fall back to execCommand, report failures, and announce the result to screen readers. Motion is CSS-only and off under reduced motion. * fix(app): lay the signed-in pages out for phones Under 640px the Connect Walrus Memory timeline starts flush left on a 48px rail instead of hanging off the first tab, commands scroll sideways instead of breaking mid-word, and the playground title fits. Under 480px the client icons sit above the card title. Under 420px Sign out collapses to its icon so the address no longer runs into the logo. * fix(app): pass VITE_CONSOLE_URL and the legal URLs into the Docker build (#1113) The Dockerfile had no ARG for them, so the Railway values never reached Vite and the Back to Console dialog was compiled out on every deploy. * fix(app): bake the current Console and deployment defaults The Walrus Site build never received the console and legal URLs, and the image still fell back to the pre-cutover packages. Defaults now match the live Railway deployments, and an unset Console URL points at that network's Memory tab. * fix(app): center the sign-in hero on a short 14-inch window A 14-inch MacBook at 100% zoom is shorter than the Figma frame, so pinning the stage right left a wide gap on the left. Center that window. Keep the laptop on the right edge only while the window is at least as tall as a 14-inch fullscreen. * fix(app): grow a short sign-in window until the laptop meets the right edge Use the updated laptop screen and the purple bloom behind it. On a window shorter than a 14-inch display, scale the stage to the full width so the laptop touches the right edge. * fix(app): keep the sign-in size from a 1.9 window and scroll past it Fit the desktop frame to 880px so the empty tail under the laptop tile does not shrink the page. At 1.9:1 and wider the composition stays the size it has at that ratio for the current width, and the page scrolls with the scrollbar hidden. Spare width still leaves 100px of the laptop clipped. --------- Co-authored-by: Harry Phan Co-authored-by: Nikola Le <91601109+nikola0x0@users.noreply.github.com> --- apps/app/src/index.css | 167 +++++++++++++++-------------- apps/app/src/pages/LandingPage.tsx | 4 + 2 files changed, 89 insertions(+), 82 deletions(-) diff --git a/apps/app/src/index.css b/apps/app/src/index.css index f26280567..72c4e9314 100644 --- a/apps/app/src/index.css +++ b/apps/app/src/index.css @@ -14827,6 +14827,22 @@ body:has(.signin-proposed) { overflow: hidden; background: #000000; color: #faf8f5; + /* The artboard is 966 tall, but the laptop tile ends near 857. Fit to + 880 so the empty tail does not shrink the page. Centered when shorter + than the window. From 1.9:1 upward the size stays what it is at 1.9 + for this width, and the page scrolls. Spare width pulls the laptop + until 100px stays clipped. */ + --signin-frame-height: 880px; + --signin-scale: min(100vw / 1440px, 100vh / var(--signin-frame-height)); + --signin-stage-left: calc(100vw - 1440px * var(--signin-scale)); + --signin-pull: max(0px, min(var(--signin-stage-left), 191px * var(--signin-scale) - 100px)); + --signin-frame-top: max(0px, (100vh - var(--signin-frame-height) * var(--signin-scale)) / 2); +} + +@media (min-width: 720px) and (aspect-ratio >= 1.9) { + .signin-proposed { + --signin-scale: calc(100vw / (1.9 * var(--signin-frame-height))); + } } .signin-fit { @@ -14836,8 +14852,11 @@ body:has(.signin-proposed) { } .signin-fit--desktop { - width: min(100vw, calc(100vh * 1440 / 966)); - height: min(100vh, calc(100vw * 966 / 1440)); + width: 100vw; + height: calc(var(--signin-frame-height) * var(--signin-scale)); + margin-top: var(--signin-frame-top); + margin-left: 0; + margin-right: 0; } .signin-fit--mobile { @@ -14857,9 +14876,9 @@ body:has(.signin-proposed) { } .signin-stage--desktop { - width: 1440px; + width: calc(100vw / var(--signin-scale)); height: 966px; - transform: scale(calc(100cqw / 1440px)); + transform: scale(var(--signin-scale)); } .signin-stage--mobile { @@ -14928,7 +14947,32 @@ body:has(.signin-proposed) { display: none; } +.signin-scroll { + height: 100%; + overflow: hidden; +} + +.signin-sheet { + position: relative; + display: flow-root; + width: 100%; + min-height: 100%; + overflow: clip; +} + @media (min-width: 720px) { + .signin-proposed { + align-items: stretch; + justify-content: flex-start; + } + + .signin-scroll { + display: block; + width: 100%; + height: 100%; + overflow: hidden; + } + .signin-stage--desktop { background: transparent; } @@ -14938,108 +14982,67 @@ body:has(.signin-proposed) { display: none; } + .signin-fit--desktop { + z-index: 1; + } + .signin-glow--pin { display: block; - position: fixed; + position: absolute; z-index: 0; + top: calc(var(--signin-frame-top) - 252px * var(--signin-scale)); + left: calc(var(--signin-stage-left) - var(--signin-pull) - 187px * var(--signin-scale)); + width: calc(1814px * var(--signin-scale)); + height: calc(776px * var(--signin-scale)); background-image: url('/signin/glow-desktop.png'); } - .signin-fit--desktop { - z-index: 1; - } - .signin-logo--pin { display: block; - position: fixed; - top: 28px; + position: absolute; + top: calc(var(--signin-frame-top) + 28px * var(--signin-scale)); + left: calc(50px * var(--signin-scale)); z-index: 30; - width: calc(117px * min(100cqw, 100cqh * 1440 / 966) / 1440px); - height: calc(53.929px * min(100cqw, 100cqh * 1440 / 966) / 1440px); + width: calc(117px * var(--signin-scale)); + height: calc(53.929px * var(--signin-scale)); + } + + /* The stage spans the viewport. Slide the laptop group toward the right, + then back left by the spare width so more of the clipped screen shows. */ + .signin-stage--desktop .signin-hero { + position: absolute; + inset: 0; + translate: calc((var(--signin-stage-left) - var(--signin-pull)) / var(--signin-scale)) 0; } /* Same ellipse as Figma, but outside the stage so the blur is not cut off at the stage edge and the purple continues into the right corner. */ .signin-bloom--pin { display: block; - position: fixed; + position: absolute; z-index: 0; - width: calc(368px * min(100cqw, 100cqh * 1440 / 966) / 1440px); - height: calc(480px * min(100cqw, 100cqh * 1440 / 966) / 1440px); - top: calc( - (100cqh - min(100cqh, 100cqw * 966 / 1440)) / 2 - + 224px * min(100cqw, 100cqh * 1440 / 966) / 1440px - ); + top: calc(var(--signin-frame-top) + 224px * var(--signin-scale)); + left: calc(var(--signin-stage-left) - var(--signin-pull) + 882px * var(--signin-scale)); + width: calc(368px * var(--signin-scale)); + height: calc(480px * var(--signin-scale)); border-radius: 50%; background: linear-gradient(to bottom, #2f43cb, #9550c2); - filter: blur(calc(387px * min(100cqw, 100cqh * 1440 / 966) / 1440px)); + filter: blur(calc(387px * var(--signin-scale))); pointer-events: none; } } -/* Pin right only while the window is at least as tall as a 14-inch fullscreen - (1512×982). The spare width there is about 48px, so the laptop still meets - the edge. */ -@media (min-width: 720px) and (aspect-ratio <= 1512 / 982) { - .signin-fit--desktop { - margin-left: auto; - } - - .signin-glow--pin { - top: calc(-252px * 100cqw / 1440px); - left: calc(-187px * 100cqw / 1440px); - width: calc(1814px * 100cqw / 1440px); - height: calc(776px * 100cqw / 1440px); - } - - .signin-logo--pin { - left: calc( - (100cqw - min(100cqw, 100cqh * 1440 / 966)) - + 50px * min(100cqw, 100cqh * 1440 / 966) / 1440px - ); - } - - .signin-bloom--pin { - left: calc( - (100cqw - min(100cqw, 100cqh * 1440 / 966)) - + 882px * min(100cqw, 100cqh * 1440 / 966) / 1440px - ); - } -} - -/* Shorter than a 14-inch window: grow the whole stage to the window width - so the laptop meets the right edge. The bottom of the artboard clips. */ -@media (min-width: 720px) and (aspect-ratio > 1512 / 982) { - .signin-proposed { - align-items: flex-start; - } - - .signin-fit--desktop { - width: 100vw; - height: calc(100vw * 966 / 1440); - margin-left: 0; - } - - .signin-glow--pin { - top: calc(-252px * 100cqw / 1440px); - left: calc(-187px * 100cqw / 1440px); - width: calc(1814px * 100cqw / 1440px); - height: calc(776px * 100cqw / 1440px); - } - - .signin-logo--pin { - top: calc(28px * 100cqw / 1440px); - left: calc(50px * 100cqw / 1440px); - width: calc(117px * 100cqw / 1440px); - height: calc(53.929px * 100cqw / 1440px); +@media (min-width: 720px) and (aspect-ratio >= 1.9) { + .signin-scroll { + overflow-x: hidden; + overflow-y: auto; + scrollbar-width: none; } - .signin-bloom--pin { - top: calc(224px * 100cqw / 1440px); - left: calc(882px * 100cqw / 1440px); - width: calc(368px * 100cqw / 1440px); - height: calc(480px * 100cqw / 1440px); - filter: blur(calc(387px * 100cqw / 1440px)); + .signin-scroll::-webkit-scrollbar { + display: none; + width: 0; + height: 0; } } diff --git a/apps/app/src/pages/LandingPage.tsx b/apps/app/src/pages/LandingPage.tsx index 3934e8985..b74fbd204 100644 --- a/apps/app/src/pages/LandingPage.tsx +++ b/apps/app/src/pages/LandingPage.tsx @@ -150,6 +150,8 @@ export default function LandingPage() { return (
+
+
+
+
) } From 6cfac38779fb020837000ea57ad711035a5398e5 Mon Sep 17 00:00:00 2001 From: ducnmm <165614309+ducnmm@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:59:48 +0700 Subject: [PATCH 58/70] docs: add a guide for importing existing agent memory (#1197) * docs: add a guide for importing existing agent memory Workshop developers asked for a page that copies local memory, standing context, and session transcripts into Walrus Memory in batches, then keeps using that account. * docs: cover import rate limits, timeouts, and Antigravity Hosted mainnet allows 120 weighted requests per minute and 1000 per hour per account, and 60 per minute per delegate key. A bulk timeout means the upload may still be running, so the guide says to poll those job ids instead of sending the batch again. * docs: drop the staging relayer from the import sample Kimberly confirmed staging stays closed to the public, so the sample no longer points readers at relayer-staging. * docs: make the import sample safe to copy Send Claude memory files through analyze, keep only user and assistant text from a session transcript, and retry a refused call after Retry-After without resubmitting job ids the relayer already accepted. * docs: apply the import-guide review --- docs/docs.json | 2 + docs/getting-started/choose-your-path.md | 2 + docs/guides/import-existing-memory.md | 358 +++++++++++++++++++++++ docs/mcp/overview.md | 2 + docs/python-sdk/api-reference.md | 6 +- 5 files changed, 368 insertions(+), 2 deletions(-) create mode 100644 docs/guides/import-existing-memory.md diff --git a/docs/docs.json b/docs/docs.json index 6d2de4989..5b827cb18 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -59,6 +59,7 @@ "group": "Guides", "pages": [ "guides/system-prompt-templates", + "guides/import-existing-memory", "guides/agent-runtimes", "guides/delete-old-memories", "guides/delete-memories-programmatically" @@ -73,6 +74,7 @@ "group": "Guides", "pages": [ "guides/system-prompt-templates", + "guides/import-existing-memory", "guides/returning-user", "guides/manage-your-memory", "guides/delete-old-memories", diff --git a/docs/getting-started/choose-your-path.md b/docs/getting-started/choose-your-path.md index 8cda5dc69..240331a9d 100644 --- a/docs/getting-started/choose-your-path.md +++ b/docs/getting-started/choose-your-path.md @@ -97,3 +97,5 @@ Use Walrus Memory's MCP server when you want Cursor, Claude Desktop, Claude Code - or run the local stdio package with `npx -y @mysten-incubation/memwal-mcp` Go to: [MCP](/mcp/overview) + +If the agent already has local memory, standing context, or session history, copy those facts in first: [Import Your Existing Memory](/guides/import-existing-memory). diff --git a/docs/guides/import-existing-memory.md b/docs/guides/import-existing-memory.md new file mode 100644 index 000000000..f829aa606 --- /dev/null +++ b/docs/guides/import-existing-memory.md @@ -0,0 +1,358 @@ +--- +title: Import Your Existing Memory +description: >- + Copy durable facts from your current session, standing context, and local agent memory + into Walrus Memory in batches, then keep using them from Claude Code, Codex, Cursor, or Antigravity. +keywords: + - import memory + - existing memory + - local memory + - session + - CLAUDE.md + - AGENTS.md + - rememberBulk + - analyze + - onboarding + - Walrus Memory +goal: + description: Extract durable facts from a coding agent's local memory, standing context, and session transcripts, then batch-upload them into the same Walrus Memory namespace the agent will recall from. + requires: + - has_frontmatter: + - title + - description + - keywords + label: Has required frontmatter fields + - min_words: 300 + label: Needs more content depth + - has_questions: true + label: Needs questions for AI search visibility + - has_answer: true + label: Needs answer summary for AI citation +questions: + - How do I import my existing Claude, Codex, Cursor, or Antigravity memory into Walrus Memory? + - How do I batch upload local memories and session history to Walrus Memory? + - Where do Claude Code, Codex, Cursor, and Antigravity store local memory on disk? +answer: >- + Read the durable facts already on disk: local memory files, standing context such as + CLAUDE.md, AGENTS.md, or GEMINI.md, and recent session transcripts. Then write them + into the same namespace your agent recalls from. Use memwal_analyze or analyze for a + Claude memory file and for a transcript. Use memwal_remember_bulk or rememberBulk only for + facts that are already one statement each. + After recall returns one of those facts, new sessions keep writing to Walrus Memory + through the plugin. +--- + +You already have memory on this machine: facts an agent saved locally, standing instructions it reads every session, and the transcript of the session you are in. Walrus Memory starts empty until you write those facts through it. The steps below copy the durable ones across, in batches, so the next session recalls what you already had. + +Connect Walrus Memory in the client first, and sign in, so the upload and the later recall use the same account. See [MCP](/mcp/overview) for Claude Code, Codex, Cursor, and Antigravity. + +## What to copy + +Treat the 3 sources differently. Recall works best on short, self-contained facts. + +| Source | What it holds | What to upload | +| --- | --- | --- | +| Local memory | Facts the agent already decided to keep | Each fact, as its own memory | +| Standing context | `CLAUDE.md`, `AGENTS.md`, `GEMINI.md`, Cursor rules | Durable preferences, decisions, and identity details pulled out of the file | +| Session transcript | The current chat and recent chats | Facts extracted from the transcript. The SDK can stamp the session time onto those facts. | + +Leave repository instructions, tool routing, and one-off task notes in the original files. Recall cannot match cleanly against a whole `CLAUDE.md` or a raw transcript stored as one memory. + +Write into the namespace your client already uses. With no `MEMWAL_NAMESPACE` and no `--namespace`, that is `default`. A fact imported into another namespace stays invisible to the agent that recalls `default`. + + +The MCP tools strip obvious credentials before a write. Treat that as a last check, not a guarantee. The SDK stores the string you pass, and `analyze` sends that string to the relayer's model. Read each passage before an SDK upload, and leave out passwords, API keys, tokens, private keys, and seed phrases. + + +## Where the files are + +Paths below are the ones these clients write on a developer machine. The project directory is encoded by turning `/` into `-`. Claude Code turns the leading `/` into a leading `-`, so `/Users/me/app` becomes `-Users-me-app`. Cursor drops the leading slash, so the same path becomes `Users-me-app`. A folder named `empty-window` is a Cursor window with no project open. Transcripts for a project stay under its encoded path. + +### Claude Code + +Local memory for one project lives in `~/.claude/projects//memory/`. + +- `MEMORY.md` is an index of links. Skip it, or you store the index on top of the facts. +- Every other `*.md` file is one memory. YAML frontmatter carries `name` and `description`. The body under the frontmatter often has **Why:**, **How to apply:**, and `[[links]]`. Send that body through `analyze` rather than storing it whole. + +Session transcripts are `~/.claude/projects//*.jsonl`. A turn has `type` of `user` or `assistant`, and a `timestamp`. The words are in `message.content`: a string, or blocks whose `type` is `text`. Skip `tool_use`, `tool_result`, and `thinking` blocks. + +### Codex + +Standing context is `~/.codex/AGENTS.md` plus `AGENTS.md` in the repo. + +Where a Codex install has written consolidated memory, it is in `~/.codex/memories_1.sqlite`, table `stage1_outputs`, column `raw_memory`. When that table has rows, each `raw_memory` value is one thread's summary. + +Session transcripts are `~/.codex/sessions/YYYY/MM/DD/rollout-*.jsonl`. Each line has `timestamp` and `payload`. User text is a line whose `type` is `response_item` and whose `payload.role` is `user`; the words are in `payload.content` blocks of type `input_text`. Assistant text is the same line with `payload.role` of `assistant` and blocks of type `output_text`. Skip every other line. + +### Cursor + +Project rules live in `.cursor/rules/`. User rules live in Cursor Settings. + +Transcripts live in `~/.cursor/projects//agent-transcripts//.jsonl`. Each line has `role` and `message`. The words are in `message.content` blocks of type `text`. + +### Antigravity + +Standing context is `GEMINI.md` and `AGENTS.md` in the project. Install the plugin with: + +```bash +npx degit MystenLabs/MemWal/packages/mcp/plugin ~/.gemini/config/plugins/memwal +``` + +See [Antigravity](/mcp/antigravity). + +## Ask the agent to upload + +Once `memwal_remember_bulk` and `memwal_analyze` are available, paste this into the client that already holds the session. The agent can see the current conversation, and it can read the files above. + +```text +Import my existing memory into Walrus Memory. Use the namespace this client already recalls from. + +1. Local memory. Read the memory files for this project (Claude: ~/.claude/projects//memory/*.md except MEMORY.md; Codex: raw_memory rows in ~/.codex/memories_1.sqlite; Cursor: .cursor/rules and user rules). Send each Claude memory file body through memwal_analyze, one call per chunk of at most 64 KiB. A Codex raw_memory value, or a Cursor rule you have already reduced to one statement, goes through memwal_remember_bulk, at most 20 per call. Wait for that call to finish before the next. +2. Standing context. Read CLAUDE.md, AGENTS.md, GEMINI.md, and Cursor rules. Extract only durable preferences, decisions, constraints, and identity details. Upload those with memwal_remember_bulk, one statement per entry. Leave tool instructions and repo conventions in the files. +3. This session and other recent sessions I name. Keep only the user and assistant text. Claude: message.content text, not tool_use, tool_result, or thinking. Codex: input_text and output_text. Cursor: message.content blocks of type text. Extract durable facts from that text with memwal_analyze, one call per chunk of at most 64 KiB. This tool has no session-time parameter, so include the date in the fact when it matters. Skip one-off tasks, the current bug, and small talk. +4. Skip anything that looks like a secret. +5. If a tool replies ACCEPTED, NOT YET SAVED, call memwal_remember_status with those job_ids and wait until they are saved. If that wait times out, call memwal_remember_status again with the same job_ids. Do not upload those facts again. On HTTP 429, wait for Retry-After, then continue. Then call memwal_recall with a fact you just imported and show me the match. +6. Do not run this import a second time. memwal_remember_bulk has no idempotency key, so a second run stores each fact again. memwal_analyze collapses onto a stored fact only when the new sentence is identical. +``` + +`memwal_remember_bulk` takes 1 to 20 strings. `memwal_analyze` takes one passage of at most 64 KiB and saves each extracted fact as its own memory. It does not accept a session time. See [MCP Reference](/mcp/reference) for the parameters. + +## Batch upload with the SDK + +Use the SDK when you want to run the copy yourself. `analyze` is for a Claude memory file and for a transcript: the relayer extracts facts, then stores each one. `rememberBulk` is only for text you have already reduced to one statement per item, at most 20 items per call. `analyze` accepts at most 64 KiB and returns at most 20 facts per call, so walk a long passage in chunks of about 60,000 bytes. A dense chunk can still drop facts past that cap of 20. + +On hosted Mainnet, requests per minute, requests per hour, and delegate-key requests per minute are deployment settings (`RATE_LIMIT_REQUESTS_PER_MINUTE`, `RATE_LIMIT_REQUESTS_PER_HOUR`, and `RATE_LIMIT_DELEGATE_KEY_PER_MINUTE`). This repo does not record the hosted values. A self-hosted relayer with those 3 variables unset uses the code defaults in `services/server/src/rate_limit.rs`: 60 per minute, 500 per hour, and 30 per minute per key. + +| Call | Cost | +| --- | --- | +| `rememberBulk` | 10 | +| `analyze` | 5, plus 1 per extracted fact | +| Status poll | 1 | + +Send one chunk, wait until it settles, then send the next. On HTTP 429, wait for the `Retry-After` header before retrying the call that was refused. The hourly refusal asks for 300 seconds. The hourly account cap is what stops a long import. + +`rememberBulk` and `memwal_remember_bulk` send no idempotency key. Running the script a second time stores each bulk fact again. `analyze` collapses onto the copy already stored only when the extracted fact text is the same for that owner and namespace. A reworded sentence is stored again. + +Pass `occurredAt` as the session time for a transcript. Leave it unset for a memory file. The extractor writes that date into the fact text, which is how a relative date such as `yesterday` in an old transcript stays attached to the day it was said. Walrus Memory does not keep a separate event-time field you can filter on. + +```ts +// Run with: npx tsx import.ts (ESM; top-level await) +import { readFile, readdir } from "node:fs/promises"; +import { MemWal } from "@mysten-incubation/memwal"; + +// MEMWAL_PRIVATE_KEY, MEMWAL_ACCOUNT_ID +// CLAUDE_MEMORY_DIR: directory of Claude memory *.md files +// SESSION_FILE: a session .jsonl, or conversation text you already reduced +// SESSION_TIME: optional RFC 3339 time of that session, e.g. 2026-10-01T09:00:00Z + +const memwal = MemWal.create({ + key: process.env.MEMWAL_PRIVATE_KEY!, + accountId: process.env.MEMWAL_ACCOUNT_ID!, + serverUrl: "https://relayer.memory.walrus.xyz", + namespace: process.env.MEMWAL_NAMESPACE ?? "default", +}); + +function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +// Hourly refusals send Retry-After: 300, which is longer than the 120s wait. +function retryAfterMs(err: unknown): number { + const header = (err as { retryAfterSeconds?: number }).retryAfterSeconds; + if (typeof header === "number" && header > 0) return header * 1000; + const cause = (err as { cause?: unknown }).cause; + if (typeof cause === "string") { + try { + const seconds = Number(JSON.parse(cause).retry_after_seconds); + if (Number.isFinite(seconds) && seconds > 0) return seconds * 1000; + } catch { + // body was not the rate-limit JSON + } + } + return 300_000; +} + +function isRateLimited(err: unknown): boolean { + return ( + (err as { status?: number }).status === 429 || + (err as { name?: string }).name === "MemWalRateLimited" + ); +} + +// A 429 here means the call was not accepted. Retry that same call. +async function accept(call: () => Promise): Promise { + for (;;) { + try { + return await call(); + } catch (err) { + if (!isRateLimited(err)) throw err; + const wait = retryAfterMs(err); + console.log(`429; waiting ${Math.round(wait / 1000)}s, then retrying this call`); + await sleep(wait); + } + } +} + +// Job ids already returned. Poll again. Do not send the text again. +async function confirm(jobIds: string[]) { + let pending = jobIds; + const deadline = Date.now() + 26 * 60 * 1000; + while (pending.length > 0 && Date.now() < deadline) { + try { + const result = await memwal.waitForRememberJobs(pending); + const still = result.results + .filter((item) => item.status === "timeout") + .map((item) => item.id); + console.log(`saved ${result.succeeded}/${result.total}`); + for (const item of result.results) { + if (item.status === "failed") console.log(`failed ${item.id}: ${item.error}`); + } + if (still.length === 0) return; + console.log("still running; polling these job ids again"); + pending = still; + } catch (err) { + if (!isRateLimited(err)) throw err; + const ids = (err as { jobIds?: string[] }).jobIds; + if (ids && ids.length > 0) pending = ids; + const wait = retryAfterMs(err); + console.log( + `429 while confirming; waiting ${Math.round(wait / 1000)}s, then polling the same job ids`, + ); + await sleep(wait); + } + } + if (pending.length > 0) { + console.log("still running; poll these job ids later, do not send them again"); + console.log(pending.join("\n")); + } +} + +function factFromMemoryFile(markdown: string): string { + return markdown.replace(/^---[\s\S]*?---\s*/, "").trim(); +} + +function textFromContent(content: unknown): string { + if (typeof content === "string") return content.trim(); + if (!Array.isArray(content)) return ""; + const parts: string[] = []; + for (const block of content) { + if (!block || typeof block !== "object") continue; + const rec = block as { type?: string; text?: string }; + if (rec.type !== "text" && rec.type !== "input_text" && rec.type !== "output_text") continue; + if (typeof rec.text === "string" && rec.text.trim()) parts.push(rec.text.trim()); + } + return parts.join("\n"); +} + +// Keep user and assistant words. Drop tool output. Plain text passes through. +function dialogueText(raw: string): string { + const kept: string[] = []; + let jsonLines = 0; + for (const line of raw.split("\n")) { + const trimmed = line.trim(); + if (!trimmed.startsWith("{")) continue; + let row: { + type?: string; + role?: string; + message?: { content?: unknown }; + payload?: { role?: string; content?: unknown }; + }; + try { + row = JSON.parse(trimmed); + } catch { + continue; + } + jsonLines += 1; + let text = ""; + if (row.type === "user" || row.type === "assistant") { + text = textFromContent(row.message?.content); + } else if ( + row.type === "response_item" && + (row.payload?.role === "user" || row.payload?.role === "assistant") + ) { + text = textFromContent(row.payload?.content); + } else if (row.role === "user" || row.role === "assistant") { + text = textFromContent(row.message?.content); + } + if (text) kept.push(text); + } + if (jsonLines === 0) return raw.trim(); + return kept.join("\n\n"); +} + +function transcriptChunks(text: string): string[] { + const buf = Buffer.from(text); + const chunks: string[] = []; + let offset = 0; + while (offset < buf.length) { + let end = Math.min(offset + 60_000, buf.length); + while (end > offset && end < buf.length && (buf[end] & 0xc0) === 0x80) end--; + if (end === offset) end = Math.min(offset + 60_000, buf.length); + const chunk = buf.subarray(offset, end).toString("utf8").trim(); + if (chunk) chunks.push(chunk); + offset = end; + } + return chunks; +} + +// One statement per string. A Claude memory file does not belong here. +async function uploadFacts(facts: string[]) { + for (let i = 0; i < facts.length; i += 20) { + const batch = facts.slice(i, i + 20).map((text) => ({ text })); + const accepted = await accept(() => memwal.rememberBulk(batch)); + await confirm(accepted.job_ids); + } +} + +async function analyzePassage(text: string, occurredAt?: string) { + const accepted = await accept(() => + memwal.analyze(text, occurredAt ? { occurredAt } : undefined), + ); + console.log(`extracted ${accepted.facts.length}`); + if (accepted.job_ids.length > 0) await confirm(accepted.job_ids); +} + +// Claude Code local memory for one project. Skip the MEMORY.md index. +const memoryDir = process.env.CLAUDE_MEMORY_DIR!; +const files = (await readdir(memoryDir)).filter( + (name) => name.endsWith(".md") && name !== "MEMORY.md", +); +for (const name of files) { + const body = factFromMemoryFile(await readFile(`${memoryDir}/${name}`, "utf8")); + for (const chunk of transcriptChunks(body)) await analyzePassage(chunk); +} + +// The server rejects analyze text over 64 KiB (65536 bytes). Each call keeps at most 20 facts. +const dialogue = dialogueText(await readFile(process.env.SESSION_FILE!, "utf8")); +if (!dialogue) { + console.log("no user or assistant text in the session file"); +} else { + for (const chunk of transcriptChunks(dialogue)) { + await analyzePassage(chunk, process.env.SESSION_TIME); + } +} +``` + +The sample splits the accept from the wait. `rememberBulk` and `analyze` throw on HTTP 429, and that call was not stored, so the sample sleeps for `Retry-After` and retries the same call. A 429 after job ids have come back, or a `timeout` while a congested upload is still running (about 25 minutes), polls those ids again. Sending them again pays for a second blob. `waitForRememberJobs` itself stops after 120 seconds, which is shorter than the hourly `Retry-After` of 300 seconds, so the sample sleeps that long on its own. + +The Python SDK has the same split: `remember_bulk` or `analyze`, then `wait_for_remember_jobs`. Wrapping `remember_bulk_and_wait` or `analyze_and_wait` in a retry resubmits the batch when the 429 came after the relayer had already accepted it. `occurred_at` is the session time. See the [Python API reference](/python-sdk/api-reference). + +Codex `raw_memory` text and a Cursor rule you have already reduced to one statement per item go through `uploadFacts`. A Codex or Cursor transcript goes through `dialogueText`, then `analyzePassage`. + + +Confirm one fact before you copy a large history. `recall({ query: "", limit: 3 })` should return that fact. The local files stay on disk; this copy does not remove them. + + +## Keep using what you imported + +Install the plugin for the client you work in, so later sessions write to Walrus Memory as you go: + +- [Claude Code](/mcp/claude-code) +- [Codex](/mcp/codex) +- [Cursor](/mcp/cursor) +- [Antigravity](/mcp/antigravity) + +On Claude Code, merge the Walrus Memory block into `~/.claude/CLAUDE.md` and turn off built-in memory, or the agent keeps writing `MEMORY.md` beside Walrus Memory. That step is in the Claude Code page. + +Open a new session and ask for something only the imported fact would answer. When recall returns it, the account is carrying the memory you already had. diff --git a/docs/mcp/overview.md b/docs/mcp/overview.md index 2e0e734d1..0ac200db7 100644 --- a/docs/mcp/overview.md +++ b/docs/mcp/overview.md @@ -34,6 +34,8 @@ answer: >- The **MemWal MCP server** exposes your portable Walrus Memory as Model Context Protocol tools, so an AI agent can decide when to save and recall memories on its own. It works with any MCP-aware client, and on **Claude Code**, **Codex**, **Cursor**, and **Antigravity** it can be installed as a **plugin** that adds automatic memory through lifecycle hooks. +If you already have local memory, standing context, or session history, [import those facts](/guides/import-existing-memory) so the account starts with what you have. + ## MCP vs Plugin There are two ways to use MemWal. The difference is whether you also get the **lifecycle hooks**: diff --git a/docs/python-sdk/api-reference.md b/docs/python-sdk/api-reference.md index 81d65361f..2050abfb1 100644 --- a/docs/python-sdk/api-reference.md +++ b/docs/python-sdk/api-reference.md @@ -152,10 +152,12 @@ RecallResult( `distance` is cosine distance — lower is more similar. -### `analyze(text, namespace=None) -> AnalyzeResult` +### `analyze(text, namespace=None, occurred_at=None) -> AnalyzeResult` Extract memorable facts via an LLM and enqueue one background remember job per fact. +`occurred_at` is optional. Pass a timezone-aware `datetime`, or an RFC 3339 / ISO-8601 string with a `Z` suffix or a UTC offset. A naive `datetime`, or a string with no offset, raises `ValueError`. Any other type raises `TypeError`. When set, the extractor uses it as a temporal anchor and writes resolved dates into the fact text. The server stores no separate event-time field. Omit it when you have no session time; the server does not fill in the current time. + ```python AnalyzeResult( facts: list[AnalyzedFact], # AnalyzedFact(text, id, blob_id) @@ -166,7 +168,7 @@ AnalyzeResult( ) ``` -`AnalyzeResult.total` is a backward-compat alias for `fact_count`. Use `analyze_and_wait(text, namespace=None, opts=None) -> AnalyzeWaitResult` to wait for every fact job to settle. +`AnalyzeResult.total` is a backward-compat alias for `fact_count`. Use `analyze_and_wait(text, namespace=None, opts=None, occurred_at=None) -> AnalyzeWaitResult` to wait for every fact job to settle. `occurred_at` means the same thing as on `analyze`. ### `ask(question, limit=5, namespace=None) -> AskResult` From 11d8016bc83f05c5f80d039dc82cc723c37b527f Mon Sep 17 00:00:00 2001 From: ducnmm <165614309+ducnmm@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:08:24 +0700 Subject: [PATCH 59/70] fix(sdk): drop the dead Walrus import and split recallManual results (#1222) /manual still dynamically imported @mysten/walrus for a client nothing called, which pulled the WASM package into Turbopack and Vite. Downloads stay on the HTTP aggregator. MemWal.recallManual returns RecallManualHitResult. MemWalManual.recallManual returns RecallManualMemoryResult. RecallManualResult stays as a deprecated alias of those two, not a mixed list. WALM-145, WALM-163 --- docs/sdk/api-reference.md | 6 +-- docs/sdk/changelog.mdx | 2 + packages/sdk/CHANGELOG.md | 2 + packages/sdk/src/index.ts | 2 + packages/sdk/src/manual-entry.ts | 4 +- packages/sdk/src/manual.ts | 73 ++++---------------------------- packages/sdk/src/memwal.ts | 8 ++-- packages/sdk/src/types.ts | 18 ++++++-- 8 files changed, 39 insertions(+), 76 deletions(-) diff --git a/docs/sdk/api-reference.md b/docs/sdk/api-reference.md index 888f81507..b9c90abc9 100644 --- a/docs/sdk/api-reference.md +++ b/docs/sdk/api-reference.md @@ -395,7 +395,7 @@ These exist on the `MemWal` class for advanced use cases: | Method | Description | |--------|-------------| | `rememberManual({ encryptedData, vector, namespace? })` | Send SEAL-encrypted bytes + a pre-computed vector; the relayer uploads to Walrus | -| `recallManual({ vector, limit?, namespace? })` | Search with a pre-computed query vector (returns blob IDs, no decryption) | +| `recallManual({ vector, limit?, namespace? })` | Search with a pre-computed query vector. Returns `RecallManualHitResult`: blob IDs and distances, no decrypted text | | `embed(text)` | Generate an embedding vector for text (no storage) | ## `MemWalManual` @@ -410,9 +410,9 @@ See [MemWalManual usage](/sdk/usage/memwal-manual) for the full setup and flow d Embed locally, SEAL encrypt locally, send encrypted payload + vector to relayer for Walrus upload and vector registration. -### `recallManual(query, limit?, namespace?): Promise` +### `recallManual(query, limit?, namespace?): Promise` -Embed locally, search via relayer, download from Walrus, SEAL decrypt locally. Returns decrypted text results. +Embed locally, search via relayer, download from Walrus, SEAL decrypt locally. Returns decrypted text results. This is not the hit-only `RecallManualHitResult` returned by `MemWal.recallManual()`. - `recallManual(query, { limit?, namespace?, scoringWeights? })` is valid - `recallManual(query, namespace)` throws `TypeError`; namespace is not a valid second argument diff --git a/docs/sdk/changelog.mdx b/docs/sdk/changelog.mdx index fa32e7879..0dce880f6 100644 --- a/docs/sdk/changelog.mdx +++ b/docs/sdk/changelog.mdx @@ -36,6 +36,8 @@ answer: >- ### Fixed - `getRememberStatus` and `waitForRememberJob` reject a job id outside `[A-Za-z0-9_-]+` before signing. `.`, `..`, `\`, and `%2e%2e` used to pass a denylist and be rewritten by `fetch` after the path was signed. Valid UUID job ids are unchanged. (#1087, #1216) +- `/manual` no longer imports `@mysten/walrus`. `getWalrusClient()` and `walrusUpload()` had no callers, but the dynamic import still pulled the WASM client into Turbopack and Vite. Downloads stay on the HTTP aggregator. (WALM-145, #330) +- `MemWal.recallManual()` is typed as `RecallManualHitResult` (blob id and distance, no text) and `MemWalManual.recallManual()` as `RecallManualMemoryResult` (decrypted text). `RecallManualResult` remains a deprecated alias of those two result types, not a mixed-item array. (WALM-163, #298) ## 0.1.9 diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index 9a44f43a1..41d5dae0f 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -6,6 +6,8 @@ - `createAccount`, `addDelegateKey`, `removeDelegateKey`, and `MemWalManual` work again without a caller-supplied `suiClient`. They looked for `SuiClient` in `@mysten/sui/client`, which current `@mysten/sui` no longer exports, and threw `SuiClient not found`. The default is now `SuiGrpcClient` against the public fullnode for `suiNetwork`, because those fullnodes no longer serve JSON-RPC. An explicit `suiClient` still wins. (WALM-164, #300) - `getRememberStatus` and `waitForRememberJob` reject a job id outside `[A-Za-z0-9_-]+` before signing. `.`, `..`, `\`, and `%2e%2e` used to pass a denylist and be rewritten by `fetch` after the path was signed. Valid UUID job ids are unchanged. (#1087, #1216) +- `/manual` no longer imports `@mysten/walrus`. `getWalrusClient()` and `walrusUpload()` had no callers, but the dynamic import still pulled the WASM client into Turbopack and Vite. Downloads stay on the HTTP aggregator. (WALM-145, #330) +- `MemWal.recallManual()` is typed as `RecallManualHitResult` (blob id and distance, no text) and `MemWalManual.recallManual()` as `RecallManualMemoryResult` (decrypted text). `RecallManualResult` remains a deprecated alias of those two result types, not a mixed-item array. (WALM-163, #298) - `MemWalMock.listNamespaces` rejects a non-positive or non-integer `limit` with `limit must be positive`, matching the relayer. `limit: 0` used to return `has_more: true` and `next_cursor: null`, so the documented pagination loop never finished, and `limit: -1` silently dropped the last namespace. (#1144, #1145) - `MemWalMock.rememberBulkAsync` rejects an empty or non-array batch with the same error as `MemWal`. (#1146) - `MemWalMock` honors `idempotencyKey`. A retry of the same text and namespace returns the original job, a different payload throws, and `forget` / `clear` release the key so the text can be written again. (#1147) diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 8d90f0169..902d83297 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -80,6 +80,8 @@ export type { RememberManualOptions, RememberManualResult, RecallManualOptions, + RecallManualHitResult, + RecallManualMemoryResult, RecallManualResult, RecallManualHit, MinSupportedSdk, diff --git a/packages/sdk/src/manual-entry.ts b/packages/sdk/src/manual-entry.ts index 650d29836..87d9bef92 100644 --- a/packages/sdk/src/manual-entry.ts +++ b/packages/sdk/src/manual-entry.ts @@ -2,7 +2,7 @@ * @mysten-incubation/memwal/manual * * Manual (client-side) mode entry point. - * Requires: @mysten/seal, @mysten/walrus, @mysten/sui + * Requires: @mysten/seal, @mysten/sui. Walrus downloads use HTTP, not @mysten/walrus. * * Usage: * import { MemWalManual } from "@mysten-incubation/memwal/manual"; @@ -25,6 +25,8 @@ export type { RememberManualResult, RecallManualOptions, MemWalManualRecallOptions, + RecallManualHitResult, + RecallManualMemoryResult, RecallManualResult, RecallManualHit, RecallManualMemory, diff --git a/packages/sdk/src/manual.ts b/packages/sdk/src/manual.ts index 506baabaf..b7e09517d 100644 --- a/packages/sdk/src/manual.ts +++ b/packages/sdk/src/manual.ts @@ -3,7 +3,7 @@ * * User-side flow where the SDK handles everything locally: * - SEAL encrypt/decrypt via @mysten/seal (user's own Sui wallet) - * - Walrus upload/download via @mysten/walrus + * - Walrus upload is done by the relayer; downloads use the HTTP aggregator * - Embedding via OpenAI-compatible API (user's own key) * - Vector registration via Walrus Memory server (Ed25519 signed) * @@ -32,7 +32,7 @@ import type { MemWalManualConfig, WalletSigner, RememberManualResult, - RecallManualResult, + RecallManualMemoryResult, RecallManualMemory, MemWalManualRecallOptions, RestoreResult, @@ -175,7 +175,6 @@ export class MemWalManual { // Lazily initialized heavy clients (typed as any to avoid peer dep compile errors) private _suiClient: any = null; private _sealClient: any = null; - private _walrusClient: any = null; private _keypair: any = null; private constructor(config: MemWalManualConfig) { @@ -215,7 +214,7 @@ export class MemWalManual { /** * Create a new MemWalManual client. * - * Requires peer dependencies: @mysten/sui, @mysten/seal, @mysten/walrus + * Requires peer dependencies: @mysten/sui, @mysten/seal * * @param config.key - Ed25519 delegate private key (hex) for server auth * @param config.suiPrivateKey - Sui private key (bech32) for SEAL + Walrus (OR walletSigner) @@ -396,28 +395,6 @@ export class MemWalManual { return totalWeight > 0 ? Math.min(2, totalWeight) : 2; } - private async getWalrusClient() { - if (!this._walrusClient) { - // @ts-ignore — optional peer dependency - const { WalrusClient } = await import("@mysten/walrus"); - const suiClient = await this.getSuiClient(); - const network = this.config.suiNetwork ?? "mainnet"; - const uploadRelayHost = - network === "testnet" - ? "https://upload-relay.testnet.walrus.space" - : "https://upload-relay.mainnet.walrus.space"; - this._walrusClient = new WalrusClient({ - network: network as any, - suiClient, - uploadRelay: { - host: uploadRelayHost, - sendTip: { max: 10_000_000 }, - }, - }); - } - return this._walrusClient; - } - // ============================================================ // Core Manual API // ============================================================ @@ -460,13 +437,13 @@ export class MemWalManual { * `recallManual(query, { limit?, namespace? })` is valid. * `recallManual(query, namespace)` throws `TypeError`. */ - async recallManual(query: string, options: MemWalManualRecallOptions): Promise; - async recallManual(query: string, limit?: number, namespace?: string): Promise; + async recallManual(query: string, options: MemWalManualRecallOptions): Promise; + async recallManual(query: string, limit?: number, namespace?: string): Promise; async recallManual( query: string, limitOrOptions: number | MemWalManualRecallOptions = 10, namespace?: string - ): Promise { + ): Promise { if (!query.trim()) throw new Error("Query cannot be empty"); const options = resolveLimitOrOptions( @@ -824,45 +801,11 @@ export class MemWalManual { } // ============================================================ - // Internal: Walrus Upload/Download + // Internal: Walrus download // ============================================================ - private async walrusUpload(data: Uint8Array): Promise { - // Direct HTTP PUT to Walrus publisher (works in both browser and Node.js, - // unlike @mysten/walrus SDK which uses WASM and requires Node.js) - const network = this.config.suiNetwork ?? "mainnet"; - const defaultPublisher = - network === "testnet" - ? "https://publisher.walrus-testnet.walrus.space" - : "https://publisher.walrus-mainnet.walrus.space"; - const publisherUrl = this.config.walrusPublisherUrl ?? defaultPublisher; - const epochs = this.config.walrusEpochs ?? 50; - - const resp = await fetch(`${publisherUrl}/v1/blobs?epochs=${epochs}&deletable=true`, { - method: "PUT", - headers: { "Content-Type": "application/octet-stream" }, - body: data as unknown as BodyInit, - }); - - if (!resp.ok) { - const errText = await resp.text(); - throw new Error(`Walrus upload failed (${resp.status}): ${errText}`); - } - - const result = (await resp.json()) as any; - // Response can be { newlyCreated: { blobObject: { blobId } } } - // or { alreadyCertified: { blobId } } - const blobId = result.newlyCreated?.blobObject?.blobId ?? result.alreadyCertified?.blobId; - - if (!blobId) { - throw new Error(`Walrus upload: unexpected response: ${JSON.stringify(result)}`); - } - return blobId; - } - private async walrusDownload(blobId: string): Promise { - // Direct HTTP fetch to Walrus aggregator (works in both browser and Node.js, - // unlike @mysten/walrus SDK which requires Node.js APIs) + // Direct HTTP fetch to the Walrus aggregator. Upload goes through the relayer. const network = this.config.suiNetwork ?? "mainnet"; const defaultAggregator = network === "testnet" diff --git a/packages/sdk/src/memwal.ts b/packages/sdk/src/memwal.ts index 59225fa2a..5c266ae19 100644 --- a/packages/sdk/src/memwal.ts +++ b/packages/sdk/src/memwal.ts @@ -43,7 +43,7 @@ import type { RememberManualOptions, RememberManualResult, RecallManualOptions, - RecallManualResult, + RecallManualHitResult, RestoreResult, NamespacesResult, ListNamespacesOptions, @@ -1266,7 +1266,7 @@ export class MemWal { * @param opts.vector - Pre-computed query embedding vector * @param opts.limit - Max results (default: 10) * @param opts.scoringWeights - Optional composite-scoring weights - * @returns RecallManualResult with blob_id + distance pairs (no decrypted text) + * @returns RecallManualHitResult with blob_id + distance pairs (no decrypted text) * * @example * ```typescript @@ -1284,14 +1284,14 @@ export class MemWal { * } * ``` */ - async recallManual(opts: RecallManualOptions): Promise { + async recallManual(opts: RecallManualOptions): Promise { const limit = opts.limit !== undefined ? opts.limit : 10; if (typeof limit !== "number" || !Number.isInteger(limit) || limit <= 0) { throw new TypeError( `limit must be a positive integer, got ${limit}`, ); } - return this.signedRequest( + return this.signedRequest( "POST", "/api/recall/manual", { diff --git a/packages/sdk/src/types.ts b/packages/sdk/src/types.ts index 4f5cc5e3c..4d455ecf4 100644 --- a/packages/sdk/src/types.ts +++ b/packages/sdk/src/types.ts @@ -721,12 +721,24 @@ export interface RecallManualMemory { distance: number; } -/** Result from recallManual() — full client-side variant with decrypted text */ -export interface RecallManualResult { - results: (RecallManualHit | RecallManualMemory)[]; +/** `MemWal.recallManual()` — blob id and distance, no decrypted text. */ +export interface RecallManualHitResult { + results: RecallManualHit[]; total: number; } +/** `MemWalManual.recallManual()` — decrypted memories. */ +export interface RecallManualMemoryResult { + results: RecallManualMemory[]; + total: number; +} + +/** + * @deprecated Use {@link RecallManualHitResult} or {@link RecallManualMemoryResult}. + * Each method returns one of those, not a mixed list of hits and memories. + */ +export type RecallManualResult = RecallManualHitResult | RecallManualMemoryResult; + // ============================================================ // Account Management Types // ============================================================ From 002539d71d37dd6af7a3e96a9aea989ecd6c7d13 Mon Sep 17 00:00:00 2001 From: Nikola Le <91601109+nikola0x0@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:07:05 +0700 Subject: [PATCH 60/70] fix(server): hold back migration 025 until the next release (#1225) Keep the legacy (owner, idempotency_key) unique index for this release so a relayer rollback stays safe. The (owner, namespace, idempotency_key) index from 013 and the WALM-781 code still ship; 025 returns in the release after. Co-authored-by: Le Tien Phat <91601109+Niko1444@users.noreply.github.com> --- .../migrations/025_drop_owner_idempotency_key_index.sql | 7 ------- services/server/src/storage/db.rs | 2 -- 2 files changed, 9 deletions(-) delete mode 100644 services/server/migrations/025_drop_owner_idempotency_key_index.sql diff --git a/services/server/migrations/025_drop_owner_idempotency_key_index.sql b/services/server/migrations/025_drop_owner_idempotency_key_index.sql deleted file mode 100644 index c01d37237..000000000 --- a/services/server/migrations/025_drop_owner_idempotency_key_index.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Drop the legacy account-wide idempotency index (WALM-781). --- Idempotency keys are now scoped per (owner, namespace) by --- uq_remember_jobs_owner_namespace_idempotency_key (migration 013). --- --- Own file: DROP INDEX CONCURRENTLY cannot run inside a transaction block --- and avoids taking ACCESS EXCLUSIVE lock on remember_jobs at boot. -DROP INDEX CONCURRENTLY IF EXISTS uq_remember_jobs_owner_idempotency_key; diff --git a/services/server/src/storage/db.rs b/services/server/src/storage/db.rs index d9e5e7ef2..33a58bbd0 100644 --- a/services/server/src/storage/db.rs +++ b/services/server/src/storage/db.rs @@ -160,8 +160,6 @@ const MIGRATIONS_AFTER_INDEX_RECOVERY: &[Migration] = &[ // deleted_at range index for the admin activity delete counts. // CONCURRENTLY, own file -- see 024's header. migration!("024_memory_tombstones_deleted_at_index.sql"), - // Drop the legacy account-wide idempotency index (WALM-781). - migration!("025_drop_owner_idempotency_key_index.sql"), ]; /// Every migration the pipeline applies, in the order it applies them. From 1a17c24f549719db259b7c574d2676e8a358ec3f Mon Sep 17 00:00:00 2001 From: ducnmm <165614309+ducnmm@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:28:24 +0700 Subject: [PATCH 61/70] fix(indexer): decode checkpoints carrying the Validity expiration (WALM-668) (#1224) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit testnet-v1.79.0 added `TransactionExpiration::Validity` as variant index 3. The indexer was pinned to testnet-v1.75.1, whose `TransactionExpiration` has three variants, so once testnet began producing `Validity` transactions every checkpoint carrying one failed to convert: transaction.bcs: invalid value: integer 3, expected variant index 0 <= i < 3 The indexer decodes whole checkpoints rather than individual events, so this is a hard stop and not a skipped field: `accounts_v1` retried the same checkpoint indefinitely and its watermark backlog grew to 48,527 pending commits on staging, leaving GET /api/accounts/{owner}/exists answering false for accounts that exist on-chain. Move both Sui pins to testnet-v1.80.0 (ceaaff1c), the current testnet release. The remote store layout is unchanged between the two revisions (same `{checkpoint}.binpb.zst` objects, same zstd + protobuf decode), so this needs no ingestion or deployment configuration change and the saved watermark stays valid. Cover it two ways: `tests/affected_checkpoint.rs` decodes testnet checkpoint 384534623 — the one named in the incident, committed as a fixture in the exact form the remote store serves — through the real `StoreIngestionClient`, and `tests/checkpoint_decoding.rs` pins `Validity` to variant index 3 while holding the earlier variants at their own indices, which already-indexed history depends on. (cherry picked from commit ca3f72d5e96e001c7cda67e43df321739b995696) (cherry picked from commit 69908acc81fce294309d3c85b8ce9db5c87f404f) Co-authored-by: Harry Phan --- services/indexer/Cargo.lock | 1607 ++++++++++------- services/indexer/Cargo.toml | 20 +- services/indexer/tests/affected_checkpoint.rs | 57 + services/indexer/tests/checkpoint_decoding.rs | 82 + .../tests/fixtures/384534623.binpb.zst | Bin 0 -> 9663 bytes 5 files changed, 1131 insertions(+), 635 deletions(-) create mode 100644 services/indexer/tests/affected_checkpoint.rs create mode 100644 services/indexer/tests/checkpoint_decoding.rs create mode 100644 services/indexer/tests/fixtures/384534623.binpb.zst diff --git a/services/indexer/Cargo.lock b/services/indexer/Cargo.lock index e8a5af002..5fa6b0504 100644 --- a/services/indexer/Cargo.lock +++ b/services/indexer/Cargo.lock @@ -94,9 +94,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -116,11 +116,17 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android_system_properties" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" dependencies = [ "libc", ] @@ -128,7 +134,7 @@ dependencies = [ [[package]] name = "anemo" version = "0.0.0" -source = "git+https://github.com/mystenlabs/anemo.git?rev=b1bda3f8555c5594a4966e850170cc11ecfcf557#b1bda3f8555c5594a4966e850170cc11ecfcf557" +source = "git+https://github.com/mystenlabs/anemo.git?rev=68adc31d4ea1f4573f08c2c75317fcb205b823de#68adc31d4ea1f4573f08c2c75317fcb205b823de" dependencies = [ "anyhow", "async-trait", @@ -143,7 +149,7 @@ dependencies = [ "pkcs8", "quinn", "quinn-proto", - "rand 0.8.7", + "rand 0.8.8", "rcgen", "ring", "rustls", @@ -154,7 +160,7 @@ dependencies = [ "tap", "thiserror 1.0.69", "tokio", - "tokio-util", + "tokio-util 0.7.19", "tower 0.4.13", "tracing", "x509-parser", @@ -163,7 +169,7 @@ dependencies = [ [[package]] name = "anemo-tower" version = "0.0.0" -source = "git+https://github.com/mystenlabs/anemo.git?rev=b1bda3f8555c5594a4966e850170cc11ecfcf557#b1bda3f8555c5594a4966e850170cc11ecfcf557" +source = "git+https://github.com/mystenlabs/anemo.git?rev=68adc31d4ea1f4573f08c2c75317fcb205b823de#68adc31d4ea1f4573f08c2c75317fcb205b823de" dependencies = [ "anemo", "bytes", @@ -238,7 +244,7 @@ dependencies = [ "libloading", "linkme", "once_cell", - "rand 0.8.7", + "rand 0.8.8", "rand_core 0.6.4", "rustc_version_runtime", "serde", @@ -247,9 +253,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.103" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "archery" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +checksum = "33ca55ee147b1926dbea904f50fe4902494e97bc742205abbbf10c709e43815f" [[package]] name = "ark-bn254" @@ -443,7 +455,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94893f1e0c6eeab764ade8dc4c0db24caf4fe7cbbaafc0eba0a9030f447b5185" dependencies = [ "num-traits", - "rand 0.8.7", + "rand 0.8.8", ] [[package]] @@ -470,7 +482,7 @@ dependencies = [ "nom", "num-traits", "rusticata-macros", - "thiserror 2.0.18", + "thiserror 2.0.20", "time", ] @@ -482,8 +494,8 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", - "synstructure", + "syn 2.0.119", + "synstructure 0.13.2", ] [[package]] @@ -494,14 +506,14 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "async-compression" -version = "0.4.42" +version = "0.4.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +checksum = "fb61aea1a7def73ee7c350a184f0e70b32c182344e2e75bf70c9b621b83417fd" dependencies = [ "compression-codecs", "compression-core", @@ -509,15 +521,20 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-task" +version = "4.3.0" +source = "git+https://github.com/mystenmark/async-task?rev=4e45b26e11126b191701b9b2ce5e2346b8d7682f#4e45b26e11126b191701b9b2ce5e2346b8d7682f" + [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.6", ] [[package]] @@ -600,7 +617,7 @@ checksum = "7aa268c23bfbbd2c4363b9cd302a4f504fb2a9dfe7e3451d66f35dd392e20aca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -613,7 +630,7 @@ dependencies = [ "getrandom 0.2.17", "instant", "pin-project-lite", - "rand 0.8.7", + "rand 0.8.8", "tokio", ] @@ -657,6 +674,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -792,30 +815,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" - -[[package]] -name = "bitmaps" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" -dependencies = [ - "typenum", -] - -[[package]] -name = "bitvec" -version = "0.20.4" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7774144344a4faa177370406a7ff5f1da24303817368584c6206c8303eb07848" -dependencies = [ - "funty 1.1.0", - "radium 0.6.2", - "tap", - "wyz 0.2.0", -] +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "bitvec" @@ -823,10 +825,10 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" dependencies = [ - "funty 2.0.0", - "radium 0.7.0", + "funty", + "radium", "tap", - "wyz 0.5.1", + "wyz", ] [[package]] @@ -840,22 +842,20 @@ dependencies = [ [[package]] name = "blake2b_simd" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b79834656f71332577234b50bfc009996f7449e0c056884e6a02492ded0ca2f3" +checksum = "3560a7b1951efe814fcd721938313adc56753ca39f4b23847d7e9a2402f5dbff" dependencies = [ - "arrayref", "arrayvec", "constant_time_eq", ] [[package]] name = "blake2s_simd" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee29928bad1e3f94c9d1528da29e07a1d3d04817ae8332de1e8b846c8439f4b3" +checksum = "2380c0236432f7b22a70229df30f218f5293870c056beb76f5ca068e3273a366" dependencies = [ - "arrayref", "arrayvec", "constant_time_eq", ] @@ -898,9 +898,9 @@ dependencies = [ [[package]] name = "blst" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcdb4c7013139a150f9fc55d123186dbfaba0d912817466282c73ac49e71fb45" +checksum = "c20659f9bbee16cbbd2f7393e40ab6309f5a98f76a2eb57a995ec508b72387fe" dependencies = [ "cc", "glob", @@ -978,7 +978,7 @@ dependencies = [ "digest 0.10.7", "group", "merlin", - "rand 0.8.7", + "rand 0.8.8", "rand_core 0.6.4", "serde", "serde_derive", @@ -1001,9 +1001,9 @@ checksum = "7575182f7272186991736b70173b0ea045398f984bf5ebbb3804736ce1330c9d" [[package]] name = "bytemuck" -version = "1.25.1" +version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6aedf8ae72766347502cf3cb4f41cf5e9cc37d28bee90f1fdaaae15f9cf9424" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" [[package]] name = "byteorder" @@ -1040,9 +1040,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" dependencies = [ "find-msvc-tools", "jobserver", @@ -1052,24 +1052,24 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "cfg_aliases" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -1126,9 +1126,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" dependencies = [ "clap_builder", "clap_derive", @@ -1136,9 +1136,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" dependencies = [ "anstream", "anstyle", @@ -1149,21 +1149,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.6", ] [[package]] name = "clap_lex" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" [[package]] name = "clear_on_drop" @@ -1188,32 +1188,32 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "compression-codecs" -version = "0.4.38" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +checksum = "bef16c47ba2797aa6a909cc37d39911f3a6743811fe7408ac0b0cc0276b656e9" dependencies = [ "brotli", "compression-core", "flate2", "memchr", - "zstd 0.13.3", - "zstd-safe 7.2.4", + "zstd 0.14.0", + "zstd-safe 8.0.0", ] [[package]] name = "compression-core" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" +checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" [[package]] name = "consensus-config" version = "0.1.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "fastcrypto", "mysten-network", - "rand 0.8.7", + "rand 0.8.8", "serde", "shared-crypto", ] @@ -1221,7 +1221,7 @@ dependencies = [ [[package]] name = "consensus-types" version = "0.1.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "base64 0.21.7", "consensus-config", @@ -1247,6 +1247,27 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" +[[package]] +name = "const_format" +version = "0.2.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4481a617ad9a412be3b97c5d403fef8ed023103368908b9c50af598ff467cc1e" +dependencies = [ + "const_format_proc_macros", + "konst", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + [[package]] name = "constant_time_eq" version = "0.4.2" @@ -1296,27 +1317,27 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ "libc", ] [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-utils" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crunchy" @@ -1401,7 +1422,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1439,12 +1460,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" dependencies = [ - "darling_core 0.23.0", - "darling_macro 0.23.0", + "darling_core 0.24.1", + "darling_macro 0.24.1", ] [[package]] @@ -1472,20 +1493,20 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "darling_core" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" dependencies = [ "ident_case", "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.118", + "syn 3.0.6", ] [[package]] @@ -1507,18 +1528,18 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "darling_macro" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ - "darling_core 0.23.0", + "darling_core 0.24.1", "quote", - "syn 2.0.118", + "syn 3.0.6", ] [[package]] @@ -1536,15 +1557,15 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" [[package]] name = "data-encoding-macro" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3259c913752a86488b501ed8680446a5ed2d5aeac6e596cb23ba3800768ea32c" +checksum = "c6a127ecbb3c4632e1525380e04c0c3fcf8dcb44d32a79ea290d8a36906edcd8" dependencies = [ "data-encoding", "data-encoding-macro-internal", @@ -1552,12 +1573,43 @@ dependencies = [ [[package]] name = "data-encoding-macro-internal" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" +checksum = "c54e03a951783e8b327515db3f2a2fd0e3bed362a96b066f341ce66ed49b4ead" dependencies = [ "data-encoding", - "syn 2.0.118", + "syn 3.0.6", +] + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.20", ] [[package]] @@ -1615,7 +1667,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1635,21 +1687,21 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", "unicode-xid", ] [[package]] name = "diesel" -version = "2.3.11" +version = "2.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e54d1f576cd3a3460f212a4615fd12ce1b6303c095b79a44449ffbe627753dc1" +checksum = "e3b934ddbdcb2abb9f9fc9c30bd47bcc5618b615eea1d334cda5fdf8ff9b072a" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.2", "byteorder", "chrono", "diesel_derives", - "downcast-rs", + "downcast-rs 2.0.2", "itoa", "pq-sys", ] @@ -1671,15 +1723,15 @@ dependencies = [ [[package]] name = "diesel_derives" -version = "2.3.9" +version = "2.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1817b7f4279b947fc4cafddec12b0e5f8727141706561ce3ac94a60bddd1cf5" +checksum = "ecbd51fb6c020672543641167efa4e6417ff7ad76849ed556ace3595e72de03a" dependencies = [ "diesel_table_macro_syntax", "dsl_auto_type", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1699,7 +1751,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe2444076b48641147115697648dc743c2c00b61adade0f01ce67133c7babe8c" dependencies = [ - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1737,13 +1789,13 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.6", ] [[package]] @@ -1752,6 +1804,12 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + [[package]] name = "downcast-rs" version = "2.0.2" @@ -1769,7 +1827,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1820,9 +1878,9 @@ dependencies = [ [[package]] name = "either" -version = "1.16.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" [[package]] name = "elliptic-curve" @@ -1847,7 +1905,7 @@ dependencies = [ [[package]] name = "enum-compat-util" version = "0.1.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "serde_yaml", ] @@ -1861,7 +1919,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -1870,6 +1928,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erasable" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "437cfb75878119ed8265685c41a115724eae43fb7cc5a0bf0e4ecc3b803af1c4" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "erased-discriminant" version = "1.0.1" @@ -1897,10 +1965,11 @@ checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f" [[package]] name = "eyre" -version = "0.6.12" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +checksum = "c08309dbcc659c5549a24ddb9b27027640641b282ef5768267c7e675558986a3" dependencies = [ + "autocfg", "indenter", "once_cell", ] @@ -1925,8 +1994,8 @@ dependencies = [ [[package]] name = "fastcrypto" -version = "0.1.9" -source = "git+https://github.com/MystenLabs/fastcrypto?rev=c6010b9080b00e29d039a48c08898d90c2c1cbdd#c6010b9080b00e29d039a48c08898d90c2c1cbdd" +version = "0.1.11" +source = "git+https://github.com/MystenLabs/fastcrypto?rev=076bf5ad8cea140dc7cf5c5861c24a36acdf64ce#076bf5ad8cea140dc7cf5c5861c24a36acdf64ce" dependencies = [ "aes", "aes-gcm", @@ -1946,6 +2015,7 @@ dependencies = [ "bs58 0.4.0", "bulletproofs", "cbc", + "ciborium", "ctr", "curve25519-dalek", "derive_more 0.99.20", @@ -1964,10 +2034,12 @@ dependencies = [ "num-bigint 0.4.8", "once_cell", "p256", - "rand 0.8.7", + "p384", + "rand 0.8.8", "readonly", "rfc6979", "rsa", + "rustls-pki-types", "schemars 0.8.22", "secp256k1", "serde", @@ -1981,13 +2053,14 @@ dependencies = [ "tokio", "tracing", "typenum", + "x509-parser", "zeroize", ] [[package]] name = "fastcrypto-derive" -version = "0.1.3" -source = "git+https://github.com/MystenLabs/fastcrypto?rev=c6010b9080b00e29d039a48c08898d90c2c1cbdd#c6010b9080b00e29d039a48c08898d90c2c1cbdd" +version = "0.1.4" +source = "git+https://github.com/MystenLabs/fastcrypto?rev=076bf5ad8cea140dc7cf5c5861c24a36acdf64ce#076bf5ad8cea140dc7cf5c5861c24a36acdf64ce" dependencies = [ "quote", "syn 1.0.109", @@ -1996,14 +2069,14 @@ dependencies = [ [[package]] name = "fastcrypto-tbls" version = "0.1.0" -source = "git+https://github.com/MystenLabs/fastcrypto?rev=c6010b9080b00e29d039a48c08898d90c2c1cbdd#c6010b9080b00e29d039a48c08898d90c2c1cbdd" +source = "git+https://github.com/MystenLabs/fastcrypto?rev=076bf5ad8cea140dc7cf5c5861c24a36acdf64ce#076bf5ad8cea140dc7cf5c5861c24a36acdf64ce" dependencies = [ "bcs", "digest 0.10.7", "fastcrypto", "hex", "itertools 0.10.5", - "rand 0.8.7", + "rand 0.8.8", "reed-solomon-erasure", "serde", "sha3", @@ -2015,8 +2088,8 @@ dependencies = [ [[package]] name = "fastcrypto-zkp" -version = "0.1.3" -source = "git+https://github.com/MystenLabs/fastcrypto?rev=c6010b9080b00e29d039a48c08898d90c2c1cbdd#c6010b9080b00e29d039a48c08898d90c2c1cbdd" +version = "0.1.4" +source = "git+https://github.com/MystenLabs/fastcrypto?rev=076bf5ad8cea140dc7cf5c5861c24a36acdf64ce#076bf5ad8cea140dc7cf5c5861c24a36acdf64ce" dependencies = [ "ark-bn254", "ark-ec", @@ -2030,7 +2103,7 @@ dependencies = [ "derive_more 0.99.20", "fastcrypto", "ff", - "im", + "imbl", "itertools 0.12.1", "lazy_static", "neptune", @@ -2041,14 +2114,15 @@ dependencies = [ "schemars 0.8.22", "serde", "serde_json", + "tracing", "typenum", ] [[package]] name = "fastrand" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "ff" @@ -2056,7 +2130,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" dependencies = [ - "bitvec 1.1.1", + "bitvec", "byteorder", "ff_derive", "rand_core 0.6.4", @@ -2086,18 +2160,18 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" [[package]] name = "fixed-hash" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" +checksum = "835c052cb0c08c1acf6ffd71c022172e18723949c8282f2b9f27efbc51e64534" dependencies = [ "byteorder", - "rand 0.8.7", + "rand 0.8.8", "rustc-hex", "static_assertions", ] @@ -2110,12 +2184,13 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flate2" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -2145,12 +2220,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "funty" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" - [[package]] name = "funty" version = "2.0.0" @@ -2159,9 +2228,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -2174,9 +2243,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -2184,15 +2253,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -2201,32 +2270,32 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.6", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-timer" @@ -2236,9 +2305,9 @@ checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -2314,9 +2383,9 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" [[package]] name = "governor" @@ -2333,7 +2402,7 @@ dependencies = [ "parking_lot 0.12.5", "portable-atomic", "quanta", - "rand 0.8.7", + "rand 0.8.8", "smallvec", "spinning_top", ] @@ -2345,7 +2414,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ "ff", - "rand 0.8.7", + "rand 0.8.8", "rand_core 0.6.4", "rand_xorshift 0.3.0", "subtle", @@ -2353,9 +2422,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.15" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", @@ -2363,10 +2432,10 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.14.0", + "indexmap 2.14.2", "slab", "tokio", - "tokio-util", + "tokio-util 0.7.19", "tracing", ] @@ -2419,12 +2488,17 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "hdrhistogram" -version = "7.5.4" +version = "7.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" +checksum = "f49d1053f4708f0af3cf9fc5bffc7e68a914a3c45becb231c80068c9c3f78bea" dependencies = [ "byteorder", "num-traits", @@ -2444,9 +2518,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" [[package]] name = "hex" @@ -2492,9 +2566,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -2502,9 +2576,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", "http", @@ -2512,9 +2586,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -2549,18 +2623,18 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" -version = "0.4.13" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" dependencies = [ "typenum", ] [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -2580,9 +2654,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.9" +version = "0.27.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" dependencies = [ "http", "hyper", @@ -2592,7 +2666,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.8", + "webpki-roots 1.0.9", ] [[package]] @@ -2625,7 +2699,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.4", + "socket2 0.6.5", "tokio", "tower-service", "tracing", @@ -2657,9 +2731,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -2671,9 +2745,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -2684,9 +2758,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -2698,16 +2772,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -2718,15 +2793,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", @@ -2765,33 +2840,40 @@ dependencies = [ ] [[package]] -name = "im" -version = "15.1.0" +name = "imbl" +version = "7.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" +checksum = "46bad832b9b463ed9398b8506488cc2e3b897a9d44f13af115f382aac71f4fec" dependencies = [ - "bitmaps", - "rand_core 0.6.4", + "archery", + "equivalent", + "imbl-sized-chunks", + "rand_core 0.9.5", "rand_xoshiro", - "sized-chunks", - "typenum", "version_check", + "wide", ] +[[package]] +name = "imbl-sized-chunks" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0813be332553f857953298749fa19549e8b61b80589757c29b4e2a804fa9c6" + [[package]] name = "impl-codec" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "161ebdfec3c8e3b52bf61c4f3550a1eea4f9579d10dc1b936f3171ebdcd6c443" +checksum = "ba6a270039626615617f3f36d15fc827041df3b78c439da2cadfa47455a77f2f" dependencies = [ "parity-scale-codec", ] [[package]] name = "impl-serde" -version = "0.3.2" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4551f042f3438e64dbd6226b20527fc84a6e1fe65688b58746a2f53623f25f5c" +checksum = "ebc88fc67028ae3db0c853baa36269d398d5f45b6982f95549ff5def78c935cd" dependencies = [ "serde", ] @@ -2804,7 +2886,7 @@ checksum = "a0eb5a3343abf848c0984fe4604b2b105da9539376e24fc0a3b0007411ae4fd9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -2826,9 +2908,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown 0.17.1", @@ -2857,19 +2939,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" - -[[package]] -name = "iri-string" -version = "0.7.12" +version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" -dependencies = [ - "memchr", - "serde", -] +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" [[package]] name = "is_terminal_polyfill" @@ -2919,6 +2991,60 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jiff" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1baf72f08796de0260609515130699b890ac25f30e610ad894bc5856cafdb" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "378268a1116ad67ae6228701118ac9f491d78fda38a40a1f1a9e1348de6f7212" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jobserver" version = "0.1.35" @@ -2931,9 +3057,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.103" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" dependencies = [ "cfg-if", "futures-util", @@ -2949,6 +3075,21 @@ dependencies = [ "cpufeatures 0.2.17", ] +[[package]] +name = "konst" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128133ed7824fcd73d6e7b17957c5eb7bacb885649bd8c69708b2331a10bcefb" +dependencies = [ + "konst_macro_rules", +] + +[[package]] +name = "konst_macro_rules" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" + [[package]] name = "lazy_static" version = "1.5.0" @@ -2966,9 +3107,9 @@ checksum = "c83bff1d572d6b9aeef67ddfc8448e4a3737909cb28e81f97c791b9018703e52" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libloading" @@ -2996,15 +3137,15 @@ dependencies = [ "hkdf", "multihash", "sha2 0.10.9", - "thiserror 2.0.18", + "thiserror 2.0.20", "tracing", ] [[package]] name = "libredox" -version = "0.1.18" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" dependencies = [ "libc", ] @@ -3017,22 +3158,22 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linkme" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83272d46373fb8decca684579ac3e7c8f3d71d4cc3aa693df8759e260ae41cf" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" dependencies = [ "linkme-impl", ] [[package]] name = "linkme-impl" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d59e20403c7d08fe62b4376edfe5c7fb2ef1e6b1465379686d0f21c8df444b" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.6", ] [[package]] @@ -3043,9 +3184,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "lock_api" @@ -3058,9 +3199,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lru" @@ -3073,28 +3214,28 @@ dependencies = [ [[package]] name = "lru" -version = "0.10.1" +version = "0.18.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718e8fae447df0c7e1ba7f5189829e63fd536945c8988d61444c19039f16b670" +checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" dependencies = [ - "hashbrown 0.13.2", + "hashbrown 0.17.1", ] [[package]] name = "lru-slab" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" [[package]] name = "match-lookup" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "757aee279b8bdbb9f9e676796fd459e4207a1f986e87886700abf589f5abf771" +checksum = "549e39695cc0b640f3cb378053832db3d2133422d49e8dcae5c866a2aaf1f730" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3158,6 +3299,7 @@ dependencies = [ "dotenvy", "hex", "move-core-types", + "object_store", "rustls", "serde", "sui-indexer-alt-framework", @@ -3185,7 +3327,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36c791ecdf977c99f45f23280405d7723727470f6689a5e6dbf513ac547ae10d" dependencies = [ "serde", - "toml", + "toml 0.9.12+spec-1.1.0", ] [[package]] @@ -3223,9 +3365,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.9" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", "simd-adler32", @@ -3233,9 +3375,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.1" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -3245,16 +3387,16 @@ dependencies = [ [[package]] name = "move-abstract-interpreter" version = "0.1.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" [[package]] name = "move-binary-format" version = "0.0.3" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "anyhow", "enum-compat-util", - "indexmap 2.14.0", + "indexmap 2.14.2", "move-abstract-interpreter", "move-core-types", "move-proc-macros", @@ -3266,10 +3408,10 @@ dependencies = [ [[package]] name = "move-bytecode-utils" version = "0.1.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "anyhow", - "indexmap 2.14.0", + "indexmap 2.14.2", "move-binary-format", "move-core-types", "petgraph", @@ -3279,18 +3421,18 @@ dependencies = [ [[package]] name = "move-core-types" version = "0.0.4" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "anyhow", "bcs", "ethnum", "hex", - "indexmap 2.14.0", + "indexmap 2.14.2", "leb128", "move-proc-macros", "num", "primitive-types", - "rand 0.8.7", + "rand 0.8.8", "ref-cast", "serde", "serde_with", @@ -3301,17 +3443,17 @@ dependencies = [ [[package]] name = "move-proc-macros" version = "0.1.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "enum-compat-util", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "move-trace-format" version = "0.0.1" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "move-binary-format", "move-core-types", @@ -3323,7 +3465,7 @@ dependencies = [ [[package]] name = "move-vm-config" version = "0.1.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "move-binary-format", ] @@ -3331,7 +3473,7 @@ dependencies = [ [[package]] name = "move-vm-profiler" version = "0.1.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "move-trace-format", "move-vm-config", @@ -3340,10 +3482,39 @@ dependencies = [ "tracing", ] +[[package]] +name = "msim" +version = "0.1.0" +source = "git+https://github.com/MystenLabs/mysten-sim.git?rev=6b8e1e8cdce408e73d46538773fc85a56c082db7#6b8e1e8cdce408e73d46538773fc85a56c082db7" +dependencies = [ + "ahash 0.7.8", + "async-task", + "bincode", + "bytes", + "cc", + "downcast-rs 1.2.1", + "erasable", + "futures", + "lazy_static", + "libc", + "msim-macros", + "naive-timer", + "pin-project-lite", + "rand 0.8.8", + "real_tokio", + "serde", + "socket2 0.4.10", + "tap", + "tokio-util 0.7.18", + "toml 0.5.11", + "tracing", + "tracing-subscriber", +] + [[package]] name = "msim-macros" version = "0.1.0" -source = "git+https://github.com/MystenLabs/mysten-sim.git?rev=2c2ec3ffbc1e559c575dc7de1ff2498d1f6700f6#2c2ec3ffbc1e559c575dc7de1ff2498d1f6700f6" +source = "git+https://github.com/MystenLabs/mysten-sim.git?rev=6b8e1e8cdce408e73d46538773fc85a56c082db7#6b8e1e8cdce408e73d46538773fc85a56c082db7" dependencies = [ "darling 0.14.4", "proc-macro2", @@ -3395,16 +3566,18 @@ dependencies = [ [[package]] name = "mysten-common" version = "0.1.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "antithesis_sdk", "either", "futures", + "msim", "mysten-metrics", "once_cell", + "oneshot", "parking_lot 0.12.5", "prost-types", - "rand 0.8.7", + "rand 0.8.8", "serde_json", "sui-macros", "tempfile", @@ -3416,7 +3589,7 @@ dependencies = [ [[package]] name = "mysten-metrics" version = "0.7.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "async-trait", "axum", @@ -3437,11 +3610,12 @@ dependencies = [ [[package]] name = "mysten-network" version = "0.2.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "anemo", "anemo-tower", "anyhow", + "base64 0.21.7", "bcs", "bytes", "dashmap", @@ -3456,20 +3630,28 @@ dependencies = [ "once_cell", "pin-project-lite", "prometheus", + "prost-reflect", "quinn-proto", - "rand 0.8.7", + "rand 0.8.8", "rustls", "serde", "snap", "sui-http", + "sui-macros", "tokio", "tokio-rustls", "tonic", "tower 0.5.3", - "tower-http 0.5.2", + "tower-http", "tracing", ] +[[package]] +name = "naive-timer" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "034a0ad7deebf0c2abcf2435950a6666c3c15ea9d8fad0c0f48efa8a7f843fed" + [[package]] name = "neptune" version = "13.0.0" @@ -3562,7 +3744,7 @@ checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", - "rand 0.8.7", + "rand 0.8.8", ] [[package]] @@ -3576,7 +3758,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.7", + "rand 0.8.8", "smallvec", "zeroize", ] @@ -3598,9 +3780,9 @@ checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" dependencies = [ "num-traits", ] @@ -3661,10 +3843,10 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96667db765a921f7b295ffee8b60472b686a51d4f21c2ee4ffdb94c7013b65a6" dependencies = [ - "proc-macro-crate", + "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -3673,7 +3855,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.2", ] [[package]] @@ -3709,14 +3891,14 @@ dependencies = [ "parking_lot 0.12.5", "percent-encoding", "quick-xml", - "rand 0.10.2", + "rand 0.10.3", "reqwest", "ring", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", "url", @@ -3746,6 +3928,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oneshot" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe21416a02c693fb9f980befcb230ecc70b0b3d1cc4abf88b9675c4c1457f0c" + [[package]] name = "opaque-debug" version = "0.3.1" @@ -3758,6 +3946,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + [[package]] name = "p256" version = "0.13.2" @@ -3793,28 +3990,30 @@ dependencies = [ [[package]] name = "parity-scale-codec" -version = "2.3.1" +version = "3.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373b1a4c1338d9cd3d1fa53b3a11bdab5ab6bd80a20f7f7becd76953ae2be909" +checksum = "799781ae679d79a948e13d4824a40970bfa500058d245760dd857301059810fa" dependencies = [ "arrayvec", - "bitvec 0.20.4", + "bitvec", "byte-slice-cast", + "const_format", "impl-trait-for-tuples", "parity-scale-codec-derive", + "rustversion", "serde", ] [[package]] name = "parity-scale-codec-derive" -version = "2.3.1" +version = "3.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1557010476e0595c9b568d16dcfb81b93cdeb157612726f5170d31aa707bed27" +checksum = "34b4653168b563151153c9e4c08ebed57fb8262bebfa79711552fa983c623e7a" dependencies = [ - "proc-macro-crate", + "proc-macro-crate 3.5.0", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.119", ] [[package]] @@ -3871,14 +4070,14 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77144664f6aac5f629d7efa815f5098a054beeeca6ccafee5ec453fd2b0c53f9" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.2", "ciborium", "coset", "data-encoding", "getrandom 0.2.17", "hmac 0.12.1", - "indexmap 2.14.0", - "rand 0.8.7", + "indexmap 2.14.2", + "rand 0.8.8", "serde", "serde_json", "sha2 0.10.9", @@ -3889,16 +4088,16 @@ dependencies = [ [[package]] name = "pasta_curves" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e57598f73cc7e1b2ac63c79c517b31a0877cd7c402cdcaa311b5208de7a095" +checksum = "3437083215c505e867eea5478371feba43d7689d6d15ec0a209eb46fb0d4cda6" dependencies = [ "blake2b_simd", "ff", "group", "hex", "lazy_static", - "rand 0.8.7", + "rand 0.8.8", "serde", "static_assertions", "subtle", @@ -3943,7 +4142,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.5", - "indexmap 2.14.0", + "indexmap 2.14.2", "serde", ] @@ -3983,7 +4182,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4015,9 +4214,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "polyval" @@ -4033,9 +4232,18 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.13.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] [[package]] name = "postgres-protocol" @@ -4050,7 +4258,7 @@ dependencies = [ "hmac 0.13.0", "md-5 0.11.0", "memchr", - "rand 0.10.2", + "rand 0.10.3", "sha2 0.11.0", "stringprep", ] @@ -4068,9 +4276,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] @@ -4112,9 +4320,9 @@ dependencies = [ [[package]] name = "primitive-types" -version = "0.10.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05e4722c697a58a99d5d06a08c30821d7c082a4632198de1eaa5a6c22ef42373" +checksum = "0b34d9fd68ae0b74a41b21c03c2f62847aa0ffea044eee893b4c140b37e244e2" dependencies = [ "fixed-hash", "impl-codec", @@ -4129,14 +4337,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" dependencies = [ "once_cell", - "toml_edit", + "toml_edit 0.19.15", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit 0.25.15+spec-1.1.0", ] [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -4153,13 +4370,13 @@ dependencies = [ "memchr", "parking_lot 0.12.5", "protobuf", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] name = "prometheus-closure-metric" version = "0.1.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "anyhow", "prometheus", @@ -4173,7 +4390,7 @@ checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.13.0", + "bitflags 2.13.2", "num-traits", "rand 0.9.5", "rand_chacha 0.9.0", @@ -4192,7 +4409,7 @@ checksum = "4ee1c9ac207483d5e7db4940700de86a9aae46ef90c48b57f99fe7edb8345e49" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4215,7 +4432,20 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", +] + +[[package]] +name = "prost-reflect" +version = "0.16.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01b80ea363c31af2de2b92e3c07ed1156628f7838c4afb4df75ee78a37fedbd1" +dependencies = [ + "base64 0.22.1", + "prost", + "prost-types", + "serde", + "serde-value", ] [[package]] @@ -4280,9 +4510,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.11" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" dependencies = [ "bytes", "cfg_aliases", @@ -4292,8 +4522,8 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.4", - "thiserror 2.0.18", + "socket2 0.6.5", + "thiserror 2.0.20", "tokio", "tracing", "web-time", @@ -4301,22 +4531,22 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.16" +version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" dependencies = [ "bytes", "fastbloom", "getrandom 0.4.3", "lru-slab", - "rand 0.10.2", + "rand 0.10.3", "rand_pcg", "ring", "rustc-hash", "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.18", + "thiserror 2.0.20", "tinyvec", "tracing", "web-time", @@ -4331,16 +4561,16 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.4", + "socket2 0.6.5", "tracing", "windows-sys 0.61.2", ] [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -4357,12 +4587,6 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "radium" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643f8f41a8ebc4c5dc4515c82bb8abd397b527fc20fd681b7c011c2aee5d44fb" - [[package]] name = "radium" version = "0.7.0" @@ -4371,9 +4595,9 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "rand" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -4392,9 +4616,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -4474,11 +4698,11 @@ dependencies = [ [[package]] name = "rand_xoshiro" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" dependencies = [ - "rand_core 0.6.4", + "rand_core 0.9.5", ] [[package]] @@ -4487,7 +4711,7 @@ version = "11.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.2", ] [[package]] @@ -4505,13 +4729,29 @@ dependencies = [ [[package]] name = "readonly" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2a62d85ed81ca5305dc544bd42c8804c5060b78ffa5ad3c64b0fb6a8c13d062" +checksum = "67a77cd6f1a55ff3cb1969c2618f73107efaa6de1daf796ad60ce8f737ae97c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.6", +] + +[[package]] +name = "real_tokio" +version = "1.52.1" +source = "git+https://github.com/MystenLabs/tokio-msim-fork.git?rev=e8ce7593bfbfbbd7c7ee8671faecb062a8966b2d#e8ce7593bfbfbbd7c7ee8671faecb062a8966b2d" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot 0.12.5", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.6.5", + "tokio-macros", + "windows-sys 0.61.2", ] [[package]] @@ -4529,7 +4769,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.2", ] [[package]] @@ -4547,29 +4787,29 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.25" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.25" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.6", ] [[package]] name = "regex" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -4579,9 +4819,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.15" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -4626,16 +4866,16 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", - "tokio-util", + "tokio-util 0.7.19", "tower 0.5.3", - "tower-http 0.6.11", + "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.8", + "webpki-roots 1.0.9", ] [[package]] @@ -4664,9 +4904,9 @@ dependencies = [ [[package]] name = "roaring" -version = "0.11.4" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9" +checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1" dependencies = [ "bytemuck", "byteorder", @@ -4735,11 +4975,11 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys", @@ -4748,9 +4988,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.41" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "log", "once_cell", @@ -4775,9 +5015,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "web-time", "zeroize", @@ -4785,9 +5025,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "ring", "rustls-pki-types", @@ -4818,6 +5058,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "safe_arch" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323" +dependencies = [ + "bytemuck", +] + [[package]] name = "same-file" version = "1.0.6" @@ -4863,9 +5112,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" dependencies = [ "dyn-clone", "ref-cast", @@ -4882,7 +5131,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -4921,7 +5170,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25996b82292a7a57ed3508f052cfff8640d38d32018784acd714758b43da9c8f" dependencies = [ "bitcoin_hashes", - "rand 0.8.7", + "rand 0.8.8", "secp256k1-sys", ] @@ -4940,7 +5189,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.2", "core-foundation", "core-foundation-sys", "libc", @@ -4965,9 +5214,9 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -5007,24 +5256,34 @@ dependencies = [ "typeid", ] +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float", + "serde", +] + [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.6", ] [[package]] @@ -5035,16 +5294,16 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.2", "itoa", "memchr", "serde", @@ -5065,13 +5324,13 @@ dependencies = [ [[package]] name = "serde_repr" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.6", ] [[package]] @@ -5097,18 +5356,19 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.21.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +checksum = "935177bb8c0cd8ca1a4e6d1a2ac8988bea69cab4f9d3a31311e012ad27868ea4" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bs58 0.5.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.14.0", + "indexmap 2.14.2", + "jiff", "schemars 0.9.0", - "schemars 1.2.1", + "schemars 1.2.2", "serde_core", "serde_json", "serde_with_macros", @@ -5117,14 +5377,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.21.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +checksum = "1d607aa01a3cb0ad757d6fd216136910db3c97b102fe686585689615a02dbcdc" dependencies = [ - "darling 0.23.0", + "darling 0.24.1", "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.6", ] [[package]] @@ -5181,7 +5441,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "digest 0.11.3", ] @@ -5207,7 +5467,7 @@ dependencies = [ [[package]] name = "shared-crypto" version = "0.0.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "bcs", "eyre", @@ -5250,9 +5510,9 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" [[package]] name = "simple-server-timing-header" @@ -5266,16 +5526,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" -[[package]] -name = "sized-chunks" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" -dependencies = [ - "bitmaps", - "typenum", -] - [[package]] name = "slab" version = "0.4.12" @@ -5284,15 +5534,25 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.2" +version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" [[package]] name = "snap" -version = "1.1.1" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" + +[[package]] +name = "socket2" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" +dependencies = [ + "libc", + "winapi", +] [[package]] name = "socket2" @@ -5306,9 +5566,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -5316,9 +5576,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" [[package]] name = "spinning_top" @@ -5402,7 +5662,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5414,7 +5674,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -5431,9 +5691,10 @@ checksum = "734676eb262c623cec13c3155096e08d1f8f29adce39ba17948b18dad1e54142" [[package]] name = "sui-crypto" -version = "0.3.0" -source = "git+https://github.com/MystenLabs/sui-rust-sdk.git?rev=43c5bc13202ae398b1519a3eead1f40df8ca277b#43c5bc13202ae398b1519a3eead1f40df8ca277b" +version = "0.3.1" +source = "git+https://github.com/MystenLabs/sui-rust-sdk.git?rev=618b6c8484a2394f73d9b5e21cce0ac228b2cf31#618b6c8484a2394f73d9b5e21cce0ac228b2cf31" dependencies = [ + "base64ct", "blst", "rand_core 0.6.4", "signature 2.2.0", @@ -5443,23 +5704,23 @@ dependencies = [ [[package]] name = "sui-enum-compat-util" version = "0.1.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "serde_yaml", ] [[package]] name = "sui-field-count" -version = "1.75.1" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +version = "1.80.0" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "sui-field-count-derive", ] [[package]] name = "sui-field-count-derive" -version = "1.75.1" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +version = "1.80.0" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "quote", "syn 1.0.109", @@ -5467,41 +5728,45 @@ dependencies = [ [[package]] name = "sui-futures" -version = "1.75.1" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +version = "1.80.0" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "anyhow", "futures", + "pin-project", "tap", "thiserror 1.0.69", "tokio", "tracing", + "tracing-error", ] [[package]] name = "sui-http" -version = "0.0.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b46940575ea7ee7eaf2a7000da28d82a61b76f8d4160590674946263975782" dependencies = [ "bytes", + "futures-core", "http", "http-body", "http-body-util", "hyper", "hyper-util", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.5", "tokio", "tokio-rustls", - "tokio-util", + "tokio-util 0.7.19", "tower 0.5.3", "tracing", ] [[package]] name = "sui-indexer-alt-framework" -version = "1.75.1" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +version = "1.80.0" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "anyhow", "async-trait", @@ -5515,19 +5780,19 @@ dependencies = [ "diesel_migrations", "futures", "http", - "mysten-network", "num_cpus", "object_store", "prometheus", "prost", "prost-types", - "rand 0.8.7", + "rand 0.8.8", "reqwest", "scoped-futures", "serde", "serde_json", "sui-field-count", "sui-futures", + "sui-http", "sui-indexer-alt-framework-store-traits", "sui-indexer-alt-metrics", "sui-pg-db", @@ -5538,16 +5803,17 @@ dependencies = [ "tokio", "tokio-stream", "tonic", + "tower 0.5.3", "tracing", "tracing-subscriber", "url", - "zstd 0.12.4", + "zstd 0.13.3", ] [[package]] name = "sui-indexer-alt-framework-store-traits" -version = "1.75.1" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +version = "1.80.0" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "anyhow", "async-trait", @@ -5557,8 +5823,8 @@ dependencies = [ [[package]] name = "sui-indexer-alt-metrics" -version = "1.75.1" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +version = "1.80.0" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "anyhow", "axum", @@ -5574,7 +5840,7 @@ dependencies = [ [[package]] name = "sui-macros" version = "0.7.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "futures", "once_cell", @@ -5584,8 +5850,8 @@ dependencies = [ [[package]] name = "sui-pg-db" -version = "1.75.1" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +version = "1.80.0" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "anyhow", "async-trait", @@ -5614,19 +5880,19 @@ dependencies = [ [[package]] name = "sui-proc-macros" version = "0.7.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "msim-macros", "proc-macro2", "quote", "sui-enum-compat-util", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "sui-protocol-config" version = "0.1.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "clap", "fastcrypto", @@ -5634,7 +5900,7 @@ dependencies = [ "move-core-types", "move-vm-config", "mysten-common", - "rand 0.8.7", + "rand 0.8.8", "schemars 0.8.22", "serde", "serde-env", @@ -5647,7 +5913,7 @@ dependencies = [ [[package]] name = "sui-protocol-config-macros" version = "0.1.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "proc-macro2", "quote", @@ -5656,8 +5922,8 @@ dependencies = [ [[package]] name = "sui-rpc" -version = "0.3.1" -source = "git+https://github.com/MystenLabs/sui-rust-sdk.git?rev=43c5bc13202ae398b1519a3eead1f40df8ca277b#43c5bc13202ae398b1519a3eead1f40df8ca277b" +version = "0.3.2" +source = "git+https://github.com/MystenLabs/sui-rust-sdk.git?rev=618b6c8484a2394f73d9b5e21cce0ac228b2cf31#618b6c8484a2394f73d9b5e21cce0ac228b2cf31" dependencies = [ "base64 0.22.1", "bcs", @@ -5665,6 +5931,7 @@ dependencies = [ "futures", "http", "http-body", + "hyper", "prost", "prost-types", "serde", @@ -5680,8 +5947,8 @@ dependencies = [ [[package]] name = "sui-sdk-types" -version = "0.3.1" -source = "git+https://github.com/MystenLabs/sui-rust-sdk.git?rev=43c5bc13202ae398b1519a3eead1f40df8ca277b#43c5bc13202ae398b1519a3eead1f40df8ca277b" +version = "0.3.2" +source = "git+https://github.com/MystenLabs/sui-rust-sdk.git?rev=618b6c8484a2394f73d9b5e21cce0ac228b2cf31#618b6c8484a2394f73d9b5e21cce0ac228b2cf31" dependencies = [ "base64ct", "bcs", @@ -5701,8 +5968,8 @@ dependencies = [ [[package]] name = "sui-sql-macro" -version = "1.75.1" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +version = "1.80.0" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "quote", "syn 1.0.109", @@ -5712,7 +5979,7 @@ dependencies = [ [[package]] name = "sui-types" version = "0.1.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "anemo", "anyhow", @@ -5733,10 +6000,10 @@ dependencies = [ "fastcrypto", "fastcrypto-tbls", "fastcrypto-zkp", - "im", - "indexmap 2.14.0", + "imbl", + "indexmap 2.14.2", "itertools 0.13.0", - "lru 0.10.1", + "lru 0.18.4", "move-binary-format", "move-bytecode-utils", "move-core-types", @@ -5758,7 +6025,7 @@ dependencies = [ "proptest-derive", "prost", "prost-types", - "rand 0.8.7", + "rand 0.8.8", "roaring", "rustls-pki-types", "schemars 0.8.22", @@ -5797,9 +6064,20 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" dependencies = [ "proc-macro2", "quote", @@ -5823,7 +6101,18 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", ] [[package]] @@ -5866,11 +6155,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.18", + "thiserror-impl 2.0.20", ] [[package]] @@ -5881,18 +6170,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.6", ] [[package]] @@ -5915,9 +6204,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.53" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", "num-conv", @@ -5935,9 +6224,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" dependencies = [ "num-conv", "time-core", @@ -5945,9 +6234,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -5955,18 +6244,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.12.0" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] name = "tokio" @@ -5980,20 +6260,20 @@ dependencies = [ "parking_lot 0.12.5", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.4", + "socket2 0.6.5", "tokio-macros", "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.7.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.6", ] [[package]] @@ -6015,10 +6295,10 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.10.2", - "socket2 0.6.4", + "rand 0.10.3", + "socket2 0.6.5", "tokio", - "tokio-util", + "tokio-util 0.7.19", "whoami", ] @@ -6038,9 +6318,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.4" +version = "0.26.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" dependencies = [ "rustls", "tokio", @@ -6048,14 +6328,14 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", "tokio", - "tokio-util", + "tokio-util 0.7.19", ] [[package]] @@ -6073,16 +6353,42 @@ dependencies = [ [[package]] name = "tokio-util" version = "0.7.18" +source = "git+https://github.com/MystenLabs/tokio-msim-fork.git?rev=e8ce7593bfbfbbd7c7ee8671faecb062a8966b2d#e8ce7593bfbfbbd7c7ee8671faecb062a8966b2d" +dependencies = [ + "bytes", + "futures-core", + "futures-io", + "futures-sink", + "futures-util", + "hashbrown 0.15.5", + "pin-project-lite", + "real_tokio", + "slab", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + [[package]] name = "toml" version = "0.9.12+spec-1.1.0" @@ -6111,24 +6417,45 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + [[package]] name = "toml_edit" version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.2", "toml_datetime 0.6.11", "winnow 0.5.40", ] +[[package]] +name = "toml_edit" +version = "0.25.15+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +dependencies = [ + "indexmap 2.14.2", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "winnow 1.0.4", +] + [[package]] name = "toml_parser" -version = "1.1.2+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ - "winnow 1.0.3", + "winnow 1.0.4", ] [[package]] @@ -6150,7 +6477,7 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "socket2 0.6.4", + "socket2 0.6.5", "sync_wrapper", "tokio", "tokio-rustls", @@ -6159,7 +6486,7 @@ dependencies = [ "tower-layer", "tower-service", "tracing", - "webpki-roots 1.0.8", + "webpki-roots 1.0.9", "zstd 0.13.3", ] @@ -6186,10 +6513,10 @@ dependencies = [ "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand 0.8.7", + "rand 0.8.8", "slab", "tokio", - "tokio-util", + "tokio-util 0.7.19", "tower-layer", "tower-service", "tracing", @@ -6204,12 +6531,12 @@ dependencies = [ "futures-core", "futures-util", "hdrhistogram", - "indexmap 2.14.0", + "indexmap 2.14.2", "pin-project-lite", "slab", "sync_wrapper", "tokio", - "tokio-util", + "tokio-util 0.7.19", "tower-layer", "tower-service", "tracing", @@ -6217,13 +6544,13 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.5.2" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", - "base64 0.21.7", - "bitflags 2.13.0", + "base64 0.22.1", + "bitflags 2.13.2", "bytes", "futures-core", "futures-util", @@ -6232,36 +6559,18 @@ dependencies = [ "http-body-util", "http-range-header", "httpdate", - "iri-string", "mime", "mime_guess", "percent-encoding", "pin-project-lite", "tokio", - "tokio-util", - "tower 0.4.13", - "tower-layer", - "tower-service", - "tracing", - "uuid", -] - -[[package]] -name = "tower-http" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" -dependencies = [ - "bitflags 2.13.0", - "bytes", - "futures-util", - "http", - "http-body", - "pin-project-lite", + "tokio-util 0.7.19", "tower 0.5.3", "tower-layer", "tower-service", + "tracing", "url", + "uuid", ] [[package]] @@ -6296,7 +6605,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6309,6 +6618,27 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-error" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db" +dependencies = [ + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + [[package]] name = "tracing-serde" version = "0.2.0" @@ -6337,6 +6667,7 @@ dependencies = [ "time", "tracing", "tracing-core", + "tracing-log", "tracing-serde", ] @@ -6370,13 +6701,13 @@ dependencies = [ "log", "rand 0.9.5", "sha1", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] name = "typed-store-error" version = "0.4.0" -source = "git+https://github.com/MystenLabs/sui.git?rev=623521008f1c3afa3a1adbb6c3c44c39c46c5e17#623521008f1c3afa3a1adbb6c3c44c39c46c5e17" +source = "git+https://github.com/MystenLabs/sui.git?rev=ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f#ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" dependencies = [ "serde", "thiserror 1.0.69", @@ -6413,7 +6744,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "621963e302416b389a1ec177397e9e62de849a78bd8205d428608553def75350" dependencies = [ "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6448,9 +6779,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" [[package]] name = "unicode-normalization" @@ -6521,13 +6852,13 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.4" +version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" +checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" dependencies = [ "getrandom 0.4.3", "js-sys", - "rand 0.10.2", + "rand 0.10.3", "wasm-bindgen", ] @@ -6545,7 +6876,7 @@ checksum = "17c13b28baa922722e35ded153b394d0247b4942b4ad54f85713a7de672cdf0e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6623,9 +6954,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.126" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" dependencies = [ "cfg-if", "once_cell", @@ -6636,9 +6967,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.76" +version = "0.4.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" dependencies = [ "js-sys", "wasm-bindgen", @@ -6646,9 +6977,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.126" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6656,22 +6987,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.126" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.6", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.126" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" dependencies = [ "unicode-ident", ] @@ -6691,9 +7022,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.103" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" dependencies = [ "js-sys", "wasm-bindgen", @@ -6715,23 +7046,23 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.8", + "webpki-roots 1.0.9", ] [[package]] name = "webpki-roots" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" dependencies = [ "rustls-pki-types", ] [[package]] name = "whoami" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "998767ef88740d1f5b0682a9c53c24431453923962269c2db68ee43788c5a40d" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" dependencies = [ "libc", "libredox", @@ -6740,6 +7071,16 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wide" +version = "0.7.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03" +dependencies = [ + "bytemuck", + "safe_arch", +] + [[package]] name = "winapi" version = "0.3.9" @@ -6792,7 +7133,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6803,7 +7144,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -6932,9 +7273,12 @@ dependencies = [ [[package]] name = "winnow" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] [[package]] name = "wit-bindgen" @@ -6944,15 +7288,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" - -[[package]] -name = "wyz" -version = "0.2.0" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "wyz" @@ -6996,7 +7334,7 @@ dependencies = [ "oid-registry", "ring", "rusticata-macros", - "thiserror 2.0.18", + "thiserror 2.0.20", "time", ] @@ -7031,34 +7369,34 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", - "synstructure", + "syn 3.0.6", + "synstructure 0.14.0", ] [[package]] name = "zerocopy" -version = "0.8.54" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.54" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] @@ -7072,14 +7410,14 @@ dependencies = [ [[package]] name = "zerofrom-derive" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", - "synstructure", + "syn 3.0.6", + "synstructure 0.14.0", ] [[package]] @@ -7099,14 +7437,14 @@ checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 2.0.119", ] [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -7115,9 +7453,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ "yoke", "zerofrom", @@ -7126,63 +7464,68 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.6", ] +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" [[package]] name = "zstd" -version = "0.12.4" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ - "zstd-safe 6.0.6", + "zstd-safe 7.3.0", ] [[package]] name = "zstd" -version = "0.13.3" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +checksum = "bf06bd8162af0734b344780deb55b42a2429ae430870d13fcc12f238e880fe6e" dependencies = [ - "zstd-safe 7.2.4", + "zstd-safe 8.0.0", ] [[package]] name = "zstd-safe" -version = "6.0.6" +version = "7.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" dependencies = [ - "libc", "zstd-sys", ] [[package]] name = "zstd-safe" -version = "7.2.4" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "ae42c0555055784c70058d19ba8e275528e8a99a706684868ace5da4e716a4ab" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.0+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" dependencies = [ "cc", "pkg-config", diff --git a/services/indexer/Cargo.toml b/services/indexer/Cargo.toml index b16f88b74..ff7ea3fe8 100644 --- a/services/indexer/Cargo.toml +++ b/services/indexer/Cargo.toml @@ -22,10 +22,24 @@ tracing = "0.1" # Checkpoint-ingestion framework: reads static checkpoint files from a public bucket, # not the fullnode event API (testnet disabled ledger-history indexing). Same resolved # graph dopamint-arena uses (diesel-async 0.9 -> diesel 2.3). Pin the exact -# testnet-v1.75.1 commit so a mutable branch update cannot change the build. -sui-indexer-alt-framework = { git = "https://github.com/MystenLabs/sui.git", rev = "623521008f1c3afa3a1adbb6c3c44c39c46c5e17" } -move-core-types = { git = "https://github.com/MystenLabs/sui.git", rev = "623521008f1c3afa3a1adbb6c3c44c39c46c5e17" } +# testnet-v1.80.0 commit so a mutable branch update cannot change the build. +# +# The pin must not trail the network: these types decode whole checkpoints, so a +# transaction field the pinned types don't know is a hard decode failure, not a +# skipped field. testnet-v1.79.0 added `TransactionExpiration::Validity` as variant +# index 3, and the previous testnet-v1.75.1 pin (variants 0..=2) failed every +# checkpoint carrying one with `invalid value: integer 3, expected variant index +# 0 <= i < 3`, stalling `accounts_v1` behind a growing watermark backlog (WALM-668). +# tests/ guards that variant and decodes an affected checkpoint; run them after any bump. +sui-indexer-alt-framework = { git = "https://github.com/MystenLabs/sui.git", rev = "ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" } +move-core-types = { git = "https://github.com/MystenLabs/sui.git", rev = "ceaaff1cc84c1ba2f12c4e7928914ece81fbdd0f" } diesel = { version = "2.3", features = ["postgres"] } diesel-async = { version = "0.9", features = ["bb8", "postgres"] } diesel_migrations = "2.3" rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } + +[dev-dependencies] +# Serves tests/fixtures as a local checkpoint store, so the affected-checkpoint +# regression runs the real `StoreIngestionClient` path offline. Same version the +# framework already resolves. +object_store = "0.13" diff --git a/services/indexer/tests/affected_checkpoint.rs b/services/indexer/tests/affected_checkpoint.rs new file mode 100644 index 000000000..61068519f --- /dev/null +++ b/services/indexer/tests/affected_checkpoint.rs @@ -0,0 +1,57 @@ +//! Decodes the checkpoint that stalled staging, through the real ingestion client (WALM-668). +//! +//! `tests/fixtures/384534623.binpb.zst` is testnet checkpoint 384534623 exactly as the +//! remote store serves it — the sample checkpoint named in the staging incident. Serving +//! the fixture directory as a local object store runs the same +//! `StoreIngestionClient::checkpoint` path production uses (zstd -> protobuf -> types), +//! so a pin that cannot decode what testnet produces fails here offline. +//! +//! Under the previous testnet-v1.75.1 pin this checkpoint failed with +//! `Failed to convert checkpoint protobuf to checkpoint data: transaction.bcs: invalid +//! value: integer 3, expected variant index 0 <= i < 3`, because it carries a transaction +//! whose expiration is `TransactionExpiration::Validity` — variant index 3, added in +//! testnet-v1.79.0. + +use std::sync::Arc; + +use object_store::local::LocalFileSystem; +use sui_indexer_alt_framework::ingestion::store_client::StoreIngestionClient; +use sui_indexer_alt_framework::types::transaction::{TransactionDataAPI, TransactionExpiration}; + +const AFFECTED_CHECKPOINT: u64 = 384_534_623; + +#[tokio::test] +async fn decodes_the_checkpoint_that_stalled_staging() { + let fixtures = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures"); + let store = LocalFileSystem::new_with_prefix(fixtures).expect("fixture dir should be readable"); + let client = StoreIngestionClient::new(Arc::new(store), None); + + let checkpoint = client + .checkpoint(AFFECTED_CHECKPOINT) + .await + .expect("checkpoint 384534623 should decode; a pin older than testnet-v1.79.0 fails here"); + + assert_eq!( + checkpoint.summary.sequence_number, AFFECTED_CHECKPOINT, + "fixture should be the checkpoint named in the incident" + ); + + // Guards the fixture itself: if this checkpoint stopped carrying a `Validity` + // expiration it would still decode on the old pin, and the test above would pass + // without exercising anything. + let validity_transactions = checkpoint + .transactions + .iter() + .filter(|tx| { + matches!( + tx.transaction.expiration(), + TransactionExpiration::Validity { .. } + ) + }) + .count(); + + assert!( + validity_transactions > 0, + "fixture no longer carries a Validity expiration, so it no longer reproduces the failure" + ); +} diff --git a/services/indexer/tests/checkpoint_decoding.rs b/services/indexer/tests/checkpoint_decoding.rs new file mode 100644 index 000000000..fa185fcff --- /dev/null +++ b/services/indexer/tests/checkpoint_decoding.rs @@ -0,0 +1,82 @@ +//! Wire-compatibility guard for the pinned Sui types (WALM-668). +//! +//! The indexer does not read individual events off a fullnode — it decodes whole +//! checkpoint files, so *every* transaction in a checkpoint has to deserialize before +//! `AccountPipeline` ever sees an event. A transaction field the pinned types don't +//! know is therefore a hard stop for the checkpoint, not a field that gets skipped: +//! `accounts_v1` retries the same checkpoint forever and its watermark backlog grows. +//! +//! That is what happened on staging. testnet-v1.79.0 added +//! `TransactionExpiration::Validity` as variant index 3; the indexer was pinned to +//! testnet-v1.75.1, which knows variants 0..=2, so once testnet started producing +//! `Validity` transactions every affected checkpoint failed with +//! `transaction.bcs: invalid value: integer 3, expected variant index 0 <= i < 3`. +//! +//! These tests fail on any pin that trails the network in the same way. Run them after +//! bumping the `rev` in Cargo.toml. + +use sui_indexer_alt_framework::types::digests::{ChainIdentifier, CheckpointDigest}; +use sui_indexer_alt_framework::types::transaction::TransactionExpiration; + +fn testnet_chain() -> ChainIdentifier { + ChainIdentifier::from(CheckpointDigest::new([0x11; 32])) +} + +/// The regression: the pinned types must decode the expiration variant testnet emits. +#[test] +fn decodes_validity_expiration() { + let expiration = TransactionExpiration::Validity { + min_epoch: Some(42), + max_epoch: Some(43), + min_timestamp: None, + max_timestamp: None, + chain: testnet_chain(), + nonce: 7, + allowed_proposers: None, + }; + + let bytes = bcs::to_bytes(&expiration).expect("Validity should serialize"); + assert_eq!( + bytes.first(), + Some(&3u8), + "Validity must stay at variant index 3 — that index is what testnet puts on the wire" + ); + + let decoded: TransactionExpiration = + bcs::from_bytes(&bytes).expect("Validity should decode; a pin older than testnet-v1.79.0 fails here"); + assert_eq!(decoded, expiration); +} + +/// Bumping the pin must not renumber the older variants: the checkpoint bucket still +/// serves the whole history, and the indexer replays it from its saved watermark. +#[test] +fn preserves_earlier_expiration_variant_indices() { + let cases: [(u8, TransactionExpiration); 3] = [ + (0, TransactionExpiration::None), + (1, TransactionExpiration::Epoch(99)), + ( + 2, + TransactionExpiration::ValidDuring { + min_epoch: Some(42), + max_epoch: Some(43), + min_timestamp: None, + max_timestamp: None, + chain: testnet_chain(), + nonce: 7, + }, + ), + ]; + + for (index, expiration) in cases { + let bytes = bcs::to_bytes(&expiration).expect("expiration should serialize"); + assert_eq!( + bytes.first(), + Some(&index), + "{expiration:?} moved off variant index {index}, breaking already-indexed history" + ); + + let decoded: TransactionExpiration = + bcs::from_bytes(&bytes).expect("expiration should round-trip"); + assert_eq!(decoded, expiration); + } +} diff --git a/services/indexer/tests/fixtures/384534623.binpb.zst b/services/indexer/tests/fixtures/384534623.binpb.zst new file mode 100644 index 0000000000000000000000000000000000000000..7f20c551b776be44dee4df443a79c7e82927a2ad GIT binary patch literal 9663 zcmV;wB|zFJwJ-euShXzydi5oGI}i}<)d8vh>K!HEBJg`lJ@D`+@ZgAM=Ccb*We=Xe4!aprSm2M#44$i>0qhmOJ_tiaqBc*1XL3pu z4+Rhb!SAZn7$`id23AH~s(c>KVbB)HT5_4jh@}8kpc5paVL+M)|8Un-Z>A=A7146D zpsP}8U9%yEDF&R>P%Aevm_{`^bP~7}gPiTi^ThA}|7{KIVYm0*Y@X`TS{=1kNF)5N z!sxn1*HC*L=zR%c8DbKclyn2)3lFd9lT?kK;D8&!L!_FyG*Gh7$y+&0a0Xi#>Y!AbCAYh=kwO_e$V zffqOFJr8_&h;X3z)&eA`D_s;ShXFCGo{VCQS)})~xwr6Oho12Q0bA?_T$% z$0#pLN?_+eAcagXN(osmDOL3NU=+gPS5v62(4M$ACpZ(W!RFJ{1ZWvr5Tu~v`7L?IBEqp}hS2n`B7Uf-)MHSSLi=;o9hS2KQu1;O;k zAB`0lVhl^9hdhl6Gs%uiJaOMUizwO4CI=THP8S_2mRw#1F{5B0g@epx4B_}N@O}al zU+9jwZDnnDe|ZE9ka#@$L(+TTxF;k%B&1Ld1JGaFu1|`tgI=lEUA~isM1*bvA^C9T zM7X?!IU$H zrh_FC)lO53e7ZQSFytdhi9XUkFfCjdkr4qGre+HYt$)e^lf!!l7E)PorNi_x*fsF5 z;zHwPpO~W5oYTiO1q)l05K*%j{g(|+bHwoaLZqT5(a&teV6fOsdTtDOkr|ieBo;b} zf3b@6LgT}orb;B?5bd)n!jrOTsyIG}Q0`zUEGBlD zoTA#~g2l+G0vmg0A!qOof2all52YZy=xC0um%=0#vZ|wt!eMNO;f+ruA2N_=`DkXr%a?Hdl!)}^`V#NQl@BXKUnU$n8AWnZ zv3r2$)=(`h{M%`TSdC9#NEquZg$)FzM~Ydv6cE@xONj>>3U0(Awma3_k5WttVY2z{7sXm${Q zuCOR_`t>_*J39XC15m0T2Ms~J-33u>ss&?2t0>t9kjRkim^w`J8vt}*nELzTk+I_$4b#QT z&=A}Lj}RPGp`TVRLOo~jtyyk^m;wL=F;apQ6e9K*#psB-A?E3~qmM+R=20L5Dq9>A z)yo*~m!gN`l^O5zs;f|eYZL7l;osM?R*s2@U4D*G;!IV|IW z&}szi19jukkKKq>W6RnDT<7+2>|wXleLJm|>e2c%(fT}5`s$zlWgq>?{@6eNuy6kS z-@pFJzw3AS9R7x{;dA{AU+ZIi4FAHX`WAkLPvKAf318|*_)p)#clzTm{@~yJ`RD$o z&)_Tl1wZL4_(wm%NAQjQfG_lezJUMpeLm0M`8q$q&-pmN=4Ze4SO5M0-}zGi{(tLq z=MmTv#h4@ta!PdIRIxyr(xnxRyr4@?o^#EXTXRcecEaZ2i?kPN*slR~0ErqhfEN&M zdu%3a?n|S8A$@=C=Yj;u5J5XCjK5+~qr2FMC-sb2VPgQpAXfOdnKf0!!|(h5yKe`h zq2>So>Q;sD^#8pWrjy6}|9wPDqu22NZ^Xf8bbE$#tSV*2l1oRFp@AJo&>}eP;+W(D za1m)ZwO9?605!w!Zy67J3(jyE8d09%j&QCldY( zBbaD^!R__JAqi)sYSJ-c+0+j3lyH37)=V(@ZFRHQ!aojs;VOjGks^pXt0Va^i{Cz%DOp>LkD$rn330ClY+M= zQ$k-1PUM(8@xNagY#zHq*eO|UDwaS(2FZZmca%R(8ct5VPQ<8!Z87F$|13#+G;Lu3 z00$98tH=Omqa^(PJ(7J#IK+@6G9>)Lq*>{6XNG~1=L+H%X$GaOZP1<*C}Q^O0F@&R zNp1wOp<$>EzkhFb5^`aY2$Vyx#4|T_q(3f5Xf6_o`ivB;a*K-6KnYtE&8M+`1YNQ~ zP2Ne40G0I1@#6s#u^7``=a`t8C+iPZGXD-W_}C<#6z29u%?a+VFmdE$p(VLeo%}6c z4z?`-Dnd!-NCaNB^0Pv{V`2peFijd|-~zk&X!R9of(VF9?p_JB00`8e*a`N|o^-(h z1^JEKDTofcgqAM&6%>3e*z%6&lV35hA0V5$vRb${yE$uKky# zqwQ%<$R2)wGav&q@N5K-Qs`LZWQK`BPc$|t(Dng4#}Kf0EQ=Ay=TFvo4sbM)qKUY} zVv-(1x+tMPMl@!`iY8UMGpUpI=78Tch&7l7##t6*0W(xmF$LJAD5ix-?I6Z($Q2%{ z4#)O$?<}HdJQAkl+1rSML;M5???`Ha!IKYPm^dMQbkZU4s~J?PaDVa2;nwv@U@O9U zCWBVf;%-jK@QF(g9?|=O24Gl~?tHEUya09xC3Qf^2{LFzRzqEeIRK(hom?eO;X1Dc zjoR>=&x~Uf&r?18(qg-Xuw+PsI)H;tjIB#65%vnM+`T#r+-_E-U^EYwOvO;JhNTKy z&d-XfBdie|R2vzHMh#^^Ox-neIXVji;nXBSs2$D~+V`3W_A8g;}3t+_gM$-tj>xCpAS!^FjC z2%#Z{DmBgtQ5Kw;Un!`9McNkZSWyOxhv3)q?t+!dqqP4HoHeV>(@PM`kEJOpVewYELDz~s^Y?fIjWyuEJO_&SzB7m z9wJ?Mrs9d)sUhV4x8C>7=F@0qa#`Q={Tip=(>hu&(@15AvhMA&6--J5&b_n+al;cY zS}Vfp1gMEQR<aQV`kl5e3OkmohXz@9yna-NqD_QNd!tks1JYUj=N73PlpD5{AFsGOXU%i;*r6q_XGBtwku`}+Si^17+LLh12&%>Vy{ zOY5)14Bdfh$RjX zQ7|47%6;ZNEN&MPo9NNtad-B9LaZTIshPwbLdRgok0OjKZUjcs1lbazw%-TX){20$ z?pbFje*b@;&S|SJrUKZYvaP}a5UNIjMbRP(Rs~Fl;sAJyQ8mG8D0gB-h6?}pp60Se zauRNYCaMz!T~J5OU@&!rsFG5dH)nnDBC~{m8+i9u1AwskH)=IbTczu{3g*_yPa2>* zVz77D%ib_Jte3kyuk&Ml%bW@_F85x& zU)x4$s{;7{EAc{E1&nxQw}os%Z9y?unnStE|!^m3bLiNt%*2VBT4FD~a zlM-1yFh}L~AUFFIak?YTA5JuLT3WZM$~MkzASt&oim5FNL5(V@Hje`$A-wv3Rj+!e zwZ8q^`q_P0S{K{N?Y$o>tI7OYz4z+_dRyo9c3<{Wwcfva|LpJ4!qjN>+KlYePWNR! zy4~*CnvjW-hlh~MKxkWCuEU_9dJfIBG z_w#mnHGRt4k-%+~D!~tk92RAwe>V{`X0r@opb{`7EN=&^$90?q6;N_86G3eFqT<=m zBu8!MUy(Y!*N9+1 zz+i*S5uxr94aXRiFDP6ua|RnqICMU-e!~2XZ7AbmvgE-x<6CE^636-ChzAmJGB8PjJtV_C3sB-}0;5Z63K&R4g|s*_6;E;*2+Ohu zlPr#$_s~;(JB&8KW zf*A-=H;CAocO@$Do38FK=?EDxK4MG0O4yKJpelk_X|ceNd8@?yIrPH2xh0?siBZ$S z*_RJhqM>01xOOd2TgooqrvP_Emt;1ol(-Tl`PI%s@yY~@z~M@Qga1DS)SEG=Ybm;x zl4#T5@G_w#sdDIj37Ho-5FE44m>mP)_nrNdQ+#PCI4~64c6DSvcH#_)F(g>fydgBh zLnt1*4bR35{_oQj2QURYD8UJ3Nv&2FRLGt{k>SV-l0EGNVE6Si-irBJZ6|H4z z#K)g>UO^IAZtk?6vW9TtX5cozBS!W4k}D$Z#$6X5r;YBd(bJJh_Zsj z%Z$K{Hbd%D6Mlb@t#(!&S}P$9whC9H{nsK;v{xxTHFWu}`TyUAm+9YDn{BD-S~PmE zg+WVm^SxGDeQxV%>z-Z4;M1pdG`UCb`dWQ+SGSnaT1P1Txw^cF5!vdpYDSnGl%>fN z7bZH2j?|Dnt#BQ@{*AjxHvFEW!-}_*>30f{0$2>oA?Y(R7a@WN98u;1l&-CHSp7}8 zDz4%8k3y=&mKt5KI}-t8B!@#z43W1OTn;oLo8zwFz$Y08h5}A(3Wl@BpH2vRmN?4I z-{Kae_TeLi<)K|gfqzVFAYnjO^hPXZMpy%ss}0+sKsPPTAZX@d{cbUJd=K) zMJ{RzkR^_PU^o|^;!F^V63E!_gu(CZN39V?NXg5URZaj}#z`sz-xf(op#+lk0W$E2 zmmrb{Mxf&Vrc^-p!HqIqfiuSlOblm)$Tv={sq&lw+>V|h^Th2>+J%o{Z=27T5DxbmP!79rIt%r4G3_Mr4zJM zBd$GP5W!!vIAN~HBC*6sz^dbSCU}G2Tzf&3pe)M}$Z^~loRbU8BT|(y#u zpv1ZzoWD|g2c|(gR2Yk40t*Tz(Q}WxOeey`>`psPD zc4UGEjRoZN)fyCS1>VyYjTE36vj#aMDemRF-qkPb|5wi|8ST}KP0h{E-XAN@xoG4Y zpsRJ~<2!iGcTv@9e>N~HM4NT&xrxkkBoZ1~Gk1!oE}*89~z;KBqk*2B-1-c;NOpae?J!b0R!9tb8rT-6LTVn`MI!|!U;Dil+zA_J%dp{fj^HYi21 zb0LexN`)@+;4<+=f=K3Qs^6OB``&ExBy^HYX4225S(;tlR_6+aV~WT#za$cr!fZOT()J<* z{=TjAW@u!c&F?+AN0Zsi=XIY(=F>X4XRmcLeBB42qsd}zHZ$!37l7L~1YF#Y)YL?pjfG+k4Yh6dY3(ESbiSN zIE}mS8|TcJ>;GBODL0aOzSSDm$xj-fLL{!Nkm6#8mcTkKKd_Q`t%UB9B22im1w!+x zq%t5U2i=VhurEeLv7M%fIk=~1$%)}eT8yqSYsm_yb6ulFmy(FYt1b(afd1-d>j7w| z(bQ4{P|#_sons~gz7-2%9-dIxyXErvxX<(L-s{`4*{gfa;C=fQ_q{IOc`yI%^1N== zb?{yekMmt0i+A((+b!$*x9ipM0sBrLL`+D!e~4R!_SV!^Lp{Oh35j`cA0C@??>O)F z+&=Glo$Cc}%e=W8?#Fo_&coepo|nNn*WCA9^H}G4yL$iC!Q<_?ocBjbs(O4)hiF>{ z!(~f6GiP16xU&1-i@KX~Ui88{WTc1p)qwx|ODz%V;$({q0Fsv#Z~;meH)Ju2CWu-1 zWV68Is?L$Z9@+o@lc`PnJO~Iu;&iLf0%usd$xCD5VwI0u6ykACheu5N+&bbv>w)`r z{8;R->ZtzET1({I@4nT=yZAdEo5ehLTkdsmna{uczIVCc@iuJs@?|c!+vjDk-s`=t z^Ih*y}&=H_(WoR7KP@Ho%M{BADi`renp zWbxu3AG|75#$2 zV7)cMs>@GH=e!`#7BR zU=C~oRW*CBdoweAy@BRuvuZ_aEu_L>^a9uvu{a^}nv&<&Hs^YKFSje^a=ZCh{N?vv z@AJLe=Y78C-7Q!LcXQrb&Ur3)oR`754)%7xyQOu5F`wy133TNl`Wezy;IzR5Ua7t3)+vk_VqO0Yc&NS2SiM-2OoW&a>gIY6?z!Ia z4jzAZu6XaidsiRp;4kH*1KE_te>mMCbmg}TDXBvPxk-tbDT3?!FW*neN%QMA>-Ik8!sf4zB_ z-}$n67lXT7?p7==kIQ227Yq)U!(pzM-{xJu%Fu32hd!D<TORol88;Hcd6GGR4kL=1lD zdSG}1WTc}YZb5Y=0)=PUdNT5UMwijd$a}j?4pN_+&S>nWrgoohv-@^ag9c7)vvynT z=!#yWk<;?zc;_#foPE!D}N@Z&$`&o zN2_;rTfH{Z*L}JVBlmk+ohHJvwb5w0nY^Z#k$1DQBK+%ZbkCFdUQLZg>tr;(Uz6Kn zVdq|ryqlF7{haJ)x6@~}8JT@%>t%BKOy14Ph`^$6pIt^ytEqXkTD_yu{hro$1pjA$ z+4^iwtM?)WLq5BV%%-Dzp4^w6+vN6{tv2gn%D8B)St z#E5%9G(G?#bP)yi#*e^gX4|Zs=7LF}tE5qKiq56V5DpZ+Oqx`aBmZ-#qQ&#W7>SjJ z6!)4S9hpjzh!>F_SArA?o+u+Qu4qsuL=BI#7+bf1J&Un*G3@^~G;9R5Hl|@P*Ua6= z;c>3_Tb?)T=5o&!^ZVE<=5=rX4R^zP59V&We774O>)RdN^>%eI_sf^T+^$d;*bQ|} z@P(;xJL2e3(n~;_O zQ<6>ovAt0C&(?i9O?6X6WYk($Tzh015!#yh?^xf%`nTNc+-=t!?rz86vUvxG!}*un z4f9`bdEDjp;vUTF;$0taw_@GQ{f@Z@YJfR%rVE!CjY{F{QcMzJugVeR&2^IqEOkoZ z%Y*G?amEWS_J5cjWewBI*6Ov&SIwcViea0>f&@~A&>rT!0fOy9o=@T z0Ovv-3q|!?yZBVdd^@oX!Oe@w|GnY4o*JJ?eS)F%6zbFd<&X|c1$DIr5ZdgdR)6G* z#E;n!+h)8~t-gd|2<7++Ui#3hIyWpP>z-)K;m?Y@GK_l}jt|w1b&SMQde_H^0Rw14 zZiE;e>h@Y|+w zEDtA~7{1TsJ+Q@KK zxNk;Ckdfd4pT!L)bSe1uYml~m_@2k~p@+LQ3}X1AKswm3JsWwjn&!Wc$A+k;{n#Dw z+k4N!Ni|OK>CFiOQcM&WhJ@)L%gIimy16^{cPffL))XmRSq6dMMwWp%Yh)8n@vE5sBHa4)t4Lux0Y$Ty%>vJCij zU3Z7Nlv%xa!1wPPYC!hMR~tnpF_DP`hu*k&I=x_m$Gx2nnu)%FIH_XYVZd*F-<|Fd z>kt;y2(hf;l@mu-zqY5Y#FpAyWfFChg)NLHKcp_0LS@LV?iAAMvYK|5g+C2}iK5%u3`wB2! zwon=qk10x{j2i?&lT?`y52Czi2B3#Z4H9fa_f6^c0Aa>x! zpz$inoAQKGWnt6JzK1)i$%0Lv)-H7K)(J9`d&eh4N2`er0Ddx`mqTnNWn-)M?#z-s zx-#0yhN`gC?X4Y3uEsDN_kkYle;w-E>FhXHYQ$cLD_|tGn1!iXsT-XQRX#_}h$)9&`W`rwRr>MU&vL;ZX@;9gBc#^SQ8g{(pbA25!@#iQ54^e%j(*GKLgr@w9{>kOL4yja z1pVED2%nHR__mJ1Ull!GUIgYwh5&qxGbCJzYxg^mj;bae;9e5|t$iZE10?%+flN?_ zoN#t5rBLsf#v*-iKllC&a8bEekQlj2;8S83ck_F2$tOA@F{wP#(AVsI0K>bY1HWrw zO%ez}twAY2$^;geLz&nzRi5I1Y$#+u%g4F85lXM)&?I*}y$c^1R1I%-m{tfzUuo@| ze5R1L-<{_E7E2e)D46_?j?lpLWYNH7mZ*lZq9;yFY#eY!&F-GEV_a-dM4C~FY~4i| zkyZ^HIk9Py(u?D-p+Ll}BpIEB#84M?oc8}#2Q#&|8^mK?VB{|& zM8fB3Lizg+-ypJAQ#!+^bk#9?GeWLSejlbQ4R^*iatyCAS@Xi{61>M!iUrpS>r!96 zM6pA`U{(RTqZv`nxOIqQ!Wm1dB~JN`#(}PE;AHqI)*rYbM8lB52ntFlbO4U+wIa+4 zYSP(`QjeWioy@g1>2$W0A(~w+hMzX92LLQ0&59kRk5U(E9*)cE<{Lf)I80wmgD=fn z@^2V;O-^F;oTY}pq~mw_-AP`V8qg^!0YTdbHL*l?me^>)GbGHy-p5sd6#1x{Cw?s{ z#`lh};B^F$iI9SPk-`9zE|K`;{x~;%u@09N86Y!-^E?tmW7~xyw)fuyG&%~F9i0-m zTtK6797Zq`ay2PK?%rA@kI1Yry%wi_mcd7i18mzKXl22nuf(WTK@68s@!&Kb1RDiH z^gzJSK Date: Thu, 15 Oct 2026 14:54:17 +0700 Subject: [PATCH 62/70] fix(oauth): don't burn the session when Sui is unavailable (WALM-605) (#879) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(oauth): don't burn the session when Sui is unavailable (WALM-605) `session_complete` consumed the one-time authorization session and only then verified the delegate key on-chain, mapping every `verify_delegate_key_onchain` failure — a Sui `GetObject` 429 included — to `invalid_request("delegate key is not registered on-chain")`. A throttled fullnode therefore looked identical to a genuinely unregistered key, and because the session was already spent the user could not retry: they had to restart `/oauth/authorize` from scratch. Follow-up to WALM-429 (#811), which taught signed HTTP auth the same distinction but left this callback behind. - Classify verify failures with `OnchainVerifyError::is_unavailable()`, the same primitive `auth::cache_reverify_action` uses, so the OAuth callback and signed recall cannot drift apart on what a 429 means. - Move the atomic claim below the on-chain read: only a definitive answer (registered or not registered) spends the session; an unreachable chain leaves the row `pending` so the same request can be re-submitted. Single-use is unchanged — no code is minted until the `WHERE status = 'pending'` update wins, and the session-bound wallet proof still gates everything above it. - Return `temporarily_unavailable` 503 with `Retry-After: 5` instead of a dead-end 400, matching signed auth's `AUTH_UPSTREAM_RETRY_AFTER_SECS`. * review: let the consent UI re-send /complete alone (WALM-605) Sparing the session only helps if something re-POSTs /complete. ConnectClaude did not: any /complete failure went to the terminal error step, and a reload re-ran handleConnect including POST /account, so a first-time connect that had already landed add_delegate_key re-submitted the same pubkey, aborted EDelegateKeyAlreadyExists (code 0), and the SPA reported "this wallet is not the owner". The WALM-605 journey still dead-ended. - fetchJson keeps the HTTP status and the OAuth `error` code. - The /complete request moves behind completeSession(), which holds its payload; on temporarily_unavailable / 503 the page shows a retry step that re-sends that request and nothing else. Definitive rejections still land in the terminal error state. - Tests cover both: the retry path re-POSTs /complete once, issues no second preflight and no second transaction; a 400 stays terminal. - session_complete claims inside its match arms, so the Verified path no longer carries an Option + expect() panic on a request path. - Trim the before/after WALM-605 narration and the test-module essay. * review: survive a reload during OAuth recovery (WALM-605) The retry button worked, but it hung off a useRef. A reload dropped it, sent the user back to consent, and the first-time path then re-ran handleConnect, re-submitted add_delegate_key for a key already on chain, and aborted with code 0 — which the SPA still reports as "this wallet is not the owner". Reloading is the obvious thing to try when a page says "retry", so that was still the reported WALM-605 dead end. - The /complete payload is mirrored into sessionStorage under its session id, so a remount resumes at the retry step and re-sends only /complete. The entry is scoped to the session so one authorization's wallet signature can never be replayed against another, and it is cleared on success. - Auto-connect is suppressed while a recovery payload exists, so the resume cannot fall through into a second /account. - Retry copy no longer claims the key is registered. A 503 means Sui could not be consulted, so it says the authorization is still pending instead. - Tests: a reload resumes and completes with no second /account and no second transaction; a payload from another session is ignored. * review: do not leave a resumable crumb after a spent session (WALM-605) The payload was written to sessionStorage at the start of every `/complete` and cleared only on 200. A definitive 400 (Unregistered has already claimed the row) therefore left it behind, and since `session_view` does not filter on `status = 'pending'`, a reload showed the Sui-busy retry step for a session that was already spent, looping there because nothing ever cleared the key. - The ref is still set up front, so a hung in-flight `/complete` can resume. sessionStorage is written only on the retryable branch, and the terminal branch clears it. - Test: a 400 followed by a remount stays on the error step. - Corrected the comment that claimed the payload joins the existing breadcrumb. It gets its own key precisely because the mount effect rewrites `memwal_claude_connect` to `{ session }` and would drop it. * fix(app): stop mapping add_delegate_key abort 0 to an owner mismatch (WALM-605) `account.move` defines EDelegateKeyAlreadyExists = 0 and ENotOwner = 4, so abort 0 means the key is already registered — not that the wallet is wrong. That is the state a WALM-605 connect lands in: the tx went through and the verify behind it hit a Sui 429, so re-running the flow re-submitted the same pubkey and told the user to switch wallets. Treat abort 0 as already-on-chain and go straight to /complete; report the owner mismatch on abort 4, where it belongs. * review: decide the duplicate-key skip on chain, not from the tx error (WALM-605)`alreadyOnChain` could never fire. ConnectClaude signs through `useSponsoredTransaction`, which throws `new Error(sponsorFailureMessage(...))`, and `sponsor::mask_upstream` collapses every upstream 5xx into a bare 502 `sponsor_upstream_error` that never echoes the upstream body. An Enoki dry-run of a duplicate `add_delegate_key` therefore reaches the SPA as "The transaction was rejected by the sponsor" — no abort code, no function name. The abort 4 and abort 2 matches were dead for the same reason. The test that "covered" this mocked a raw `MoveAbort ... abort code: 0` string, which is not what production throws, so it passed while the user dead-ended.Move the decision to where the answer exists:- `/account` verifies this session's own pending key against the chain and returns `resume_pending_registration`. Verified means an earlier attempt landed the tx, so the UI skips straight to `/complete`. - A Sui outage during that check returns a retryable 503 rather than guessing. Registering walks into the masked 502; skipping sends `/complete` at a key that is not there, which is definitive Unregistered and burns the session. The preflight claims nothing, so a retry costs only that request. - The classification goes through `classify_delegate_verify`, so the preflight and `session_complete` cannot drift on what a Sui 429 means (WALM-429). - `session_account` now sanitises `account_id` before the DB lookup and the on-chain read, matching `session_complete`.Client:- Drop all three abort-code branches. The owner-mismatch guidance they carried was unreachable; `session_complete` still reports a real mismatch as a 400. - Write `PENDING_COMPLETE_KEY` before the request instead of only on a retryable failure. A reload while `/complete` is in flight has no 503 to react to, and that is the one gap the ref cannot cover. Terminal exits still clear it, so a spent session cannot loop on the retry screen. - `tx_digest` is `already-on-chain` when resuming, not `reused-delegate`.Tests:- The sponsor 502 is built from the real `SponsorHttpError` / `sponsorFailureMessage` exports, so the fixture cannot drift from the wrapper, and asserts `/complete` is NOT sent — folding every 502 into "already registered" would push `ENotOwner` / `ETooManyDelegateKeys` into `/complete` and burn the session. - `/account` 503 submits no transaction. - A reload with `/complete` still in flight resumes at the retry step with no second signature and no second preflight. - Four Rust tests pin the preflight decision, including that it and `session_complete` agree on which errors are definitive. * review: return a retryable /account 503 to consent (WALM-605) A preflight POST /account 503 leaves the authorization session pending, but the catch sent it to the terminal error, which has no way back. Return to consent and show the reason so the user can start the request again. * review: use the restyled error class on the retry screens (WALM-605) The rebase onto dev dropped errorTextStyle. Both retry messages now use mcp-error, and the finish button uses mcp-btn. --------- Co-authored-by: Harry Phan --- apps/app/src/pages/ConnectClaude.test.tsx | 370 +++++++++++++++++++ apps/app/src/pages/ConnectClaude.tsx | 268 +++++++++++--- services/server/src/oauth.rs | 30 ++ services/server/src/routes/oauth.rs | 428 ++++++++++++++++++++-- 4 files changed, 1018 insertions(+), 78 deletions(-) create mode 100644 apps/app/src/pages/ConnectClaude.test.tsx diff --git a/apps/app/src/pages/ConnectClaude.test.tsx b/apps/app/src/pages/ConnectClaude.test.tsx new file mode 100644 index 000000000..697fabc18 --- /dev/null +++ b/apps/app/src/pages/ConnectClaude.test.tsx @@ -0,0 +1,370 @@ +/** + * WALM-605 recovery path. + * + * A Sui outage during `POST /complete` leaves the authorization session + * `pending` instead of burning it. That only helps if the consent UI can + * re-send *that one request*: re-running the whole flow re-asks the wallet to + * sign and re-submits `add_delegate_key` for a key already on chain. + * + * The duplicate submission is decided server-side, by `/account`'s on-chain + * check. It cannot be decided here: this page signs through the sponsor proxy, + * and `sponsor::mask_upstream` replaces every upstream 5xx with a bare 502 that + * carries no Move abort, so the consent UI never sees which abort fired. + */ +import { cleanup, render, screen, waitFor } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { MemoryRouter } from 'react-router-dom' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + address: '0xowner', + accountId: '0xaccount', + signAndExecute: vi.fn(), + signPersonalMessage: vi.fn(), +})) + +vi.mock('../config', () => ({ + config: { + memwalServerUrl: 'https://relayer.test', + memwalPackageId: '0xpkg', + memwalRegistryId: '0xreg', + suiNetwork: 'testnet', + docsUrl: 'https://docs.test', + }, +})) +vi.mock('@mysten/dapp-kit', () => ({ + ConnectModal: () => null, + useCurrentAccount: () => ({ address: mocks.address }), + useSuiClient: () => ({ waitForTransaction: vi.fn().mockResolvedValue({}) }), + useSignPersonalMessage: () => ({ mutateAsync: mocks.signPersonalMessage }), +})) +vi.mock('../hooks/useSponsoredTransaction', () => ({ + useSponsoredTransaction: () => ({ mutateAsync: mocks.signAndExecute }), +})) +vi.mock('../utils/suiClientCompat', () => ({ + fetchAccountIdForOwner: vi.fn().mockResolvedValue(mocks.accountId), +})) +vi.mock('../utils/analytics', () => ({ + trackEvent: vi.fn(), + getAnalyticsErrorType: () => 'unknown', +})) + +import ConnectClaude from './ConnectClaude' + +const SESSION = `mws_${'a'.repeat(24)}` + +function json(body: unknown, status = 200) { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }) +} + +const SESSION_VIEW = { + client_name: 'Claude', + redirect_host: 'claude.ai', + scopes: ['memory.read'], + delegate_public_key: 'ab'.repeat(32), + delegate_sui_address: '0xdelegate', + expires_at: new Date(Date.now() + 600_000).toISOString(), +} + +/** + * Serves the session GET and the `/account` preflight, and hands each + * `/complete` POST to the next entry of `completeResponses`. Returns a counter + * of how many times each endpoint was called. + */ +function stubRelayer( + completeResponses: (() => Response | Promise)[], + { + needsOnchainRegistration = false, + resumePendingRegistration = false, + accountResponse, + }: { + needsOnchainRegistration?: boolean + resumePendingRegistration?: boolean + accountResponse?: () => Response + } = {}, +) { + const calls = { account: 0, complete: 0, completeBodies: [] as string[] } + vi.stubGlobal( + 'fetch', + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input) + if (url.endsWith(`/session/${SESSION}`)) return json(SESSION_VIEW) + if (url.endsWith('/account')) { + calls.account += 1 + if (accountResponse) return accountResponse() + // Default: reused delegate, so no add_delegate_key transaction + // is needed and the test isolates the /complete round-trip. + return json({ + needs_onchain_registration: needsOnchainRegistration, + delegate_public_key: SESSION_VIEW.delegate_public_key, + delegate_sui_address: SESSION_VIEW.delegate_sui_address, + resume_pending_registration: resumePendingRegistration, + }) + } + if (url.endsWith('/complete')) { + const next = completeResponses[calls.complete] ?? completeResponses.at(-1) + calls.complete += 1 + calls.completeBodies.push(String(init?.body ?? '')) + return next!() + } + throw new Error(`unexpected fetch: ${url}`) + }), + ) + return calls +} + +async function approve() { + render( + + + , + ) + await userEvent.click(await screen.findByRole('button', { name: 'Approve' })) +} + +beforeEach(() => { + vi.clearAllMocks() + sessionStorage.clear() + mocks.signPersonalMessage.mockResolvedValue({ signature: 'sig' }) + // jsdom refuses a real navigation; the page only needs the call to not throw. + vi.stubGlobal('location', { ...window.location, replace: vi.fn() }) +}) + +describe('ConnectClaude /complete recovery', () => { + it('offers a retry of /complete alone when Sui is unavailable', async () => { + const calls = stubRelayer([ + () => + json( + { + error: 'temporarily_unavailable', + error_description: 'could not verify the delegate key on-chain right now', + }, + 503, + ), + () => json({ redirect_url: 'https://claude.ai/callback?code=abc' }), + ]) + + await approve() + + // Not the terminal error state: the session is still pending. + const retry = await screen.findByRole('button', { name: /finish connecting/i }) + expect(screen.queryByText('Something went wrong')).toBeNull() + + await userEvent.click(retry) + + await waitFor(() => expect(calls.complete).toBe(2)) + // The retry re-sent /complete only — no second preflight, so no second + // add_delegate_key submission. + expect(calls.account).toBe(1) + expect(mocks.signAndExecute).not.toHaveBeenCalled() + await screen.findByText(/Redirecting you back/) + }) + + it('a reload mid-recovery resumes at the retry step, not at consent', async () => { + // The ref does not survive a remount. Without the sessionStorage copy + // the user lands back on consent, and a first-time connect re-asks the + // wallet and re-submits add_delegate_key for a key already on chain. + const calls = stubRelayer([ + () => json({ error: 'temporarily_unavailable', error_description: 'sui unavailable' }, 503), + () => json({ redirect_url: 'https://claude.ai/callback?code=abc' }), + ]) + + await approve() + await screen.findByRole('button', { name: /finish connecting/i }) + expect(calls.account).toBe(1) + + // Remount, as a browser reload would. + cleanup() + render( + + + , + ) + + const retry = await screen.findByRole('button', { name: /finish connecting/i }) + await userEvent.click(retry) + + await waitFor(() => expect(calls.complete).toBe(2)) + // Still no second preflight and no second transaction after the reload. + expect(calls.account).toBe(1) + expect(mocks.signAndExecute).not.toHaveBeenCalled() + }) + + it('a definitive rejection leaves nothing to resume after a reload', async () => { + // The session is spent on a 400, so a reload must land on the error and + // not loop on the "Sui is busy" retry screen. + stubRelayer([ + () => json({ error: 'invalid_request', error_description: 'delegate key is not registered on-chain' }, 400), + ]) + + await approve() + await screen.findByText('Something went wrong') + + cleanup() + render( + + + , + ) + + await screen.findByRole('button', { name: 'Approve' }) + expect(screen.queryByRole('button', { name: /finish connecting/i })).toBeNull() + }) + + it('does not replay a stored payload against a different session', async () => { + sessionStorage.setItem( + 'memwal_claude_pending_complete', + JSON.stringify({ + session: 'mws_' + 'z'.repeat(24), + payload: { account_id: '0xa', owner_address: '0xb', owner_signature: 'sig', tx_digest: 'd', reused_delegate: true }, + }), + ) + stubRelayer([() => json({ redirect_url: 'https://claude.ai/callback?code=abc' })]) + + render( + + + , + ) + + // Lands on consent, not on someone else's half-finished authorization. + await screen.findByRole('button', { name: 'Approve' }) + expect(screen.queryByRole('button', { name: /finish connecting/i })).toBeNull() + }) + + it('a reload while /complete is in flight resumes at the retry step', async () => { + // The gap a ref cannot cover, and the reason the crumb is written + // before the request rather than after it fails: /complete never + // answered, so there is no 503 to react to, yet the transaction above + // has already landed and the wallet has already signed. + // Never settles: a reload destroys the context that issued it, so this + // response is never processed by anyone. Resolving it here instead + // would let the dead page run its success path and clear the crumb, + // which no real reload can do. + const inFlight = new Promise(() => {}) + const calls = stubRelayer( + [() => inFlight, () => json({ redirect_url: 'https://claude.ai/callback?code=abc' })], + { needsOnchainRegistration: true }, + ) + mocks.signAndExecute.mockResolvedValue({ digest: '0xdigest' }) + + await approve() + await waitFor(() => expect(calls.complete).toBe(1)) + expect(mocks.signAndExecute).toHaveBeenCalledTimes(1) + + // Reload before the relayer answers. + cleanup() + render( + + + , + ) + + const retry = await screen.findByRole('button', { name: /finish connecting/i }) + await userEvent.click(retry) + + await waitFor(() => expect(calls.complete).toBe(2)) + // No second preflight, and above all no second wallet signature or + // second add_delegate_key for a key the chain already took. + expect(calls.account).toBe(1) + expect(mocks.signAndExecute).toHaveBeenCalledTimes(1) + }) + + it('skips add_delegate_key when /account says this session key is already on chain', async () => { + // The WALM-605 journey: an earlier attempt landed the transaction and + // then hit a 429 during verify. The chain, not a transaction error, is + // what tells us there is nothing left to submit. + const calls = stubRelayer([() => json({ redirect_url: 'https://claude.ai/callback?code=abc' })], { + needsOnchainRegistration: false, + resumePendingRegistration: true, + }) + + await approve() + + await screen.findByText(/Redirecting you back/) + expect(mocks.signAndExecute).not.toHaveBeenCalled() + expect(calls.complete).toBe(1) + // Named honestly: this is not a delegate reused from an earlier grant. + expect(JSON.parse(calls.completeBodies[0]!).tx_digest).toBe('already-on-chain') + }) + + it('does not read a duplicate-key abort out of the masked sponsor 502', async () => { + // Built from the real exports, so the fixture cannot drift from the + // wrapper the way a hand-written 'MoveAbort ... abort code: 0' string + // did: that string is not what production throws, so a test using it + // passes while the user dead-ends. + const { SponsorHttpError, sponsorFailureMessage } = await vi.importActual< + typeof import('../hooks/useSponsoredTransaction') + >('../hooks/useSponsoredTransaction') + const masked = sponsorFailureMessage( + new SponsorHttpError( + 'sponsor', + 502, + JSON.stringify({ + error: 'Sponsor service error', + code: 'sponsor_upstream_error', + traceId: 'tr_1', + }), + ), + ) + // The premise: an Enoki dry-run abort reaches the SPA with no Move text. + expect(masked).not.toMatch(/abort code/) + expect(masked).not.toMatch(/add_delegate_key/) + + const calls = stubRelayer([() => json({ redirect_url: 'https://claude.ai/callback?code=abc' })], { + needsOnchainRegistration: true, + }) + mocks.signAndExecute.mockRejectedValue(new Error(masked)) + + await approve() + + await screen.findByText('Something went wrong') + // The load-bearing assertion: a sponsor 502 must not be treated as + // "already registered" and POSTed to /complete. ENotOwner, + // ETooManyDelegateKeys and EDelegateKeyAlreadyExists are + // indistinguishable here, and guessing the last one burns the session + // as Unregistered for the other two. + expect(calls.complete).toBe(0) + }) + + it('returns to consent when the /account preflight cannot reach Sui', async () => { + const calls = stubRelayer([() => json({ redirect_url: 'https://claude.ai/callback?code=abc' })], { + accountResponse: () => + json( + { error: 'temporarily_unavailable', error_description: 'sui unavailable' }, + 503, + ), + }) + + await approve() + + // The session is still pending. The terminal error has no way back, + // so the user must be able to start this request again. + expect(screen.queryByText('Something went wrong')).toBeNull() + expect(await screen.findByText('sui unavailable')).toBeTruthy() + expect(screen.getByRole('button', { name: 'Approve' })).toBeTruthy() + expect(mocks.signAndExecute).not.toHaveBeenCalled() + expect(calls.complete).toBe(0) + }) + + it('still fails terminally on a definitive rejection', async () => { + stubRelayer([ + () => + json( + { + error: 'invalid_request', + error_description: 'delegate key is not registered on-chain', + }, + 400, + ), + ]) + + await approve() + + await screen.findByText('Something went wrong') + expect(screen.queryByRole('button', { name: /finish connecting/i })).toBeNull() + }) +}) diff --git a/apps/app/src/pages/ConnectClaude.tsx b/apps/app/src/pages/ConnectClaude.tsx index 0a1f1af8e..c01c7c09f 100644 --- a/apps/app/src/pages/ConnectClaude.tsx +++ b/apps/app/src/pages/ConnectClaude.tsx @@ -28,7 +28,7 @@ * 6. `window.location.replace(redirect_url)` — hands control back to * Claude (or whatever OAuth client started the flow). */ -import { useCallback, useEffect, useMemo, useState } from 'react' +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { ConnectModal, useCurrentAccount, @@ -44,11 +44,46 @@ import { fetchAccountIdForOwner } from '../utils/suiClientCompat' +/** The `POST /complete` body, plus the analytics flag that goes with it. */ +type CompletePayload = { + account_id: string + owner_address: string + owner_signature: string + tx_digest: string + reused_delegate: boolean +} + +/** Where the in-flight `POST /complete` payload survives a reload. */ +const PENDING_COMPLETE_KEY = 'memwal_claude_pending_complete' + +/** + * The saved `POST /complete` payload for this session, if any. + * + * Session-scoped on purpose: one authorization's wallet signature must never be + * replayed against another. + */ +function loadStoredComplete(sessionId: string): CompletePayload | null { + try { + const raw = sessionStorage.getItem(PENDING_COMPLETE_KEY) + if (!raw) return null + const saved = JSON.parse(raw) as { session?: string; payload?: CompletePayload } + if (saved.session !== sessionId || !saved.payload?.owner_signature) return null + return saved.payload + } catch { + return null + } +} + +function hasStoredComplete(sessionId: string): boolean { + return loadStoredComplete(sessionId) !== null +} + type Step = | 'loading' | 'consent' | 'signing' | 'finishing' + | 'retry-complete' | 'redirecting' | 'no-account' | 'error' @@ -82,6 +117,8 @@ async function resolveAccountId( } } +type OAuthRequestError = Error & { status: number; oauthError?: string } + async function fetchJson(url: string, init?: RequestInit): Promise { const res = await fetch(url, { ...init, @@ -89,15 +126,31 @@ async function fetchJson(url: string, init?: RequestInit): Promise { }) const body = await res.json().catch(() => null) if (!res.ok) { + const envelope = body && typeof body === 'object' ? (body as Record) : null const description = - body && typeof body === 'object' && 'error_description' in body - ? String((body as { error_description: unknown }).error_description) + envelope && 'error_description' in envelope + ? String(envelope.error_description) : `request failed (${res.status})` - throw new Error(description) + const err = new Error(description) as OAuthRequestError + err.status = res.status + if (envelope && typeof envelope.error === 'string') err.oauthError = envelope.error + throw err } return body as T } +/** + * The relayer could not reach Sui, so it left the authorization session + * `pending` instead of burning it (WALM-605). Re-POSTing `/complete` alone is + * the recovery: the wallet signature and the on-chain registration from the + * first attempt are both still good, so restarting `handleConnect` would only + * re-ask the wallet and re-submit a transaction the chain has already taken. + */ +function isRetryableComplete(err: unknown): boolean { + const e = err as Partial | null + return e?.oauthError === 'temporarily_unavailable' || e?.status === 503 +} + export default function ConnectClaude() { const [params] = useSearchParams() const sessionId = params.get('session') ?? '' @@ -126,7 +179,10 @@ export default function ConnectClaude() { .then((view) => { if (cancelled) return setSession(view) - setStep('consent') + // A reload mid-recovery resumes at the retry step. Landing on + // consent would re-run handleConnect, which re-asks the wallet + // to sign and re-sends a transaction the chain already took. + setStep(hasStoredComplete(sessionId) ? 'retry-complete' : 'consent') }) .catch((err) => { if (cancelled) return @@ -146,6 +202,109 @@ export default function ConnectClaude() { sessionStorage.setItem('memwal_claude_connect', JSON.stringify({ session: sessionId })) }, [sessionValid, sessionId]) + + // Everything `POST /complete` needs, kept so a retry re-sends exactly this + // request instead of re-running `handleConnect`, which would re-ask the + // wallet for a signature and re-submit `add_delegate_key` for a key that + // is already on chain. + // + // A ref does not survive a reload, and a reload is the obvious thing to try + // when the relayer says "retry", so the payload also goes to sessionStorage. + // It gets its own key rather than joining `memwal_claude_connect`, which + // the mount effect rewrites to `{ session }` on every load and would drop + // the payload. + const pendingComplete = useRef(null) + + const forgetPendingComplete = useCallback(() => { + pendingComplete.current = null + try { + sessionStorage.removeItem(PENDING_COMPLETE_KEY) + } catch { + // Nothing to clean up if storage is unavailable. + } + }, []) + + const rememberPendingComplete = useCallback( + (payload: CompletePayload) => { + pendingComplete.current = payload + try { + sessionStorage.setItem( + PENDING_COMPLETE_KEY, + JSON.stringify({ session: sessionId, payload }), + ) + } catch { + // Private mode or a full quota: the in-page button still works. + } + }, + [sessionId], + ) + + const readPendingComplete = useCallback((): CompletePayload | null => { + if (pendingComplete.current) return pendingComplete.current + const saved = loadStoredComplete(sessionId) + if (saved) pendingComplete.current = saved + return saved + }, [sessionId]) + + const completeSession = useCallback(async (payload: CompletePayload) => { + // Durable before the request, not after it fails. A reload while + // /complete is still in flight is the gap a ref cannot cover, and on + // the first-time path it is the expensive one: without the crumb the + // page lands on consent, re-runs handleConnect, and re-submits + // add_delegate_key for a key the tx above already put on chain. That + // no longer dead-ends now that /account detects it, but it still costs + // a wallet round-trip and a sponsored transaction. + // + // Writing early is safe because every terminal exit clears it: success + // below, and a definitive 400 in the catch. A spent session therefore + // cannot leave a crumb that loops the user on "Sui is busy". + rememberPendingComplete(payload) + setStep('finishing') + try { + const { redirect_url } = await fetchJson<{ redirect_url: string }>( + `${apiBase}/api/oauth/session/${sessionId}/complete`, + { + method: 'POST', + body: JSON.stringify({ + account_id: payload.account_id, + owner_address: payload.owner_address, + owner_signature: payload.owner_signature, + tx_digest: payload.tx_digest, + }), + }, + ) + sessionStorage.removeItem('memwal_claude_connect') + sessionStorage.removeItem(PENDING_COMPLETE_KEY) + setStep('redirecting') + trackEvent('claude_connect_complete', { reused_delegate: payload.reused_delegate }) + window.location.replace(redirect_url) + } catch (err) { + if (!isRetryableComplete(err)) { + // Definitive: the session is spent. Clear the crumb so a reload + // lands on the error rather than looping on "Sui is busy". + forgetPendingComplete() + throw err + } + // The session is still pending, so offer this one request again + // rather than dropping into the terminal error state. The crumb + // written above is what the retry step and a reload both read. + setErrorMsg(err instanceof Error ? err.message : String(err)) + setStep('retry-complete') + trackEvent('claude_connect_failed', { error_type: 'sui_unavailable' }) + } + }, [apiBase, sessionId, rememberPendingComplete, forgetPendingComplete]) + + const handleRetryComplete = useCallback(async () => { + const payload = readPendingComplete() + if (!payload) return + try { + await completeSession(payload) + } catch (err) { + setErrorMsg(err instanceof Error ? err.message : String(err)) + setStep('error') + trackEvent('claude_connect_failed', { error_type: getAnalyticsErrorType(err) }) + } + }, [completeSession, readPendingComplete]) const handleConnect = useCallback(async () => { if (!session) return if (!currentAccount) { @@ -168,6 +327,9 @@ export default function ConnectClaude() { needs_onchain_registration: boolean delegate_public_key: string delegate_sui_address: string + // Absent on a relayer older than WALM-605, which is the + // pre-existing behaviour: no resume, just the first-time path. + resume_pending_registration?: boolean }>(`${apiBase}/api/oauth/session/${sessionId}/account`, { method: 'POST', body: JSON.stringify({ account_id: accountId, owner_address: currentAccount.address }), @@ -187,30 +349,18 @@ export default function ConnectClaude() { tx.object('0x6'), ], }) - let result - try { - result = await signAndExecute({ transaction: tx }) - } catch (txErr: unknown) { - const m = txErr instanceof Error ? txErr.message : String(txErr) - if (m.includes('abort code: 0') && m.includes('add_delegate_key')) { - setErrorMsg( - `This wallet (${currentAccount.address.slice(0, 10)}…${currentAccount.address.slice(-6)}) is not the owner of Walrus Memory account ${accountId.slice(0, 10)}…${accountId.slice(-6)}. ` + - `Switch to the wallet that created this account, or run /setup for a new one.` - ) - trackEvent('claude_connect_failed', { error_type: 'owner_mismatch' }) - setStep('error') - return - } - if (m.includes('abort code: 2') && m.includes('add_delegate_key')) { - setErrorMsg( - `This account already has the maximum number of delegate keys (20). Go to /dashboard and revoke an unused key, then try again.` - ) - trackEvent('claude_connect_failed', { error_type: 'max_delegate_keys' }) - setStep('error') - return - } - throw txErr - } + // Deliberately no `account.move` abort-code branches here. + // This page signs through `useSponsoredTransaction`, which + // throws `new Error(sponsorFailureMessage(...))`, and the + // relayer's `sponsor::mask_upstream` collapses any upstream + // 5xx into a bare 502 `sponsor_upstream_error` that never + // echoes the Move abort. `EDelegateKeyAlreadyExists` (0), + // `ETooManyDelegateKeys` (2) and `ENotOwner` (4) therefore all + // reach us as the same opaque sponsor string; matching on them + // read like recovery while never firing. The duplicate-key + // case is settled before this point, by `/account`'s on-chain + // check (`resume_pending_registration`). + const result = await signAndExecute({ transaction: tx }) await suiClient.waitForTransaction({ digest: result.digest }) txDigest = result.digest } @@ -220,28 +370,31 @@ export default function ConnectClaude() { `Walrus Memory OAuth authorization\nsession:${sessionId}\naccount:${accountId.toLowerCase()}\nowner:${currentAccount.address.toLowerCase()}`, ) const { signature: ownerSignature } = await signPersonalMessage({ message: proofMessage }) - const { redirect_url } = await fetchJson<{ redirect_url: string }>( - `${apiBase}/api/oauth/session/${sessionId}/complete`, - { - method: 'POST', - body: JSON.stringify({ - account_id: accountId, - owner_address: currentAccount.address, - owner_signature: ownerSignature, - tx_digest: txDigest || 'reused-delegate', - }), - }, - ) - sessionStorage.removeItem('memwal_claude_connect') - setStep('redirecting') - trackEvent('claude_connect_complete', { reused_delegate: !preflight.needs_onchain_registration }) - window.location.replace(redirect_url) + const resuming = preflight.resume_pending_registration === true + await completeSession({ + account_id: accountId, + owner_address: currentAccount.address, + owner_signature: ownerSignature, + // `/complete` requires a non-empty digest. When no tx was sent + // this call names why, rather than claiming a reused delegate + // for a key this session registered on an earlier attempt. + tx_digest: txDigest || (resuming ? 'already-on-chain' : 'reused-delegate'), + reused_delegate: !preflight.needs_onchain_registration && !resuming, + }) } catch (err) { setErrorMsg(err instanceof Error ? err.message : String(err)) + if (isRetryableComplete(err)) { + // The session is still pending. The terminal error has no + // button back, so return to consent and let the user start + // this request again. + setStep('consent') + trackEvent('claude_connect_failed', { error_type: 'sui_unavailable' }) + return + } setStep('error') trackEvent('claude_connect_failed', { error_type: getAnalyticsErrorType(err) }) } - }, [session, currentAccount, suiClient, signAndExecute, signPersonalMessage, apiBase, sessionId]) + }, [session, currentAccount, suiClient, signAndExecute, signPersonalMessage, completeSession]) const handleCancel = useCallback(async () => { try { @@ -260,6 +413,7 @@ export default function ConnectClaude() { // Auto-proceed once the wallet popup resolves. useEffect(() => { if (!walletPickerOpen && currentAccount && step === 'consent') { + if (hasStoredComplete(sessionId)) return void handleConnect() } // eslint-disable-next-line react-hooks/exhaustive-deps @@ -284,7 +438,7 @@ export default function ConnectClaude() { )} {step === 'consent' && session && ( - + )} {step === 'signing' && ( @@ -322,6 +476,23 @@ export default function ConnectClaude() {
)} + {step === 'retry-complete' && ( +
+

Sui is busy — try again

+

+ We couldn't reach Sui to verify your delegate key, so this authorization is + still pending rather than spent. Nothing needs redoing — press the button to + finish. +

+

{errorMsg}

+
+ +
+
+ )} + {step === 'error' && (

Something went wrong

@@ -339,11 +510,13 @@ export default function ConnectClaude() { function ConsentCard({ session, wallet, + error, onConnect, onCancel, }: { session: SessionView wallet: string | null + error?: string onConnect: () => void onCancel: () => void }) { @@ -381,6 +554,7 @@ function ConsentCard({
+ {error ?

{error}

: null}