diff --git a/cmd/engram/doctor.go b/cmd/engram/doctor.go index fcb30c02d..f94b2804e 100644 --- a/cmd/engram/doctor.go +++ b/cmd/engram/doctor.go @@ -82,6 +82,8 @@ func cmdDoctor(cfg store.Config) { func printDoctorUsage() { fmt.Fprintln(os.Stdout, "usage: engram doctor [--json] [--project PROJECT] [--check CODE]") fmt.Fprintln(os.Stdout, " engram doctor repair --project PROJECT --check CODE (--plan|--dry-run|--apply)") + fmt.Fprintln(os.Stdout, " engram doctor repair [--project PROJECT] --check "+diagnostic.CheckSyncMutationRequiredFields+" (--plan|--dry-run|--apply)") + fmt.Fprintln(os.Stdout, "note: --project is required for every repair check except "+diagnostic.CheckSyncMutationRequiredFields+", where it optionally scopes the quarantine to one project.") fmt.Fprintln(os.Stdout, "checks: "+strings.Join(diagnostic.RegisteredCodes(), ", ")) } @@ -127,7 +129,7 @@ func cmdDoctorRepair(cfg store.Config) { project, _ = store.NormalizeProject(project) project = strings.TrimSpace(project) check = strings.TrimSpace(check) - if project == "" { + if project == "" && check != diagnostic.CheckSyncMutationRequiredFields { failDoctorRepair("--project is required") return } @@ -150,6 +152,15 @@ func cmdDoctorRepair(cfg store.Config) { return } defer s.Close() + if check == diagnostic.CheckSyncMutationRequiredFields { + report, err := s.QuarantineIrreparableSyncMutations(project, mode == diagnostic.RepairModeApply) + if err != nil { + failDoctorRepair(err.Error()) + return + } + writeDoctorRepairJSON(report) + return + } ctx := context.Background() report, err := runDiagnostics(ctx, s, project, check) @@ -200,7 +211,7 @@ func cmdDoctorRepair(cfg store.Config) { func isSupportedDoctorRepairCheck(check string) bool { switch check { - case diagnostic.CheckSessionProjectDirectoryMismatch, diagnostic.CheckManualSessionNameProjectMismatch: + case diagnostic.CheckSessionProjectDirectoryMismatch, diagnostic.CheckManualSessionNameProjectMismatch, diagnostic.CheckSyncMutationRequiredFields: return true default: return false @@ -213,8 +224,8 @@ func failDoctorRepair(message string) { exitFunc(1) } -func writeDoctorRepairJSON(plan diagnostic.RepairPlan) { - out, err := jsonMarshalIndent(plan, "", " ") +func writeDoctorRepairJSON(value any) { + out, err := jsonMarshalIndent(value, "", " ") if err != nil { fatal(err) return diff --git a/cmd/engram/doctor_test.go b/cmd/engram/doctor_test.go index fb3d13355..b14b6a80e 100644 --- a/cmd/engram/doctor_test.go +++ b/cmd/engram/doctor_test.go @@ -152,7 +152,7 @@ func TestCmdDoctorRepairValidation(t *testing.T) { {name: "missing mode", args: []string{"engram", "doctor", "repair", "--project", "sias-app", "--check", "session_project_directory_mismatch"}, want: "exactly one of --plan, --dry-run, or --apply is required"}, {name: "multiple modes", args: []string{"engram", "doctor", "repair", "--project", "sias-app", "--check", "session_project_directory_mismatch", "--plan", "--apply"}, want: "exactly one of --plan, --dry-run, or --apply is required"}, {name: "missing project", args: []string{"engram", "doctor", "repair", "--check", "session_project_directory_mismatch", "--plan"}, want: "--project is required"}, - {name: "unsupported check", args: []string{"engram", "doctor", "repair", "--project", "sias-app", "--check", "sync_mutation_required_fields", "--plan"}, want: "unsupported repair check"}, + {name: "unsupported check", args: []string{"engram", "doctor", "repair", "--project", "sias-app", "--check", "not_real", "--plan"}, want: "unsupported repair check"}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -377,6 +377,160 @@ func TestCmdDoctorSyncMutationRequiredFieldsBlockedEnvelope(t *testing.T) { } } +func TestCmdDoctorRepairQuarantinesOnlyIrreparableMutations(t *testing.T) { + cfg := testConfig(t) + s, err := store.New(cfg) + if err != nil { + t.Fatalf("store.New: %v", err) + } + if err := s.Close(); err != nil { + t.Fatalf("close store: %v", err) + } + seedDoctorPendingMutation(t, cfg, "", store.SyncEntitySession, "poison", store.SyncOpUpsert, `{"id":"poison"}`) + seedDoctorPendingMutation(t, cfg, "", store.SyncEntitySession, "later", store.SyncOpDelete, `{"id":"later"}`) + + withArgs(t, "engram", "doctor", "repair", "--check", "sync_mutation_required_fields", "--dry-run") + dryOut, dryErr := captureOutput(t, func() { cmdDoctor(cfg) }) + if dryErr != "" { + t.Fatalf("dry-run stderr=%q", dryErr) + } + dry := decodeRepairPlan(t, dryOut) + if dry["applied"] != false || len(dry["actions"].([]any)) != 1 { + t.Fatalf("dry-run=%v", dry) + } + + withArgs(t, "engram", "doctor", "repair", "--check", "sync_mutation_required_fields", "--apply") + applyOut, applyErr := captureOutput(t, func() { cmdDoctor(cfg) }) + if applyErr != "" { + t.Fatalf("apply stderr=%q", applyErr) + } + applied := decodeRepairPlan(t, applyOut) + if applied["applied"] != true || len(applied["actions"].([]any)) != 1 { + t.Fatalf("apply=%v", applied) + } + db, err := sql.Open("sqlite", filepath.Join(cfg.DataDir, "engram.db")) + if err != nil { + t.Fatalf("sql.Open: %v", err) + } + defer db.Close() + var poison, later string + if err := db.QueryRow(`SELECT disposition FROM sync_mutations WHERE entity_key = 'poison'`).Scan(&poison); err != nil { + t.Fatalf("read poison: %v", err) + } + if err := db.QueryRow(`SELECT disposition FROM sync_mutations WHERE entity_key = 'later'`).Scan(&later); err != nil { + t.Fatalf("read later: %v", err) + } + if poison != store.SyncMutationDispositionQuarantined || later != store.SyncMutationDispositionPending { + t.Fatalf("dispositions poison=%q later=%q", poison, later) + } +} + +func TestCmdDoctorRepairApplyUnblocksDoctorAndKeepsPendingWork(t *testing.T) { + cfg := testConfig(t) + s, err := store.New(cfg) + if err != nil { + t.Fatalf("store.New: %v", err) + } + if err := s.Close(); err != nil { + t.Fatalf("close store: %v", err) + } + seedDoctorPendingMutation(t, cfg, "engram", store.SyncEntitySession, "poison", store.SyncOpUpsert, `{"id":"poison"}`) + seedDoctorPendingMutation(t, cfg, "engram", store.SyncEntitySession, "keep", store.SyncOpUpsert, `{"id":"keep","directory":"/work/engram"}`) + // `engram` is enrolled on purpose: the repair contract this test pins is the + // cloud one, so the check must run past the cloud-sync gate instead of taking + // the local-only early return. + enrollDoctorProject(t, cfg, "engram") + + runDoctor := func(stage string) map[string]any { + t.Helper() + withArgs(t, "engram", "doctor", "--json", "--project", "engram", "--check", "sync_mutation_required_fields") + stdout, stderr := captureOutput(t, func() { cmdDoctor(cfg) }) + if stderr != "" { + t.Fatalf("%s stderr=%q", stage, stderr) + } + var report map[string]any + if err := json.Unmarshal([]byte(stdout), &report); err != nil { + t.Fatalf("%s doctor json invalid: %v\n%s", stage, err, stdout) + } + return report + } + + if report := runDoctor("before repair"); report["status"] != "blocked" { + t.Fatalf("expected blocked doctor before repair, got %v", report) + } + + withArgs(t, "engram", "doctor", "repair", "--project", "engram", "--check", "sync_mutation_required_fields", "--apply") + applyOut, applyErr := captureOutput(t, func() { cmdDoctor(cfg) }) + if applyErr != "" { + t.Fatalf("apply stderr=%q", applyErr) + } + applied := decodeRepairPlan(t, applyOut) + if applied["applied"] != true || len(applied["actions"].([]any)) != 1 { + t.Fatalf("apply=%v", applied) + } + + report := runDoctor("after repair") + if report["status"] == "blocked" { + t.Fatalf("doctor stayed blocked after quarantine repair: %v", report) + } + check := report["checks"].([]any)[0].(map[string]any) + if check["result"] == "blocked" || check["severity"] == "blocking" { + t.Fatalf("check stayed blocking after quarantine repair: %v", check) + } + findings := check["findings"].([]any) + if len(findings) != 1 { + t.Fatalf("expected the quarantined row to remain visible as evidence, got %v", findings) + } + finding := findings[0].(map[string]any) + if finding["severity"] != "info" || finding["reason_code"] != "sync_mutation_quarantined" || finding["requires_confirmation"] != false { + t.Fatalf("unexpected quarantined finding: %v", finding) + } + evidence := finding["evidence"].(map[string]any) + if evidence["entity_key"] != "poison" || evidence["disposition"] != store.SyncMutationDispositionQuarantined { + t.Fatalf("quarantined evidence lost mutation identity: %v", evidence) + } + + reopened, err := store.New(cfg) + if err != nil { + t.Fatalf("reopen store: %v", err) + } + defer reopened.Close() + pending, err := reopened.HasPendingSyncMutationsForProject("engram") + if err != nil || !pending { + t.Fatalf("HasPendingSyncMutationsForProject=%v err=%v", pending, err) + } + for _, targetKey := range []string{store.DefaultSyncTargetKey, store.DefaultSyncTargetKey + ":engram"} { + state, err := reopened.GetSyncState(targetKey) + if err != nil { + t.Fatalf("state for %q: %v", targetKey, err) + } + if state.Lifecycle != store.SyncLifecyclePending { + t.Fatalf("quarantine repair masked pending work for %q: lifecycle=%q", targetKey, state.Lifecycle) + } + } +} + +func TestPrintDoctorUsageMarksProjectOptionalOnlyForSyncMutationRepair(t *testing.T) { + withArgs(t, "engram", "doctor", "--help") + stdout, stderr := captureOutput(t, func() { cmdDoctor(testConfig(t)) }) + if stderr != "" { + t.Fatalf("stderr=%q", stderr) + } + wantLines := []string{ + "usage: engram doctor [--json] [--project PROJECT] [--check CODE]", + " engram doctor repair --project PROJECT --check CODE (--plan|--dry-run|--apply)", + " engram doctor repair [--project PROJECT] --check sync_mutation_required_fields (--plan|--dry-run|--apply)", + } + for _, line := range wantLines { + if !strings.Contains(stdout, line+"\n") { + t.Fatalf("usage missing line %q\n%s", line, stdout) + } + } + if !strings.Contains(stdout, "checks: ") { + t.Fatalf("usage lost the registered check list\n%s", stdout) + } +} + func TestCmdDoctorNonEnrolledPendingMutationsBlockedEnvelope(t *testing.T) { cfg := testConfig(t) seedDoctorSession(t, cfg, "manual-save-bootstrap", "bootstrap", "/work/bootstrap") diff --git a/internal/diagnostic/checks.go b/internal/diagnostic/checks.go index 691489da8..57c009502 100644 --- a/internal/diagnostic/checks.go +++ b/internal/diagnostic/checks.go @@ -152,8 +152,16 @@ func (c SyncMutationRequiredFieldsCheck) Run(ctx context.Context, scope Scope) ( if err != nil { return CheckResult{}, err } - findings := make([]Finding, 0) + blocking := make([]Finding, 0) + quarantined := make([]Finding, 0) for _, mutation := range mutations { + // A quarantined row is an explicit, already-taken disposition: it no + // longer reaches transport, so it must not keep doctor blocked. It stays + // reported as non-blocking evidence of what was dropped from sync. + if strings.TrimSpace(mutation.Disposition) == store.SyncMutationDispositionQuarantined { + quarantined = append(quarantined, c.quarantinedFinding(mutation)) + continue + } validation := store.ValidateSyncMutationPayload(mutation.Entity, mutation.Op, mutation.Payload, mutation.EntityKey) if validation.ReasonCode == "" { continue @@ -162,7 +170,7 @@ func (c SyncMutationRequiredFieldsCheck) Run(ctx context.Context, scope Scope) ( if strings.TrimSpace(scope.Project) != "" { nextStep = "Run `engram cloud upgrade doctor --project " + scope.Project + "` and inspect the mutation payload before any manual repair." } - findings = append(findings, Finding{ + blocking = append(blocking, Finding{ CheckID: c.Code(), Severity: SeverityBlocking, ReasonCode: validation.ReasonCode, @@ -173,20 +181,35 @@ func (c SyncMutationRequiredFieldsCheck) Run(ctx context.Context, scope Scope) ( RequiresConfirmation: true, }) } + // Quarantined rows are already-taken dispositions, so they never count as + // work still pending delivery. + evidence := map[string]any{"pending_mutations_evaluated": len(mutations) - len(quarantined)} + if len(quarantined) > 0 { + evidence["quarantined_mutations"] = len(quarantined) + } + // Blocking findings lead the roll-up so the check summary always describes the + // work that still needs a decision rather than already-dispositioned evidence. + rollUp := func() []Finding { return append(append([]Finding{}, blocking...), quarantined...) } + // A non-enrolled backlog is only a fault on a device that actually uses // cloud sync. The store journals sync mutations unconditionally, so on a // local-only install every pending mutation belongs to a non-enrolled // project by definition — the normal steady state, not something doctor // should block on and answer with `engram cloud enroll`. This mirrors the // autosync manager, which owns the same reason code and only evaluates it - // while cloud sync is configured and running. + // while cloud sync is configured and running. The gate is deliberately + // placed after the payload/quarantine pass so a local-only install still + // gets its quarantined evidence reported instead of silently dropped. usesCloudSync, err := cloudSyncInUse(scope) if err != nil { return CheckResult{}, err } if !usesCloudSync { - return resultFromFindings(c.Code(), map[string]any{"pending_mutations_evaluated": len(mutations)}, findings), nil + return resultFromFindings(c.Code(), evidence, rollUp()), nil } + // CountPendingNonEnrolledSyncMutations only counts rows whose disposition is + // still `pending`, so a quarantined row can never resurrect this blocking + // finding: the backlog it reports is genuinely undeliverable work. nonEnrolledCounts, err := scope.Store.CountPendingNonEnrolledSyncMutations(store.DefaultSyncTargetKey) if err != nil { return CheckResult{}, err @@ -197,7 +220,7 @@ func (c SyncMutationRequiredFieldsCheck) Run(ctx context.Context, scope Scope) ( if scopedProject != "" && project != scopedProject { continue } - findings = append(findings, Finding{ + blocking = append(blocking, Finding{ CheckID: c.Code(), Severity: SeverityBlocking, ReasonCode: constants.ReasonNonEnrolledPendingMutations, @@ -208,7 +231,31 @@ func (c SyncMutationRequiredFieldsCheck) Run(ctx context.Context, scope Scope) ( RequiresConfirmation: true, }) } - return resultFromFindings(c.Code(), map[string]any{"pending_mutations_evaluated": len(mutations)}, findings), nil + return resultFromFindings(c.Code(), evidence, rollUp()), nil +} + +func (c SyncMutationRequiredFieldsCheck) quarantinedFinding(mutation store.SyncMutation) Finding { + return Finding{ + CheckID: c.Code(), + Severity: SeverityInfo, + ReasonCode: "sync_mutation_quarantined", + Message: "Sync mutation is quarantined and no longer blocks cloud replication.", + Why: "Quarantine keeps the irreparable journal row as durable local evidence while removing it from transport, so doctor reports it instead of staying blocked forever.", + Evidence: mustJSON(map[string]any{ + "seq": mutation.Seq, + "target_key": mutation.TargetKey, + "project": mutation.Project, + "entity": mutation.Entity, + "op": mutation.Op, + "entity_key": mutation.EntityKey, + "disposition": mutation.Disposition, + "disposition_reason": mutation.DispositionReason, + "disposition_evidence": mutation.DispositionEvidence, + "disposition_at": mutation.DispositionAt, + }), + SafeNextStep: "No action required. Inspect the recorded disposition evidence if you need to know what was dropped from cloud sync.", + RequiresConfirmation: false, + } } func (c SQLiteLockContentionCheck) Run(ctx context.Context, scope Scope) (CheckResult, error) { diff --git a/internal/diagnostic/diagnostic_test.go b/internal/diagnostic/diagnostic_test.go index 04dd79a30..6bae44d27 100644 --- a/internal/diagnostic/diagnostic_test.go +++ b/internal/diagnostic/diagnostic_test.go @@ -3,6 +3,7 @@ package diagnostic import ( "context" "database/sql" + "encoding/json" "errors" "path/filepath" "strings" @@ -11,6 +12,7 @@ import ( "github.com/Gentleman-Programming/engram/internal/cloud/constants" "github.com/Gentleman-Programming/engram/internal/store" + _ "modernc.org/sqlite" ) func newDiagnosticTestStore(t *testing.T) *store.Store { @@ -34,6 +36,21 @@ func newDiagnosticTestStoreWithConfig(t *testing.T) (*store.Store, store.Config) return s, cfg } +func seedDiagnosticPendingMutation(t *testing.T, dataDir, project, entity, entityKey, op, payload string) { + t.Helper() + db, err := sql.Open("sqlite", filepath.Join(dataDir, "engram.db")) + if err != nil { + t.Fatalf("sql.Open: %v", err) + } + defer db.Close() + if _, err := db.Exec( + `INSERT INTO sync_mutations (target_key, entity, entity_key, op, payload, source, project) VALUES (?, ?, ?, ?, ?, ?, ?)`, + store.DefaultSyncTargetKey, entity, entityKey, op, payload, store.SyncSourceLocal, project, + ); err != nil { + t.Fatalf("insert sync mutation %q: %v", entityKey, err) + } +} + func TestSQLiteLockContentionBranches(t *testing.T) { s := newDiagnosticTestStore(t) tests := []struct { @@ -261,3 +278,121 @@ func TestRunnerRunAllHealthyEvaluatesEveryMVPCheck(t *testing.T) { } } } + +// TestSyncMutationRequiredFieldsSeparatesQuarantinedEvidenceFromBlockingWork +// exercises the cloud-enrolled case on purpose: `engram` is enrolled so the +// check runs past the cloud-sync gate, proving quarantined rows are reported as +// non-blocking evidence on the very path that still evaluates delivery faults. +func TestSyncMutationRequiredFieldsSeparatesQuarantinedEvidenceFromBlockingWork(t *testing.T) { + s, cfg := newDiagnosticTestStoreWithConfig(t) + dataDir := cfg.DataDir + if err := s.EnrollProject("engram"); err != nil { + t.Fatalf("EnrollProject: %v", err) + } + seedDiagnosticPendingMutation(t, dataDir, "engram", store.SyncEntitySession, "poison", store.SyncOpUpsert, `{"id":"poison"}`) + + runCheck := func(stage string) Report { + t.Helper() + report, err := NewRunner().RunOne(context.Background(), Scope{Store: s, Project: "engram"}, CheckSyncMutationRequiredFields) + if err != nil { + t.Fatalf("RunOne %s: %v", stage, err) + } + return report + } + + if report := runCheck("before quarantine"); report.Status != StatusBlocked { + t.Fatalf("expected blocked report before quarantine, got %+v", report) + } + + quarantine, err := s.QuarantineIrreparableSyncMutations("engram", true) + if err != nil || len(quarantine.Actions) != 1 { + t.Fatalf("quarantine report=%+v err=%v", quarantine, err) + } + + report := runCheck("after quarantine") + if report.Status == StatusBlocked || report.Summary.Blocked != 0 { + t.Fatalf("quarantined mutation still blocks doctor: %+v", report) + } + check := report.Checks[0] + if check.Result == StatusBlocked || check.Severity == SeverityBlocking { + t.Fatalf("quarantined mutation still blocks the check: %+v", check) + } + if len(check.Findings) != 1 { + t.Fatalf("expected the quarantined row to stay visible as evidence, got %+v", check.Findings) + } + finding := check.Findings[0] + if finding.Severity != SeverityInfo || finding.ReasonCode != "sync_mutation_quarantined" || finding.RequiresConfirmation { + t.Fatalf("unexpected quarantined finding: %+v", finding) + } + var evidence map[string]any + if err := json.Unmarshal(finding.Evidence, &evidence); err != nil { + t.Fatalf("finding evidence invalid: %v", err) + } + if evidence["entity_key"] != "poison" || evidence["disposition"] != store.SyncMutationDispositionQuarantined { + t.Fatalf("quarantined evidence lost mutation identity: %v", evidence) + } + if reason, _ := evidence["disposition_reason"].(string); strings.TrimSpace(reason) == "" { + t.Fatalf("quarantined evidence lost the disposition reason: %v", evidence) + } + + seedDiagnosticPendingMutation(t, dataDir, "engram", store.SyncEntityObservation, "obs-missing", store.SyncOpUpsert, `{"sync_id":"obs-missing"}`) + report = runCheck("with new blocking work") + if report.Status != StatusBlocked { + t.Fatalf("quarantined evidence masked genuinely blocking work: %+v", report) + } + check = report.Checks[0] + if len(check.Findings) != 2 { + t.Fatalf("expected blocking and quarantined findings, got %+v", check.Findings) + } + if check.Findings[0].Severity != SeverityBlocking || check.Findings[0].ReasonCode != "sync_mutation_payload_missing_required_fields" { + t.Fatalf("blocking finding must lead the roll-up: %+v", check.Findings[0]) + } + if check.ReasonCode != "sync_mutation_payload_missing_required_fields" { + t.Fatalf("check reason code should describe the blocking finding, got %q", check.ReasonCode) + } + if check.Findings[1].ReasonCode != "sync_mutation_quarantined" { + t.Fatalf("quarantined evidence dropped: %+v", check.Findings[1]) + } +} + +// TestSyncMutationRequiredFieldsReportsQuarantinedEvidenceWithoutCloudEnrollment +// pins the seam between the quarantine reporting and the cloud-sync gate: the +// early return taken by a local-only install must still carry the quarantined +// evidence, because quarantine is a local disposition that has nothing to do +// with whether the operator opted into cloud sync. +func TestSyncMutationRequiredFieldsReportsQuarantinedEvidenceWithoutCloudEnrollment(t *testing.T) { + s, cfg := newDiagnosticTestStoreWithConfig(t) + seedDiagnosticPendingMutation(t, cfg.DataDir, "engram", store.SyncEntitySession, "poison", store.SyncOpUpsert, `{"id":"poison"}`) + + quarantine, err := s.QuarantineIrreparableSyncMutations("engram", true) + if err != nil || len(quarantine.Actions) != 1 { + t.Fatalf("quarantine report=%+v err=%v", quarantine, err) + } + + report, err := NewRunner().RunOne(context.Background(), Scope{Store: s, Project: "engram"}, CheckSyncMutationRequiredFields) + if err != nil { + t.Fatalf("RunOne: %v", err) + } + if report.Status == StatusBlocked { + t.Fatalf("local-only install must not be blocked by a quarantined row: %+v", report) + } + check := report.Checks[0] + if len(check.Findings) != 1 || check.Findings[0].ReasonCode != "sync_mutation_quarantined" { + t.Fatalf("cloud sync gate swallowed the quarantined evidence: %+v", check.Findings) + } + if check.Findings[0].Severity != SeverityInfo || check.Findings[0].RequiresConfirmation { + t.Fatalf("quarantined finding must stay non-blocking: %+v", check.Findings[0]) + } + var evidence map[string]any + if err := json.Unmarshal(check.Findings[0].Evidence, &evidence); err != nil { + t.Fatalf("finding evidence invalid: %v", err) + } + if evidence["entity_key"] != "poison" || evidence["disposition"] != store.SyncMutationDispositionQuarantined { + t.Fatalf("quarantined evidence lost mutation identity: %v", evidence) + } + // The local-only gate must not answer a quarantined row with cloud + // enrollment guidance: there is nothing to enroll for. + if strings.Contains(check.Findings[0].SafeNextStep, "engram cloud enroll") { + t.Fatalf("local-only quarantine must not suggest cloud enrollment: %+v", check.Findings[0]) + } +} diff --git a/internal/store/diagnostic.go b/internal/store/diagnostic.go index 9392485e9..8d69fcd76 100644 --- a/internal/store/diagnostic.go +++ b/internal/store/diagnostic.go @@ -104,7 +104,7 @@ type rowQuerier interface { func (s *Store) listPendingProjectMutationsTxLike(q rowQuerier, project string) ([]SyncMutation, error) { query := ` - SELECT seq, target_key, entity, entity_key, op, payload, source, project, occurred_at, acked_at + SELECT seq, target_key, entity, entity_key, op, payload, source, project, occurred_at, acked_at, disposition, ifnull(disposition_reason, ''), ifnull(disposition_evidence, ''), disposition_at FROM sync_mutations WHERE target_key = ? AND acked_at IS NULL` args := []any{DefaultSyncTargetKey} @@ -121,7 +121,7 @@ func (s *Store) listPendingProjectMutationsTxLike(q rowQuerier, project string) mutations := make([]SyncMutation, 0) for rows.Next() { var m SyncMutation - if err := rows.Scan(&m.Seq, &m.TargetKey, &m.Entity, &m.EntityKey, &m.Op, &m.Payload, &m.Source, &m.Project, &m.OccurredAt, &m.AckedAt); err != nil { + if err := rows.Scan(&m.Seq, &m.TargetKey, &m.Entity, &m.EntityKey, &m.Op, &m.Payload, &m.Source, &m.Project, &m.OccurredAt, &m.AckedAt, &m.Disposition, &m.DispositionReason, &m.DispositionEvidence, &m.DispositionAt); err != nil { return nil, err } mutations = append(mutations, m) diff --git a/internal/store/store.go b/internal/store/store.go index a7b89efcc..0382256e8 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -286,16 +286,44 @@ type SyncState struct { } type SyncMutation struct { - Seq int64 `json:"seq"` - TargetKey string `json:"target_key"` - Entity string `json:"entity"` - EntityKey string `json:"entity_key"` - Op string `json:"op"` - Payload string `json:"payload"` - Source string `json:"source"` - Project string `json:"project"` - OccurredAt string `json:"occurred_at"` - AckedAt *string `json:"acked_at,omitempty"` + Seq int64 `json:"seq"` + TargetKey string `json:"target_key"` + Entity string `json:"entity"` + EntityKey string `json:"entity_key"` + Op string `json:"op"` + Payload string `json:"payload"` + Source string `json:"source"` + Project string `json:"project"` + OccurredAt string `json:"occurred_at"` + AckedAt *string `json:"acked_at,omitempty"` + Disposition string `json:"disposition"` + DispositionReason string `json:"disposition_reason,omitempty"` + DispositionEvidence string `json:"disposition_evidence,omitempty"` + DispositionAt *string `json:"disposition_at,omitempty"` +} + +const ( + SyncMutationDispositionPending = "pending" + SyncMutationDispositionQuarantined = "quarantined" +) + +// SyncMutationQuarantineAction records one deterministic local quarantine. +type SyncMutationQuarantineAction struct { + Seq int64 `json:"seq"` + Project string `json:"project"` + Entity string `json:"entity"` + EntityKey string `json:"entity_key"` + Op string `json:"op"` + ReasonCode string `json:"reason_code"` + Message string `json:"message"` + Evidence string `json:"evidence"` +} + +// SyncMutationQuarantineReport is the explicit local recovery result. +type SyncMutationQuarantineReport struct { + Project string `json:"project,omitempty"` + Applied bool `json:"applied"` + Actions []SyncMutationQuarantineAction `json:"actions"` } type PendingSyncMutationProjectCount struct { @@ -836,6 +864,10 @@ func (s *Store) migrate() error { source TEXT NOT NULL DEFAULT 'local', occurred_at TEXT NOT NULL DEFAULT (datetime('now')), acked_at TEXT, + disposition TEXT NOT NULL DEFAULT 'pending', + disposition_reason TEXT, + disposition_evidence TEXT, + disposition_at TEXT, FOREIGN KEY (target_key) REFERENCES sync_state(target_key) ); @@ -905,6 +937,19 @@ func (s *Store) migrate() error { if err := s.addColumnIfNotExists("sync_mutations", "project", "TEXT NOT NULL DEFAULT ''"); err != nil { return err } + for _, c := range []struct{ name, definition string }{ + {"disposition", "TEXT NOT NULL DEFAULT 'pending'"}, + {"disposition_reason", "TEXT"}, + {"disposition_evidence", "TEXT"}, + {"disposition_at", "TEXT"}, + } { + if err := s.addColumnIfNotExists("sync_mutations", c.name, c.definition); err != nil { + return err + } + } + if _, err := s.execHook(s.db, `UPDATE sync_mutations SET disposition = 'pending' WHERE disposition IS NULL OR disposition = ''`); err != nil { + return err + } if err := s.addColumnIfNotExists("sync_state", "reason_code", "TEXT"); err != nil { return err } @@ -1038,7 +1083,8 @@ func (s *Store) migrate() error { } if _, err := s.execHook(s.db, ` CREATE INDEX IF NOT EXISTS idx_cloud_upgrade_state_stage ON cloud_upgrade_state(stage); - CREATE INDEX IF NOT EXISTS idx_sync_mutations_lookup ON sync_mutations(target_key, entity, entity_key, source); + CREATE INDEX IF NOT EXISTS idx_sync_mutations_lookup ON sync_mutations(target_key, entity, entity_key, source); + CREATE INDEX IF NOT EXISTS idx_sync_mutations_transport ON sync_mutations(target_key, disposition, acked_at, seq); `); err != nil { return err } @@ -1555,13 +1601,17 @@ func (s *Store) withReadTx(fn func(tx *sql.Tx) ([]cloudUpgradeLegacyMutationEval return fn(tx) } +// listPendingProjectMutationsTx returns the transportable pending journal rows a +// cloud upgrade still has to account for. Quarantined rows are excluded: they are +// already dispositioned local evidence, so counting them would keep the upgrade +// blocked forever with no remaining action an operator could take. func (s *Store) listPendingProjectMutationsTx(tx *sql.Tx, project string) ([]SyncMutation, error) { rows, err := s.queryItHook(tx, ` SELECT seq, target_key, entity, entity_key, op, payload, source, project, occurred_at, acked_at FROM sync_mutations - WHERE target_key = ? AND project = ? AND acked_at IS NULL + WHERE target_key = ? AND project = ? AND acked_at IS NULL AND disposition = ? ORDER BY seq ASC - `, DefaultSyncTargetKey, project) + `, DefaultSyncTargetKey, project, SyncMutationDispositionPending) if err != nil { return nil, err } @@ -3994,10 +4044,10 @@ func (s *Store) ListPendingSyncMutations(targetKey string, limit int) ([]SyncMut // Only return mutations for enrolled projects or empty-project (global) mutations. // Empty-project mutations always sync regardless of enrollment. rows, err := s.queryItHook(s.db, ` - SELECT sm.seq, sm.target_key, sm.entity, sm.entity_key, sm.op, sm.payload, sm.source, sm.project, sm.occurred_at, sm.acked_at + SELECT sm.seq, sm.target_key, sm.entity, sm.entity_key, sm.op, sm.payload, sm.source, sm.project, sm.occurred_at, sm.acked_at, sm.disposition, ifnull(sm.disposition_reason, ''), ifnull(sm.disposition_evidence, ''), sm.disposition_at FROM sync_mutations sm LEFT JOIN sync_enrolled_projects sep ON sm.project = sep.project - WHERE sm.target_key = ? AND sm.acked_at IS NULL + WHERE sm.target_key = ? AND sm.acked_at IS NULL AND sm.disposition = 'pending' AND (sm.project = '' OR sep.project IS NOT NULL) ORDER BY sm.seq ASC LIMIT ?`, targetKey, limit) @@ -4009,7 +4059,7 @@ func (s *Store) ListPendingSyncMutations(targetKey string, limit int) ([]SyncMut var mutations []SyncMutation for rows.Next() { var mutation SyncMutation - if err := rows.Scan(&mutation.Seq, &mutation.TargetKey, &mutation.Entity, &mutation.EntityKey, &mutation.Op, &mutation.Payload, &mutation.Source, &mutation.Project, &mutation.OccurredAt, &mutation.AckedAt); err != nil { + if err := rows.Scan(&mutation.Seq, &mutation.TargetKey, &mutation.Entity, &mutation.EntityKey, &mutation.Op, &mutation.Payload, &mutation.Source, &mutation.Project, &mutation.OccurredAt, &mutation.AckedAt, &mutation.Disposition, &mutation.DispositionReason, &mutation.DispositionEvidence, &mutation.DispositionAt); err != nil { return nil, err } mutations = append(mutations, mutation) @@ -4023,10 +4073,10 @@ func (s *Store) ListPendingSyncMutationsAfterSeq(targetKey string, afterSeq int6 limit = 100 } rows, err := s.queryItHook(s.db, ` - SELECT sm.seq, sm.target_key, sm.entity, sm.entity_key, sm.op, sm.payload, sm.source, sm.project, sm.occurred_at, sm.acked_at + SELECT sm.seq, sm.target_key, sm.entity, sm.entity_key, sm.op, sm.payload, sm.source, sm.project, sm.occurred_at, sm.acked_at, sm.disposition, ifnull(sm.disposition_reason, ''), ifnull(sm.disposition_evidence, ''), sm.disposition_at FROM sync_mutations sm LEFT JOIN sync_enrolled_projects sep ON sm.project = sep.project - WHERE sm.target_key = ? AND sm.acked_at IS NULL + WHERE sm.target_key = ? AND sm.acked_at IS NULL AND sm.disposition = 'pending' AND sm.seq > ? AND (sm.project = '' OR sep.project IS NOT NULL) ORDER BY sm.seq ASC @@ -4039,7 +4089,7 @@ func (s *Store) ListPendingSyncMutationsAfterSeq(targetKey string, afterSeq int6 mutations := make([]SyncMutation, 0, limit) for rows.Next() { var mutation SyncMutation - if err := rows.Scan(&mutation.Seq, &mutation.TargetKey, &mutation.Entity, &mutation.EntityKey, &mutation.Op, &mutation.Payload, &mutation.Source, &mutation.Project, &mutation.OccurredAt, &mutation.AckedAt); err != nil { + if err := rows.Scan(&mutation.Seq, &mutation.TargetKey, &mutation.Entity, &mutation.EntityKey, &mutation.Op, &mutation.Payload, &mutation.Source, &mutation.Project, &mutation.OccurredAt, &mutation.AckedAt, &mutation.Disposition, &mutation.DispositionReason, &mutation.DispositionEvidence, &mutation.DispositionAt); err != nil { return nil, err } mutations = append(mutations, mutation) @@ -4047,6 +4097,92 @@ func (s *Store) ListPendingSyncMutationsAfterSeq(targetKey string, afterSeq int6 return mutations, rows.Err() } +// QuarantineIrreparableSyncMutations explicitly disposes of pending mutations +// the existing legacy validator proves cannot be repaired from local state. +// It never acknowledges, rewrites, or deletes a mutation. +func (s *Store) QuarantineIrreparableSyncMutations(project string, apply bool) (SyncMutationQuarantineReport, error) { + project, _ = NormalizeProject(project) + project = strings.TrimSpace(project) + report := SyncMutationQuarantineReport{Project: project, Applied: apply, Actions: []SyncMutationQuarantineAction{}} + err := s.withTx(func(tx *sql.Tx) error { + affectedProjects := map[string]struct{}{} + quarantinedAny := false + query := `SELECT seq, target_key, entity, entity_key, op, payload, source, project, occurred_at, acked_at + FROM sync_mutations WHERE target_key = ? AND acked_at IS NULL AND disposition = 'pending'` + args := []any{DefaultSyncTargetKey} + if project != "" { + query += ` AND project = ?` + args = append(args, project) + } + query += ` ORDER BY seq ASC` + rows, err := s.queryItHook(tx, query, args...) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var mutation SyncMutation + if err := rows.Scan(&mutation.Seq, &mutation.TargetKey, &mutation.Entity, &mutation.EntityKey, &mutation.Op, &mutation.Payload, &mutation.Source, &mutation.Project, &mutation.OccurredAt, &mutation.AckedAt); err != nil { + return err + } + evaluation, err := s.evaluateCloudUpgradeLegacyMutationTx(tx, mutation) + if err != nil { + return err + } + if !evaluation.hasIssue || evaluation.canRepair { + continue + } + evidence, err := json.Marshal(map[string]any{"check": "sync_mutation_required_fields", "finding": evaluation.finding}) + if err != nil { + return err + } + action := SyncMutationQuarantineAction{Seq: mutation.Seq, Project: mutation.Project, Entity: mutation.Entity, EntityKey: mutation.EntityKey, Op: mutation.Op, ReasonCode: evaluation.finding.ReasonCode, Message: evaluation.finding.Message, Evidence: string(evidence)} + report.Actions = append(report.Actions, action) + if !apply { + continue + } + result, err := s.execHook(tx, `UPDATE sync_mutations SET disposition = 'quarantined', disposition_reason = ?, disposition_evidence = ?, disposition_at = datetime('now') WHERE target_key = ? AND seq = ? AND acked_at IS NULL AND disposition = 'pending'`, action.ReasonCode, action.Evidence, DefaultSyncTargetKey, action.Seq) + if err != nil { + return err + } + updated, err := result.RowsAffected() + if err != nil { + return err + } + if updated == 0 { + continue + } + quarantinedAny = true + mutation.Project, _ = NormalizeProject(mutation.Project) + if mutation.Project = strings.TrimSpace(mutation.Project); mutation.Project != "" { + affectedProjects[mutation.Project] = struct{}{} + } + } + if err := rows.Err(); err != nil { + return err + } + if err := rows.Close(); err != nil { + return err + } + if !apply || !quarantinedAny { + return nil + } + if err := s.refreshSyncLifecycleTx(tx, DefaultSyncTargetKey); err != nil { + return err + } + for affectedProject := range affectedProjects { + if err := s.refreshProjectSyncLifecycleTx(tx, affectedProject); err != nil { + return err + } + } + return nil + }) + if err != nil { + return SyncMutationQuarantineReport{}, fmt.Errorf("quarantine irreparable sync mutations: %w", err) + } + return report, nil +} + func (s *Store) CountPendingNonEnrolledSyncMutations(targetKey string) ([]PendingSyncMutationProjectCount, error) { targetKey = normalizeSyncTargetKey(targetKey) rows, err := s.queryItHook(s.db, ` @@ -4055,6 +4191,7 @@ func (s *Store) CountPendingNonEnrolledSyncMutations(targetKey string) ([]Pendin LEFT JOIN sync_enrolled_projects sep ON sm.project = sep.project WHERE sm.target_key = ? AND sm.acked_at IS NULL + AND sm.disposition = 'pending' AND sm.project != '' AND sep.project IS NULL GROUP BY sm.project @@ -4085,6 +4222,7 @@ func (s *Store) SkipAckNonEnrolledMutations(targetKey string) (int64, error) { SET acked_at = datetime('now') WHERE target_key = ? AND acked_at IS NULL + AND disposition = 'pending' AND project != '' AND project NOT IN (SELECT project FROM sync_enrolled_projects)`, targetKey, @@ -4105,7 +4243,7 @@ func (s *Store) AckSyncMutations(targetKey string, lastAckedSeq int64) error { if targetKey == DefaultSyncTargetKey { rows, err := s.queryItHook(tx, `SELECT DISTINCT ifnull(project, '') FROM sync_mutations - WHERE target_key = ? AND seq <= ? AND acked_at IS NULL`, + WHERE target_key = ? AND seq <= ? AND acked_at IS NULL AND disposition = 'pending'`, targetKey, lastAckedSeq, ) if err != nil { @@ -4135,7 +4273,7 @@ func (s *Store) AckSyncMutations(targetKey string, lastAckedSeq int64) error { return err } if _, err := s.execHook(tx, - `UPDATE sync_mutations SET acked_at = datetime('now') WHERE target_key = ? AND seq <= ? AND acked_at IS NULL`, + `UPDATE sync_mutations SET acked_at = datetime('now') WHERE target_key = ? AND seq <= ? AND acked_at IS NULL AND disposition = 'pending'`, targetKey, lastAckedSeq, ); err != nil { return err @@ -4224,7 +4362,7 @@ func (s *Store) AckSyncMutationSeqs(targetKey string, seqs []int64) error { continue } if _, err := s.execHook(tx, - `UPDATE sync_mutations SET acked_at = datetime('now') WHERE target_key = ? AND seq = ? AND acked_at IS NULL`, + `UPDATE sync_mutations SET acked_at = datetime('now') WHERE target_key = ? AND seq = ? AND acked_at IS NULL AND disposition = 'pending'`, targetKey, seq, ); err != nil { return err @@ -4234,7 +4372,7 @@ func (s *Store) AckSyncMutationSeqs(targetKey string, seqs []int64) error { } } var remaining int - if err := tx.QueryRow(`SELECT COUNT(*) FROM sync_mutations WHERE target_key = ? AND acked_at IS NULL`, targetKey).Scan(&remaining); err != nil { + if err := tx.QueryRow(`SELECT COUNT(*) FROM sync_mutations WHERE target_key = ? AND acked_at IS NULL AND disposition = 'pending'`, targetKey).Scan(&remaining); err != nil { return err } lifecycle := SyncLifecyclePending @@ -4279,7 +4417,7 @@ func (s *Store) HasPendingSyncMutationsForProject(project string) (bool, error) var count int err := s.db.QueryRow( - `SELECT COUNT(*) FROM sync_mutations WHERE target_key = ? AND project = ? AND acked_at IS NULL`, + `SELECT COUNT(*) FROM sync_mutations WHERE target_key = ? AND project = ? AND acked_at IS NULL AND disposition = 'pending'`, DefaultSyncTargetKey, project, ).Scan(&count) @@ -4333,7 +4471,7 @@ func (s *Store) refreshProjectSyncStateTx(tx *sql.Tx, project string) error { if err := tx.QueryRow( `SELECT COUNT(*) FROM sync_mutations - WHERE target_key = ? AND project = ? AND acked_at IS NULL`, + WHERE target_key = ? AND project = ? AND acked_at IS NULL AND disposition = 'pending'`, DefaultSyncTargetKey, project, ).Scan(&pendingCount); err != nil { @@ -4366,6 +4504,59 @@ func (s *Store) refreshProjectSyncStateTx(tx *sql.Tx, project string) error { return err } +// refreshSyncLifecycleTx derives a target lifecycle from its remaining transportable mutations. +func (s *Store) refreshSyncLifecycleTx(tx *sql.Tx, targetKey string) error { + targetKey = normalizeSyncTargetKey(targetKey) + var pendingCount int + if err := tx.QueryRow( + `SELECT COUNT(*) FROM sync_mutations WHERE target_key = ? AND acked_at IS NULL AND disposition = ?`, + targetKey, SyncMutationDispositionPending, + ).Scan(&pendingCount); err != nil { + return err + } + return s.applySyncLifecycleTx(tx, targetKey, pendingCount) +} + +// refreshProjectSyncLifecycleTx derives the `cloud:` lifecycle from the +// journal rows the local writer actually produces. enqueueSyncMutationTx always +// stores mutations under the default `cloud` target key and keeps the project in +// its own column, so counting rows keyed by `cloud:` would always return +// zero and mark the project healthy while real pending work remains. +func (s *Store) refreshProjectSyncLifecycleTx(tx *sql.Tx, project string) error { + project, _ = NormalizeProject(project) + project = strings.TrimSpace(project) + if project == "" { + return nil + } + var pendingCount int + if err := tx.QueryRow( + `SELECT COUNT(*) FROM sync_mutations WHERE target_key = ? AND project = ? AND acked_at IS NULL AND disposition = ?`, + DefaultSyncTargetKey, project, SyncMutationDispositionPending, + ).Scan(&pendingCount); err != nil { + return err + } + return s.applySyncLifecycleTx(tx, syncTargetKeyForProject(project), pendingCount) +} + +func (s *Store) applySyncLifecycleTx(tx *sql.Tx, targetKey string, pendingCount int) error { + state, err := s.getSyncStateTx(tx, targetKey) + if err != nil { + return err + } + lifecycle := SyncLifecycleHealthy + if pendingCount > 0 { + lifecycle = SyncLifecyclePending + } + if isActivelyDegradedState(state, time.Now().UTC()) { + lifecycle = SyncLifecycleDegraded + } + if lifecycle == state.Lifecycle { + return nil + } + _, err = s.execHook(tx, `UPDATE sync_state SET lifecycle = ?, updated_at = datetime('now') WHERE target_key = ?`, lifecycle, targetKey) + return err +} + func isActivelyDegradedState(state *SyncState, now time.Time) bool { if state == nil || state.Lifecycle != SyncLifecycleDegraded { return false diff --git a/internal/store/store_migration_test.go b/internal/store/store_migration_test.go index a73d5eb7a..0df08262e 100644 --- a/internal/store/store_migration_test.go +++ b/internal/store/store_migration_test.go @@ -428,7 +428,7 @@ func TestMigrate_DoesNotTouchFTS5OrSyncMutations(t *testing.T) { t.Fatalf("smRows.Err: %v", err) } - requiredSMCols := []string{"seq", "target_key", "entity", "entity_key", "op", "payload", "source", "project", "occurred_at", "acked_at"} + requiredSMCols := []string{"seq", "target_key", "entity", "entity_key", "op", "payload", "source", "project", "occurred_at", "acked_at", "disposition", "disposition_reason", "disposition_evidence", "disposition_at"} colSet := make(map[string]bool, len(smCols)) for _, c := range smCols { colSet[c] = true diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 50cc1d42c..fbe085da3 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -8287,6 +8287,287 @@ func TestDeleteSession_EnrolledProjectEnqueuesSyncDeleteMutation(t *testing.T) { } } +func TestQuarantineIrreparableSyncMutationsPreservesJournalAndUnblocksTransport(t *testing.T) { + s := newTestStore(t) + if err := s.CreateSession("repairable", "project", "/tmp/repairable"); err != nil { + t.Fatalf("create repairable session: %v", err) + } + for _, mutation := range []struct { + entity, key, op, payload, project string + }{ + {SyncEntitySession, "poison", SyncOpUpsert, `{"id":"poison"}`, ""}, + {SyncEntitySession, "later", SyncOpDelete, `{"id":"later"}`, ""}, + {SyncEntitySession, "repairable", SyncOpUpsert, `{"id":"repairable"}`, "project"}, + } { + if _, err := s.db.Exec(`INSERT INTO sync_mutations (target_key, entity, entity_key, op, payload, source, project) VALUES (?, ?, ?, ?, ?, ?, ?)`, DefaultSyncTargetKey, mutation.entity, mutation.key, mutation.op, mutation.payload, SyncSourceLocal, mutation.project); err != nil { + t.Fatalf("seed mutation %s: %v", mutation.key, err) + } + } + var laterSeq int64 + if err := s.db.QueryRow(`SELECT seq FROM sync_mutations WHERE entity_key = 'later'`).Scan(&laterSeq); err != nil { + t.Fatalf("read later sequence: %v", err) + } + + dryRun, err := s.QuarantineIrreparableSyncMutations("", false) + if err != nil || len(dryRun.Actions) != 1 { + t.Fatalf("dry-run report=%+v err=%v", dryRun, err) + } + var disposition string + if err := s.db.QueryRow(`SELECT disposition FROM sync_mutations WHERE entity_key = 'poison'`).Scan(&disposition); err != nil || disposition != SyncMutationDispositionPending { + t.Fatalf("dry-run disposition=%q err=%v", disposition, err) + } + + report, err := s.QuarantineIrreparableSyncMutations("", true) + if err != nil || len(report.Actions) != 1 { + t.Fatalf("apply report=%+v err=%v", report, err) + } + var payload, reason, evidence string + var ackedAt, dispositionAt sql.NullString + if err := s.db.QueryRow(`SELECT payload, disposition, disposition_reason, disposition_evidence, disposition_at, acked_at FROM sync_mutations WHERE entity_key = 'poison'`).Scan(&payload, &disposition, &reason, &evidence, &dispositionAt, &ackedAt); err != nil { + t.Fatalf("read quarantined mutation: %v", err) + } + if payload != `{"id":"poison"}` || disposition != SyncMutationDispositionQuarantined || reason == "" || evidence == "" || !dispositionAt.Valid || ackedAt.Valid { + t.Fatalf("quarantine did not preserve audit state: payload=%q disposition=%q reason=%q evidence=%q at=%v acked=%v", payload, disposition, reason, evidence, dispositionAt, ackedAt) + } + pending, err := s.ListPendingSyncMutations(DefaultSyncTargetKey, 10) + if err != nil || len(pending) != 1 || pending[0].EntityKey != "later" || pending[0].Seq != laterSeq { + t.Fatalf("transport pending=%+v err=%v", pending, err) + } + state, err := s.GetSyncState(DefaultSyncTargetKey) + if err != nil || state.LastAckedSeq != 0 { + t.Fatalf("state=%+v err=%v", state, err) + } + again, err := s.QuarantineIrreparableSyncMutations("", true) + if err != nil || len(again.Actions) != 0 { + t.Fatalf("repeat report=%+v err=%v", again, err) + } + var repeatedEvidence string + if err := s.db.QueryRow(`SELECT disposition_evidence FROM sync_mutations WHERE entity_key = 'poison'`).Scan(&repeatedEvidence); err != nil || repeatedEvidence != evidence { + t.Fatalf("repeat changed evidence=%q err=%v", repeatedEvidence, err) + } +} + +func TestQuarantineIrreparableSyncMutationsRefreshesAffectedLifecycles(t *testing.T) { + t.Run("clears stale default and project lifecycle", func(t *testing.T) { + s := newTestStore(t) + const payload = `{"id":"poison"}` + if _, err := s.db.Exec(`INSERT INTO sync_mutations (target_key, entity, entity_key, op, payload, source, project) VALUES ('cloud', 'session', 'poison', 'upsert', ?, 'local', 'project-a')`, payload); err != nil { + t.Fatalf("seed poison mutation: %v", err) + } + var seq int64 + if err := s.db.QueryRow(`SELECT seq FROM sync_mutations WHERE entity_key = 'poison'`).Scan(&seq); err != nil { + t.Fatalf("read poison sequence: %v", err) + } + if err := s.MarkSyncPending(DefaultSyncTargetKey); err != nil { + t.Fatalf("mark default pending: %v", err) + } + if err := s.MarkSyncPending(syncTargetKeyForProject("project-a")); err != nil { + t.Fatalf("mark project pending: %v", err) + } + + report, err := s.QuarantineIrreparableSyncMutations("project-a", true) + if err != nil || len(report.Actions) != 1 { + t.Fatalf("apply report=%+v err=%v", report, err) + } + var gotSeq int64 + var gotPayload, evidence string + var ackedAt sql.NullString + if err := s.db.QueryRow(`SELECT seq, payload, disposition_evidence, acked_at FROM sync_mutations WHERE entity_key = 'poison'`).Scan(&gotSeq, &gotPayload, &evidence, &ackedAt); err != nil { + t.Fatalf("read quarantined mutation: %v", err) + } + if gotSeq != seq || gotPayload != payload || evidence == "" || ackedAt.Valid { + t.Fatalf("quarantine changed mutation audit data: seq=%d payload=%q evidence=%q acked=%v", gotSeq, gotPayload, evidence, ackedAt) + } + for _, targetKey := range []string{DefaultSyncTargetKey, syncTargetKeyForProject("project-a")} { + state, err := s.GetSyncState(targetKey) + if err != nil || state.Lifecycle != SyncLifecycleHealthy || state.LastAckedSeq != 0 { + t.Fatalf("state for %q = %+v, err=%v", targetKey, state, err) + } + } + + again, err := s.QuarantineIrreparableSyncMutations("project-a", true) + if err != nil || len(again.Actions) != 0 { + t.Fatalf("repeat report=%+v err=%v", again, err) + } + var repeatedEvidence string + if err := s.db.QueryRow(`SELECT disposition_evidence FROM sync_mutations WHERE entity_key = 'poison'`).Scan(&repeatedEvidence); err != nil || repeatedEvidence != evidence { + t.Fatalf("repeat changed evidence=%q err=%v", repeatedEvidence, err) + } + }) + + t.Run("preserves pending lifecycle and refreshes only quarantined project", func(t *testing.T) { + s := newTestStore(t) + for _, mutation := range []struct{ key, project, payload string }{ + {key: "poison", project: "project-a", payload: `{"id":"poison"}`}, + {key: "pending", project: "project-b", payload: `{"id":"pending","project":"project-b"}`}, + } { + if _, err := s.db.Exec(`INSERT INTO sync_mutations (target_key, entity, entity_key, op, payload, source, project) VALUES ('cloud', 'session', ?, 'upsert', ?, 'local', ?)`, mutation.key, mutation.payload, mutation.project); err != nil { + t.Fatalf("seed %s mutation: %v", mutation.key, err) + } + } + for _, targetKey := range []string{DefaultSyncTargetKey, syncTargetKeyForProject("project-a"), syncTargetKeyForProject("project-b")} { + if err := s.MarkSyncPending(targetKey); err != nil { + t.Fatalf("mark %q pending: %v", targetKey, err) + } + } + + if _, err := s.QuarantineIrreparableSyncMutations("project-a", true); err != nil { + t.Fatalf("quarantine project-a: %v", err) + } + for _, targetKey := range []string{DefaultSyncTargetKey, syncTargetKeyForProject("project-b")} { + state, err := s.GetSyncState(targetKey) + if err != nil || state.Lifecycle != SyncLifecyclePending { + t.Fatalf("state for %q = %+v, err=%v", targetKey, state, err) + } + } + state, err := s.GetSyncState(syncTargetKeyForProject("project-a")) + if err != nil || state.Lifecycle != SyncLifecycleHealthy { + t.Fatalf("affected project state=%+v err=%v", state, err) + } + }) +} + +func TestQuarantineIrreparableSyncMutationsKeepsProjectPendingWhenWorkRemains(t *testing.T) { + s := newTestStore(t) + if err := s.CreateSession("keep", "project-a", "/work/project-a"); err != nil { + t.Fatalf("create session: %v", err) + } + for _, mutation := range []struct{ key, payload string }{ + {key: "poison", payload: `{"id":"poison"}`}, + {key: "keep", payload: `{"id":"keep","directory":"/work/project-a"}`}, + } { + if _, err := s.db.Exec(`INSERT INTO sync_mutations (target_key, entity, entity_key, op, payload, source, project) VALUES (?, 'session', ?, 'upsert', ?, 'local', 'project-a')`, DefaultSyncTargetKey, mutation.key, mutation.payload); err != nil { + t.Fatalf("seed %s mutation: %v", mutation.key, err) + } + } + for _, targetKey := range []string{DefaultSyncTargetKey, syncTargetKeyForProject("project-a")} { + if err := s.MarkSyncPending(targetKey); err != nil { + t.Fatalf("mark %q pending: %v", targetKey, err) + } + } + + report, err := s.QuarantineIrreparableSyncMutations("project-a", true) + if err != nil || len(report.Actions) != 1 || report.Actions[0].EntityKey != "poison" { + t.Fatalf("apply report=%+v err=%v", report, err) + } + + // The local journal writes every row under the default `cloud` target key and + // carries the project in its own column, so the per-project lifecycle refresh + // must count that key instead of the `cloud:` bookkeeping key. + for _, targetKey := range []string{DefaultSyncTargetKey, syncTargetKeyForProject("project-a")} { + state, err := s.GetSyncState(targetKey) + if err != nil { + t.Fatalf("state for %q: %v", targetKey, err) + } + if state.Lifecycle != SyncLifecyclePending { + t.Fatalf("quarantine masked pending work for %q: lifecycle=%q", targetKey, state.Lifecycle) + } + } + pendingForProject, err := s.HasPendingSyncMutationsForProject("project-a") + if err != nil || !pendingForProject { + t.Fatalf("HasPendingSyncMutationsForProject=%v err=%v", pendingForProject, err) + } + + // Once the transportable work is acked, quarantining a newly poisoned row must + // clear the project lifecycle through that same key. + if _, err := s.db.Exec(`UPDATE sync_mutations SET acked_at = datetime('now') WHERE entity_key = 'keep'`); err != nil { + t.Fatalf("ack keep mutation: %v", err) + } + if _, err := s.db.Exec(`INSERT INTO sync_mutations (target_key, entity, entity_key, op, payload, source, project) VALUES (?, 'session', 'poison-2', 'upsert', '{"id":"poison-2"}', 'local', 'project-a')`, DefaultSyncTargetKey); err != nil { + t.Fatalf("seed second poison mutation: %v", err) + } + second, err := s.QuarantineIrreparableSyncMutations("project-a", true) + if err != nil || len(second.Actions) != 1 || second.Actions[0].EntityKey != "poison-2" { + t.Fatalf("second quarantine report=%+v err=%v", second, err) + } + state, err := s.GetSyncState(syncTargetKeyForProject("project-a")) + if err != nil || state.Lifecycle != SyncLifecycleHealthy { + t.Fatalf("project lifecycle should clear once no transportable work remains: %+v err=%v", state, err) + } +} + +func TestQuarantineIrreparableSyncMutationsClearsCloudUpgradeBlockers(t *testing.T) { + s := newTestStore(t) + if _, err := s.db.Exec(`INSERT INTO sync_mutations (target_key, entity, entity_key, op, payload, source, project) VALUES (?, 'session', 'poison', 'upsert', '{"id":"poison"}', 'local', 'project-a')`, DefaultSyncTargetKey); err != nil { + t.Fatalf("seed poison mutation: %v", err) + } + + before, err := s.DiagnoseCloudUpgradeLegacyMutations("project-a") + if err != nil || before.BlockedCount != 1 { + t.Fatalf("legacy report before quarantine=%+v err=%v", before, err) + } + + if _, err := s.QuarantineIrreparableSyncMutations("project-a", true); err != nil { + t.Fatalf("quarantine: %v", err) + } + + after, err := s.DiagnoseCloudUpgradeLegacyMutations("project-a") + if err != nil { + t.Fatalf("legacy report after quarantine: %v", err) + } + if after.BlockedCount != 0 || after.RepairableCount != 0 || len(after.Findings) != 0 { + t.Fatalf("quarantined mutation still blocks the cloud upgrade: %+v", after) + } + + // A genuinely irreparable row enqueued afterwards must still block. + if _, err := s.db.Exec(`INSERT INTO sync_mutations (target_key, entity, entity_key, op, payload, source, project) VALUES (?, 'session', 'poison-2', 'upsert', '{"id":"poison-2"}', 'local', 'project-a')`, DefaultSyncTargetKey); err != nil { + t.Fatalf("seed second poison mutation: %v", err) + } + residual, err := s.DiagnoseCloudUpgradeLegacyMutations("project-a") + if err != nil || residual.BlockedCount != 1 || len(residual.Findings) != 1 || residual.Findings[0].EntityKey != "poison-2" { + t.Fatalf("new irreparable work must still block: %+v err=%v", residual, err) + } +} + +func TestQuarantineIrreparableSyncMutationsFailsClosed(t *testing.T) { + s := newTestStore(t) + if _, err := s.db.Exec(`INSERT INTO sync_mutations (target_key, entity, entity_key, op, payload, source, project) VALUES ('cloud', 'session', 'poison', 'upsert', '{"id":"poison"}', 'local', '')`); err != nil { + t.Fatalf("seed mutation: %v", err) + } + if _, err := s.db.Exec(`CREATE TRIGGER reject_quarantine BEFORE UPDATE OF disposition ON sync_mutations BEGIN SELECT RAISE(ABORT, 'quarantine blocked'); END`); err != nil { + t.Fatalf("create reject trigger: %v", err) + } + if _, err := s.QuarantineIrreparableSyncMutations("", true); err == nil { + t.Fatal("expected quarantine persistence error") + } + var disposition string + if err := s.db.QueryRow(`SELECT disposition FROM sync_mutations WHERE entity_key = 'poison'`).Scan(&disposition); err != nil || disposition != SyncMutationDispositionPending { + t.Fatalf("failed quarantine disposition=%q err=%v", disposition, err) + } + pending, err := s.ListPendingSyncMutations(DefaultSyncTargetKey, 10) + if err != nil || len(pending) != 1 || pending[0].EntityKey != "poison" { + t.Fatalf("failed quarantine transport pending=%+v err=%v", pending, err) + } +} + +func TestQuarantineIrreparableSyncMutationsRollsBackWhenLifecycleRefreshFails(t *testing.T) { + s := newTestStore(t) + if _, err := s.db.Exec(`INSERT INTO sync_mutations (target_key, entity, entity_key, op, payload, source, project) VALUES ('cloud', 'session', 'poison', 'upsert', '{"id":"poison"}', 'local', 'project-a')`); err != nil { + t.Fatalf("seed mutation: %v", err) + } + if err := s.MarkSyncPending(DefaultSyncTargetKey); err != nil { + t.Fatalf("mark default pending: %v", err) + } + if err := s.MarkSyncPending(syncTargetKeyForProject("project-a")); err != nil { + t.Fatalf("mark project pending: %v", err) + } + if _, err := s.db.Exec(`CREATE TRIGGER reject_lifecycle_refresh BEFORE UPDATE OF lifecycle ON sync_state BEGIN SELECT RAISE(ABORT, 'lifecycle refresh blocked'); END`); err != nil { + t.Fatalf("create lifecycle refresh trigger: %v", err) + } + + if _, err := s.QuarantineIrreparableSyncMutations("project-a", true); err == nil { + t.Fatal("expected lifecycle refresh error") + } + var disposition string + var evidence sql.NullString + if err := s.db.QueryRow(`SELECT disposition, disposition_evidence FROM sync_mutations WHERE entity_key = 'poison'`).Scan(&disposition, &evidence); err != nil { + t.Fatalf("read mutation after rollback: %v", err) + } + if disposition != SyncMutationDispositionPending || evidence.Valid { + t.Fatalf("refresh failure did not roll back quarantine: disposition=%q evidence=%v", disposition, evidence) + } +} + func TestDeleteSession_NotFound(t *testing.T) { s := newTestStore(t)