diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 9f363922a..b7cd45431 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -35,6 +35,7 @@ enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 6, P #include "foundation/compat_thread.h" #include "foundation/profile.h" #include "foundation/mem.h" +#include "yyjson/yyjson.h" #include #include @@ -153,6 +154,83 @@ static const char *itoa_buf(int val) { return bufs[i]; } +const char *cbm_pipeline_mode_name(cbm_index_mode_t mode) { + switch (mode) { + case CBM_MODE_FULL: + return "full"; + case CBM_MODE_MODERATE: + return "moderate"; + case CBM_MODE_FAST: + return "fast"; + default: + return "unknown"; + } +} + +typedef enum { + INDEX_MODE_METADATA_OK = 0, + INDEX_MODE_METADATA_MISSING_OR_INVALID, + INDEX_MODE_METADATA_ERROR, +} index_mode_metadata_status_t; + +static index_mode_metadata_status_t parse_index_mode(const char *properties_json, + size_t properties_len, + cbm_index_mode_t *out_mode) { + if (!properties_json || !out_mode) { + return INDEX_MODE_METADATA_ERROR; + } + + yyjson_read_err read_error = {0}; + yyjson_doc *doc = + yyjson_read_opts((char *)properties_json, properties_len, 0, NULL, &read_error); + if (!doc) { + return read_error.code == YYJSON_READ_ERROR_MEMORY_ALLOCATION + ? INDEX_MODE_METADATA_ERROR + : INDEX_MODE_METADATA_MISSING_OR_INVALID; + } + + yyjson_val *root = yyjson_doc_get_root(doc); + yyjson_val *mode_value = yyjson_is_obj(root) ? yyjson_obj_get(root, "index_mode") : NULL; + index_mode_metadata_status_t status = INDEX_MODE_METADATA_OK; + if (yyjson_equals_str(mode_value, "full")) { + *out_mode = CBM_MODE_FULL; + } else if (yyjson_equals_str(mode_value, "moderate")) { + *out_mode = CBM_MODE_MODERATE; + } else if (yyjson_equals_str(mode_value, "fast")) { + *out_mode = CBM_MODE_FAST; + } else { + status = INDEX_MODE_METADATA_MISSING_OR_INVALID; + } + + yyjson_doc_free(doc); + return status; +} + +static bool index_mode_covers(cbm_index_mode_t stored_mode, cbm_index_mode_t requested_mode) { + return stored_mode >= CBM_MODE_FULL && stored_mode <= CBM_MODE_FAST && + requested_mode >= CBM_MODE_FULL && requested_mode <= CBM_MODE_FAST && + stored_mode <= requested_mode; +} + +static index_mode_metadata_status_t read_stored_index_mode(cbm_store_t *store, const char *project, + cbm_index_mode_t *out_mode) { + char *properties_json = NULL; + size_t properties_len = 0; + int rc = cbm_store_find_node_properties_by_qn(store, project, project, &properties_json, + &properties_len); + if (rc == CBM_STORE_NOT_FOUND) { + return INDEX_MODE_METADATA_MISSING_OR_INVALID; + } + if (rc != CBM_STORE_OK) { + return INDEX_MODE_METADATA_ERROR; + } + + index_mode_metadata_status_t status = + parse_index_mode(properties_json, properties_len, out_mode); + free(properties_json); + return status; +} + /* Log current + peak RSS at a pipeline phase boundary (memory profiling). */ static void log_phase_mem(const char *phase) { enum { PL_BYTES_PER_MB = 1024 * 1024 }; @@ -448,7 +526,11 @@ static int pass_structure(cbm_pipeline_t *p, const cbm_file_info_t *files, int f cbm_log_info("pass.start", "pass", "structure", "files", itoa_buf(file_count)); /* Project node */ - cbm_gbuf_upsert_node(p->gbuf, "Project", p->project_name, p->project_name, NULL, 0, 0, "{}"); + char project_props[CBM_SZ_64]; + snprintf(project_props, sizeof(project_props), "{\"index_mode\":\"%s\"}", + cbm_pipeline_mode_name(p->mode)); + cbm_gbuf_upsert_node(p->gbuf, "Project", p->project_name, p->project_name, NULL, 0, 0, + project_props); const char *branch_qn = p->branch_qn ? p->branch_qn : p->project_name; const char *branch_name = p->git_ctx.branch ? p->git_ctx.branch : "working-tree"; char branch_props[CBM_SZ_2K]; @@ -1116,17 +1198,37 @@ static int try_incremental_or_delete_db(cbm_pipeline_t *p, cbm_file_info_t *file if (check_store && cbm_store_check_integrity(check_store)) { cbm_file_hash_t *hashes = NULL; int hash_count = 0; + cbm_index_mode_t stored_mode = CBM_MODE_FAST; + index_mode_metadata_status_t stored_mode_status = + read_stored_index_mode(check_store, p->project_name, &stored_mode); + if (stored_mode_status == INDEX_MODE_METADATA_ERROR) { + cbm_log_error("pipeline.route", "path", "metadata_read_error", "stored_mode", "error", + "requested_mode", cbm_pipeline_mode_name(p->mode), "action", + "preserve_db"); + cbm_store_close(check_store); + free(db_path); + return CBM_PIPELINE_ABORT_PRESERVE_DB; + } + bool stored_mode_known = stored_mode_status == INDEX_MODE_METADATA_OK; cbm_store_get_file_hashes(check_store, p->project_name, &hashes, &hash_count); cbm_store_free_file_hashes(hashes, hash_count); cbm_store_close(check_store); - if (hash_count > 0 && file_count <= hash_count + (hash_count / PAIR_LEN)) { + bool mode_covered = stored_mode_known && index_mode_covers(stored_mode, p->mode); + if (hash_count > 0 && mode_covered && file_count <= hash_count + (hash_count / PAIR_LEN)) { cbm_log_info("pipeline.route", "path", "incremental", "stored_hashes", - itoa_buf(hash_count)); - int rc = cbm_pipeline_run_incremental(p, db_path, files, file_count); + itoa_buf(hash_count), "stored_mode", cbm_pipeline_mode_name(stored_mode), + "requested_mode", cbm_pipeline_mode_name(p->mode), "effective_mode", + cbm_pipeline_mode_name(stored_mode)); + int rc = cbm_pipeline_run_incremental(p, db_path, files, file_count, stored_mode, + p->persistence); free(db_path); return rc; } - if (hash_count > 0) { + if (hash_count > 0 && !mode_covered) { + cbm_log_info("pipeline.route", "path", "mode_upgrade_reindex", "stored_mode", + stored_mode_known ? cbm_pipeline_mode_name(stored_mode) : "unknown", + "requested_mode", cbm_pipeline_mode_name(p->mode)); + } else if (hash_count > 0) { cbm_log_info("pipeline.route", "path", "mode_change_reindex", "stored_hashes", itoa_buf(hash_count), "discovered", itoa_buf(file_count)); } @@ -1173,19 +1275,6 @@ static int64_t stat_mtime_ns(const struct stat *fst) { #endif } -static const char *pipeline_mode_name(cbm_index_mode_t mode) { - switch (mode) { - case CBM_MODE_FULL: - return "full"; - case CBM_MODE_MODERATE: - return "moderate"; - case CBM_MODE_FAST: - return "fast"; - default: - return "unknown"; - } -} - /* Dump graph to SQLite and persist file hashes for incremental indexing. */ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_info_t *files, int file_count, struct timespec *t) { @@ -1334,7 +1423,7 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_info_t *fil : (p->ignored_total > p->ignored_count ? "truncated" : "complete"); cbm_coverage_meta_t coverage_meta = { .generation = have_project_info ? project_info.indexed_at : NULL, - .index_mode = pipeline_mode_name(p->mode), + .index_mode = cbm_pipeline_mode_name(p->mode), .recording_status = recording_status, .ignored_files_stored = p->ignored_count, .ignored_files_total = p->ignored_total, @@ -1377,16 +1466,22 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_info_t *fil free(p->saved_adr); p->saved_adr = NULL; - /* Export persistent artifact if enabled */ - if (p->persistence) { + /* Export when persistence is explicitly enabled, or refresh an artifact + * created by an earlier run so a full rebuild cannot leave it stale. */ + bool persistence_required = p->persistence; + if (persistence_required || cbm_artifact_exists(p->repo_path)) { CBM_PROF_START(t_art); int arc = cbm_artifact_export(db_path, p->repo_path, p->project_name, CBM_ARTIFACT_BEST); CBM_PROF_END("persist", "6_artifact_export", t_art); if (arc != 0) { const char *err = cbm_artifact_export_last_error(); cbm_log_error("pipeline.err", "phase", "artifact_export", "err", err ? err : "unknown"); - /* A failed persistence export intentionally fails the run; this used to be ignored. */ - return arc; + /* An explicitly requested persistence export is part of the run's + * contract. Refreshing a pre-existing artifact remains best-effort, + * matching the incremental pipeline. */ + if (persistence_required) { + return arc; + } } } diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 8b5766978..0de7b5a6c 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -70,19 +70,6 @@ static int64_t stat_mtime_ns(const struct stat *st) { #endif } -static const char *incr_mode_name(int mode) { - switch (mode) { - case CBM_MODE_FULL: - return "full"; - case CBM_MODE_MODERATE: - return "moderate"; - case CBM_MODE_FAST: - return "fast"; - default: - return "unknown"; - } -} - /* ── File classification ─────────────────────────────────────────── */ /* Classify discovered files against stored hashes using mtime+size. @@ -639,15 +626,106 @@ static void run_postpasses(cbm_pipeline_ctx_t *ctx, cbm_file_info_t *changed_fil itoa_buf((int)elapsed_ms(t))); } } + +static cbm_coverage_row_t *build_coverage_snapshot(cbm_pipeline_t *p, cbm_coverage_row_t *old_cov, + int old_cov_count, + cbm_file_info_t *changed_files, + int changed_count, int *out_count, + cbm_coverage_meta_t *out_meta) { + cbm_file_error_t *run_errs = NULL; + int run_err_count = 0; + cbm_pipeline_get_file_errors(p, &run_errs, &run_err_count); + char **run_excluded = NULL; + int run_excluded_count = 0; + cbm_pipeline_get_excluded(p, &run_excluded, &run_excluded_count); + cbm_ignored_file_t *run_ignored = NULL; + int run_ignored_count = 0; + int run_ignored_total = 0; + cbm_pipeline_get_ignored(p, &run_ignored, &run_ignored_count, &run_ignored_total); + + int cov_cap = old_cov_count + run_err_count + run_excluded_count + run_ignored_count; + cbm_coverage_row_t *cov = NULL; + if (cov_cap > 0) { + cov = (cbm_coverage_row_t *)malloc((size_t)cov_cap * sizeof(*cov)); + } + bool coverage_rows_available = cov_cap == 0 || cov != NULL; + int cov_n = 0; + if (cov) { + CBMHashTable *changed_set = + cbm_ht_create(changed_count > 0 ? (size_t)changed_count * PAIR_LEN : CBM_SZ_64); + for (int i = 0; i < changed_count; i++) { + cbm_ht_set(changed_set, changed_files[i].rel_path, &changed_files[i]); + } + for (int i = 0; i < old_cov_count; i++) { + bool by_design = old_cov[i].kind && strncmp(old_cov[i].kind, "not_indexed", 11) == 0; + if (!by_design && old_cov[i].rel_path && + !cbm_ht_get(changed_set, old_cov[i].rel_path)) { + cov[cov_n++] = old_cov[i]; + } + } + cbm_ht_free(changed_set); + for (int i = 0; i < run_err_count; i++) { + cov[cov_n].rel_path = run_errs[i].path; + cov[cov_n].kind = run_errs[i].phase; + cov[cov_n].detail = run_errs[i].reason; + cov_n++; + } + for (int i = 0; i < run_excluded_count; i++) { + cov[cov_n].rel_path = run_excluded[i]; + cov[cov_n].kind = "not_indexed_dir"; + cov[cov_n].detail = "excluded subtree"; + cov_n++; + } + for (int i = 0; i < run_ignored_count; i++) { + cov[cov_n].rel_path = run_ignored[i].rel_path; + cov[cov_n].kind = "not_indexed_file"; + cov[cov_n].detail = run_ignored[i].reason; + cov_n++; + } + } + + *out_count = cov_n; + *out_meta = (cbm_coverage_meta_t){ + .index_mode = cbm_pipeline_mode_name((cbm_index_mode_t)cbm_pipeline_get_mode(p)), + .recording_status = + !coverage_rows_available + ? "unavailable" + : (run_ignored_total > run_ignored_count ? "truncated" : "complete"), + .ignored_files_stored = run_ignored_count, + .ignored_files_total = run_ignored_total, + .coverage_version = 1, + }; + return cov; +} + +static int export_artifact_if_needed(const char *db_path, const char *repo_path, + const char *project, bool persistence_required, + bool refresh_existing) { + if (!repo_path || + (!persistence_required && !(refresh_existing && cbm_artifact_exists(repo_path)))) { + return 0; + } + + int rc = cbm_artifact_export(db_path, repo_path, project, CBM_ARTIFACT_FAST); + if (rc == 0) { + return 0; + } + + const char *err = cbm_artifact_export_last_error(); + cbm_log_error("incremental.err", "msg", "artifact_export", "err", err ? err : "unknown"); + return persistence_required ? rc : 0; +} + /* Delete old DB and dump merged graph + hashes to disk. * Mode-skipped hash rows are preserved across the rebuild so subsequent * reindexes can correctly distinguish "never indexed" from "indexed but * not visited this pass". */ -static void dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *project, - cbm_file_info_t *files, int file_count, - const cbm_file_hash_t *mode_skipped, int mode_skipped_count, - const char *repo_path, const cbm_coverage_row_t *cov, int cov_count, - const cbm_coverage_meta_t *meta_template) { +static int dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char *project, + cbm_file_info_t *files, int file_count, + const cbm_file_hash_t *mode_skipped, int mode_skipped_count, + const char *repo_path, bool persistence_required, + const cbm_coverage_row_t *cov, int cov_count, + const cbm_coverage_meta_t *meta_template) { struct timespec t; cbm_clock_gettime(CLOCK_MONOTONIC, &t); @@ -701,20 +779,19 @@ static void dump_and_persist(cbm_gbuf_t *gbuf, const char *db_path, const char * cbm_store_close(hash_store); } - /* Auto-update artifact if one already exists (persistence was enabled previously) */ - if (repo_path && cbm_artifact_exists(repo_path)) { - cbm_artifact_export(db_path, repo_path, project, CBM_ARTIFACT_FAST); - } + return export_artifact_if_needed(db_path, repo_path, project, persistence_required, true); } /* ── Incremental pipeline entry point ────────────────────────────── */ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_file_info_t *files, - int file_count) { + int file_count, cbm_index_mode_t effective_mode, + bool persistence_required) { struct timespec t0; cbm_clock_gettime(CLOCK_MONOTONIC, &t0); const char *project = cbm_pipeline_project_name(p); + cbm_index_mode_t requested_mode = (cbm_index_mode_t)cbm_pipeline_get_mode(p); /* Open existing disk DB */ cbm_store_t *store = cbm_store_open_path(db_path); @@ -747,17 +824,62 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil itoa_buf(n_unchanged), "deleted", itoa_buf(deleted_count), "mode_skipped", itoa_buf(mode_skipped_count)); - /* Fast path: nothing changed → skip. The on-disk DB is left untouched, - * which means existing hash rows (including for any mode-skipped files - * that were already preserved by an earlier run) remain intact. */ + /* Fast path: nothing changed → skip graph work. A downgrade still needs + * its requested discovery coverage persisted even though Project metadata + * continues to record the stronger effective capabilities. */ if (n_changed == 0 && deleted_count == 0) { - cbm_log_info("incremental.noop", "reason", "no_changes"); + int noop_rc = 0; + bool coverage_refreshed = false; + if (requested_mode != effective_mode) { + cbm_coverage_row_t *old_cov = NULL; + int old_cov_count = 0; + if (cbm_store_coverage_get(store, project, &old_cov, &old_cov_count) != CBM_STORE_OK) { + cbm_log_error("incremental.err", "msg", "coverage_read_failed", "project", project); + cbm_store_free_coverage(old_cov, old_cov_count); + free(is_changed); + free(deleted); + free_mode_skipped(mode_skipped, mode_skipped_count); + cbm_store_free_file_hashes(stored, stored_count); + cbm_store_close(store); + return CBM_PIPELINE_ABORT_PRESERVE_DB; + } + + int cov_n = 0; + cbm_coverage_meta_t coverage_meta = {0}; + cbm_coverage_row_t *cov = + build_coverage_snapshot(p, old_cov, old_cov_count, NULL, 0, &cov_n, &coverage_meta); + cbm_project_t project_info = {0}; + bool have_project_info = + cbm_store_get_project(store, project, &project_info) == CBM_STORE_OK; + coverage_meta.generation = have_project_info ? project_info.indexed_at : NULL; + coverage_meta.hash_records_complete = stored_count == file_count + mode_skipped_count; + if (cbm_store_coverage_replace_ex(store, project, cov, cov_n, &coverage_meta) != + CBM_STORE_OK) { + cbm_log_error("incremental.err", "msg", "persist_coverage", "project", project); + noop_rc = CBM_PIPELINE_ABORT_PRESERVE_DB; + } else { + coverage_refreshed = true; + cbm_log_info("incremental.noop", "reason", "coverage_mode_changed"); + } + if (have_project_info) { + cbm_project_free_fields(&project_info); + } + free(cov); + cbm_store_free_coverage(old_cov, old_cov_count); + } else { + cbm_log_info("incremental.noop", "reason", "no_changes"); + } free(is_changed); free(deleted); free_mode_skipped(mode_skipped, mode_skipped_count); cbm_store_free_file_hashes(stored, stored_count); cbm_store_close(store); - return 0; + const char *repo_path = cbm_pipeline_repo_path(p); + if (noop_rc != 0) { + return noop_rc; + } + return export_artifact_if_needed(db_path, repo_path, project, persistence_required, + coverage_refreshed); } cbm_store_free_file_hashes(stored, stored_count); @@ -876,7 +998,7 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil .registry = registry, .cancelled = cbm_pipeline_cancelled_ptr(p), .pipeline = p, /* so passes can record per-file skips (Track B) */ - .mode = cbm_pipeline_get_mode(p), + .mode = effective_mode, .path_aliases = path_aliases, .excluded_dirs = excluded_dirs, .excluded_count = excluded_count, @@ -900,7 +1022,12 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil } } + /* Discovery already used requested_mode. Re-extraction must preserve the + * stronger capabilities recorded in Project.index_mode, including full-only + * C/C++ Macro nodes. Restore the process-wide gate immediately afterwards. */ + cbm_set_macro_extraction(effective_mode == CBM_MODE_FULL); run_extract_resolve(&ctx, changed_files, ci); + cbm_set_macro_extraction(requested_mode == CBM_MODE_FULL); cbm_pipeline_pass_k8s(&ctx, changed_files, ci); run_postpasses(&ctx, changed_files, ci, project); @@ -926,55 +1053,10 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil * Rows for deleted files are pruned against file_hashes inside the * replace. Borrowed strings: old_cov and the pipeline own them past the * dump_and_persist call below. */ - cbm_file_error_t *run_errs = NULL; - int run_err_count = 0; - cbm_pipeline_get_file_errors(p, &run_errs, &run_err_count); - char **run_excluded = NULL; - int run_excluded_count = 0; - cbm_pipeline_get_excluded(p, &run_excluded, &run_excluded_count); - cbm_ignored_file_t *run_ignored = NULL; - int run_ignored_count = 0; - int run_ignored_total = 0; - cbm_pipeline_get_ignored(p, &run_ignored, &run_ignored_count, &run_ignored_total); - cbm_coverage_row_t *cov = NULL; int cov_n = 0; - int cov_cap = old_cov_count + run_err_count + run_excluded_count + run_ignored_count; - if (cov_cap > 0) { - cov = (cbm_coverage_row_t *)malloc((size_t)cov_cap * sizeof(*cov)); - } - bool coverage_rows_available = cov_cap == 0 || cov != NULL; - if (cov) { - CBMHashTable *changed_set = cbm_ht_create(ci > 0 ? (size_t)ci * PAIR_LEN : CBM_SZ_64); - for (int i = 0; i < ci; i++) { - cbm_ht_set(changed_set, changed_files[i].rel_path, &changed_files[i]); - } - for (int i = 0; i < old_cov_count; i++) { - bool by_design = old_cov[i].kind && strncmp(old_cov[i].kind, "not_indexed", 11) == 0; - if (!by_design && old_cov[i].rel_path && - !cbm_ht_get(changed_set, old_cov[i].rel_path)) { - cov[cov_n++] = old_cov[i]; - } - } - cbm_ht_free(changed_set); - for (int i = 0; i < run_err_count; i++) { - cov[cov_n].rel_path = run_errs[i].path; - cov[cov_n].kind = run_errs[i].phase; - cov[cov_n].detail = run_errs[i].reason; - cov_n++; - } - for (int i = 0; i < run_excluded_count; i++) { - cov[cov_n].rel_path = run_excluded[i]; - cov[cov_n].kind = "not_indexed_dir"; - cov[cov_n].detail = "excluded subtree"; - cov_n++; - } - for (int i = 0; i < run_ignored_count; i++) { - cov[cov_n].rel_path = run_ignored[i].rel_path; - cov[cov_n].kind = "not_indexed_file"; - cov[cov_n].detail = run_ignored[i].reason; - cov_n++; - } - } + cbm_coverage_meta_t coverage_meta = {0}; + cbm_coverage_row_t *cov = build_coverage_snapshot(p, old_cov, old_cov_count, changed_files, ci, + &cov_n, &coverage_meta); free(changed_files); cbm_registry_free(registry); @@ -992,31 +1074,25 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil /* Step 7: Dump to disk (preserves mode-skipped hash rows so the next * reindex can correctly classify those files instead of seeing them - * as never-existed; also exports a fast-mode artifact when one is - * already present alongside the repo). */ + * as never-existed; also handles explicitly requested persistence and + * refreshes an artifact already present alongside the repo). */ /* Record committed counts before dump_and_persist (whose dump frees the * gbuf node index, zeroing the count) so the #334 plausibility gate also * covers incremental reindexes, not just full ones. */ cbm_pipeline_set_committed_counts(p, cbm_gbuf_node_count(existing), cbm_gbuf_edge_count(existing)); - int index_mode = cbm_pipeline_get_mode(p); - cbm_coverage_meta_t coverage_meta = { - .index_mode = incr_mode_name(index_mode), - .recording_status = - !coverage_rows_available - ? "unavailable" - : (run_ignored_total > run_ignored_count ? "truncated" : "complete"), - .ignored_files_stored = run_ignored_count, - .ignored_files_total = run_ignored_total, - .coverage_version = 1, - }; - dump_and_persist(existing, db_path, project, files, file_count, mode_skipped, - mode_skipped_count, cbm_pipeline_repo_path(p), cov, cov_n, &coverage_meta); + int persist_rc = dump_and_persist(existing, db_path, project, files, file_count, mode_skipped, + mode_skipped_count, cbm_pipeline_repo_path(p), + persistence_required, cov, cov_n, &coverage_meta); free(cov); cbm_store_free_coverage(old_cov, old_cov_count); free_mode_skipped(mode_skipped, mode_skipped_count); cbm_gbuf_free(existing); + if (persist_rc != 0) { + return persist_rc; + } + cbm_log_info("incremental.done", "elapsed_ms", itoa_buf((int)elapsed_ms(t0))); return 0; } diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index d2180a47a..3fc0cf1c2 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -31,6 +31,10 @@ * "no incremental route; continue with a full index" sentinel. */ #define CBM_PIPELINE_ABORT_PRESERVE_DB (-2) +/* Stable internal serialization used by Project metadata, coverage metadata, + * and route diagnostics. */ +const char *cbm_pipeline_mode_name(cbm_index_mode_t mode); + /* Canonicalize route-path parameter placeholders (":id", "{id}", "", * "${...}") to a single "{}" token so that client call sites and server * handlers rendezvous on the same Route QN regardless of framework syntax. @@ -620,9 +624,12 @@ int cbm_scan_project_env_urls_excluded(const char *root_path, cbm_env_binding_t /* Run incremental re-index on an existing disk DB. * Classifies files by mtime+size, deletes changed nodes, re-parses changed - * files, merges into disk DB. Returns 0 on success. */ + * files with effective_mode capabilities, then merges into disk DB. The + * pipeline's requested mode still owns discovery/exclusions. Returns 0 on + * success. */ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_file_info_t *files, - int file_count); + int file_count, cbm_index_mode_t effective_mode, + bool persistence_required); /* Pipeline accessors for incremental use */ const char *cbm_pipeline_repo_path(const cbm_pipeline_t *p); diff --git a/src/store/store.c b/src/store/store.c index 112d5ea71..edc884aca 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -108,6 +108,7 @@ struct cbm_store { sqlite3_stmt *stmt_upsert_node; sqlite3_stmt *stmt_find_node_by_id; sqlite3_stmt *stmt_find_node_by_qn; + sqlite3_stmt *stmt_find_node_properties_by_qn; sqlite3_stmt *stmt_find_node_by_qn_any; /* QN lookup without project filter */ sqlite3_stmt *stmt_find_nodes_by_name; sqlite3_stmt *stmt_find_nodes_by_name_any; /* name lookup without project filter */ @@ -174,19 +175,26 @@ static const char *safe_props(const char *s) { return (s && s[0]) ? s : "{}"; } -/* Duplicate a string onto the heap. */ -static char *heap_strdup(const char *s) { - if (!s) { +/* Duplicate an exact byte sequence and append a C-string terminator. */ +static char *heap_memdup0(const void *src, size_t len) { + if (!src && len > 0) { return NULL; } - size_t len = strlen(s); char *d = malloc(len + SKIP_ONE); if (d) { - memcpy(d, s, len + SKIP_ONE); + if (len > 0) { + memcpy(d, src, len); + } + d[len] = '\0'; } return d; } +/* Duplicate a string onto the heap. */ +static char *heap_strdup(const char *s) { + return s ? heap_memdup0(s, strlen(s)) : NULL; +} + /* Prepare a statement (cached). If already prepared, reset+clear. */ static sqlite3_stmt *prepare_cached(cbm_store_t *s, sqlite3_stmt **slot, const char *sql) { if (!s || !s->db) { @@ -958,6 +966,7 @@ void cbm_store_close(cbm_store_t *s) { finalize_stmt(&s->stmt_upsert_node); finalize_stmt(&s->stmt_find_node_by_id); finalize_stmt(&s->stmt_find_node_by_qn); + finalize_stmt(&s->stmt_find_node_properties_by_qn); finalize_stmt(&s->stmt_find_node_by_qn_any); finalize_stmt(&s->stmt_find_nodes_by_name); finalize_stmt(&s->stmt_find_nodes_by_name_any); @@ -1368,6 +1377,62 @@ int cbm_store_find_node_by_qn(cbm_store_t *s, const char *project, const char *q return CBM_STORE_NOT_FOUND; } +/* Test seam: lets the pipeline regression inject a sqlite3_step() failure + * while reading Project metadata. NULL = use sqlite3_step(). */ +int (*cbm_store_find_node_properties_step_hook_for_test)(sqlite3_stmt *) = NULL; + +int cbm_store_find_node_properties_by_qn(cbm_store_t *s, const char *project, const char *qn, + char **out_json, size_t *out_len) { + if (!s || !s->db || !project || !qn || !out_json || !out_len) { + return CBM_STORE_ERR; + } + *out_json = NULL; + *out_len = 0; + + sqlite3_stmt *stmt = prepare_cached(s, &s->stmt_find_node_properties_by_qn, + "SELECT properties FROM nodes " + "WHERE project = ?1 AND qualified_name = ?2;"); + if (!stmt) { + return CBM_STORE_ERR; + } + + int rc = bind_text(stmt, SKIP_ONE, project); + if (rc == SQLITE_OK) { + rc = bind_text(stmt, ST_COL_2, qn); + } + if (rc != SQLITE_OK) { + store_set_error_sqlite(s, "bind node properties lookup"); + return CBM_STORE_ERR; + } + + rc = cbm_store_find_node_properties_step_hook_for_test + ? cbm_store_find_node_properties_step_hook_for_test(stmt) + : sqlite3_step(stmt); + if (rc == SQLITE_DONE) { + return CBM_STORE_NOT_FOUND; + } + if (rc != SQLITE_ROW) { + store_set_error_sqlite(s, "read node properties"); + return CBM_STORE_ERR; + } + + int properties_type = sqlite3_column_type(stmt, 0); + const void *properties = sqlite3_column_text(stmt, 0); + size_t properties_len = (size_t)sqlite3_column_bytes(stmt, 0); + if (!properties && properties_type != SQLITE_NULL) { + store_set_error_sqlite(s, "convert node properties"); + return CBM_STORE_ERR; + } + char *copy = heap_memdup0(properties, properties_len); + if (!copy) { + store_set_error(s, "alloc node properties"); + return CBM_STORE_ERR; + } + *out_json = copy; + *out_len = properties_len; + return CBM_STORE_OK; +} + int cbm_store_find_node_by_qn_any(cbm_store_t *s, const char *qn, cbm_node_t *out) { if (!s || !s->db) { return CBM_STORE_ERR; diff --git a/src/store/store.h b/src/store/store.h index 963837f88..cfc2791e2 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -298,6 +298,11 @@ int cbm_store_find_node_by_id(cbm_store_t *s, int64_t id, cbm_node_t *out); /* Find node by project + qualified_name. */ int cbm_store_find_node_by_qn(cbm_store_t *s, const char *project, const char *qn, cbm_node_t *out); +/* Read a node's properties with the exact SQLite byte length preserved. + * Allocates *out_json; caller frees it with free(). */ +int cbm_store_find_node_properties_by_qn(cbm_store_t *s, const char *project, const char *qn, + char **out_json, size_t *out_len); + /* Find node by qualified_name only (no project filter — QNs are globally unique). */ int cbm_store_find_node_by_qn_any(cbm_store_t *s, const char *qn, cbm_node_t *out); diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 4f095f385..cf7de9f79 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -9,6 +9,7 @@ #include "test_framework.h" #include "test_helpers.h" #include "foundation/mem.h" // cbm_mem_init/budget (back-pressure futile-nap test) +#include "pipeline/artifact.h" #include "pipeline/pipeline.h" #include "pipeline/pipeline_internal.h" #include "store/store.h" @@ -5807,6 +5808,524 @@ TEST(incremental_new_file_added) { PASS(); } +static bool project_has_index_mode(cbm_store_t *store, const char *project, + const char *expected_mode) { + cbm_node_t project_node = {0}; + if (cbm_store_find_node_by_qn(store, project, project, &project_node) != CBM_STORE_OK) { + return false; + } + + bool matches = false; + if (project_node.properties_json) { + yyjson_doc *doc = + yyjson_read(project_node.properties_json, strlen(project_node.properties_json), 0); + yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; + yyjson_val *mode = root ? yyjson_obj_get(root, "index_mode") : NULL; + matches = yyjson_equals_str(mode, expected_mode); + if (doc) { + yyjson_doc_free(doc); + } + } + + cbm_node_free_fields(&project_node); + return matches; +} + +static int project_count_nodes_by_label(cbm_store_t *store, const char *project, + const char *label) { + cbm_node_t *nodes = NULL; + int count = 0; + if (cbm_store_find_nodes_by_label(store, project, label, &nodes, &count) != CBM_STORE_OK) { + return -1; + } + cbm_store_free_nodes(nodes, count); + return count; +} + +static int setup_mode_upgrade_repo(char *tmpdir, size_t tmpdir_size, char *dbpath, + size_t dbpath_size) { + snprintf(tmpdir, tmpdir_size, "/tmp/cbm_mode_upgrade_XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) { + return -1; + } + snprintf(dbpath, dbpath_size, "%s/test.db", tmpdir); + + if (th_write_file( + TH_PATH(tmpdir, "user_validator.go"), + "package validation\n" + "import \"errors\"\n" + "import \"strings\"\n" + "func ValidateUser(u User) error {\n" + " if u.Name == \"\" { return errors.New(\"name required\") }\n" + " if len(u.Name) > 100 { return errors.New(\"name too long\") }\n" + " if u.Age < 0 { return errors.New(\"invalid age\") }\n" + " if u.Age > 200 { return errors.New(\"age too high\") }\n" + " if u.Email == \"\" { return errors.New(\"email required\") }\n" + " if !strings.Contains(u.Email, \"@\") { return errors.New(\"invalid email\") }\n" + " if u.Phone == \"\" { return errors.New(\"phone required\") }\n" + " if len(u.Phone) < 7 { return errors.New(\"phone too short\") }\n" + " if u.Country == \"\" { return errors.New(\"country required\") }\n" + " for _, tag := range u.Tags {\n" + " if tag == \"\" { return errors.New(\"empty tag\") }\n" + " }\n" + " return nil\n" + "}\n") != 0) { + return -1; + } + + return th_write_file( + TH_PATH(tmpdir, "order_validator.go"), + "package validation\n" + "import \"errors\"\n" + "import \"strings\"\n" + "func ValidateOrder(o Order) error {\n" + " if o.Title == \"\" { return errors.New(\"title required\") }\n" + " if len(o.Title) > 100 { return errors.New(\"title too long\") }\n" + " if o.Amount < 0 { return errors.New(\"invalid amount\") }\n" + " if o.Amount > 200 { return errors.New(\"amount too high\") }\n" + " if o.Status == \"\" { return errors.New(\"status required\") }\n" + " if !strings.Contains(o.Status, \"@\") { return errors.New(\"invalid status\") }\n" + " if o.Region == \"\" { return errors.New(\"region required\") }\n" + " if len(o.Region) < 7 { return errors.New(\"region too short\") }\n" + " if o.Vendor == \"\" { return errors.New(\"vendor required\") }\n" + " for _, item := range o.Items {\n" + " if item == \"\" { return errors.New(\"empty item\") }\n" + " }\n" + " return nil\n" + "}\n"); +} + +TEST(incremental_mode_upgrade_reindexes_capabilities) { + char tmpdir[256]; + char dbpath[512]; + ASSERT_EQ(setup_mode_upgrade_repo(tmpdir, sizeof(tmpdir), dbpath, sizeof(dbpath)), 0); + + cbm_pipeline_t *pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + cbm_pipeline_set_persistence(pipeline, true); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + char *project = strdup(cbm_pipeline_project_name(pipeline)); + cbm_pipeline_free(pipeline); + + cbm_store_t *store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + ASSERT_EQ(cbm_store_count_edges_by_type(store, project, "SIMILAR_TO"), 0); + ASSERT_TRUE(project_has_index_mode(store, project, "fast")); + cbm_store_close(store); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_MODERATE); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + int similarity_edges = cbm_store_count_edges_by_type(store, project, "SIMILAR_TO"); + ASSERT_GT(similarity_edges, 0); + ASSERT_TRUE(project_has_index_mode(store, project, "moderate")); + cbm_store_close(store); + + char artifact_dbpath[512]; + snprintf(artifact_dbpath, sizeof(artifact_dbpath), "%s/artifact-upgrade.db", tmpdir); + ASSERT_EQ(cbm_artifact_import(tmpdir, artifact_dbpath), 0); + cbm_store_t *artifact_store = cbm_store_open_path(artifact_dbpath); + ASSERT_NOT_NULL(artifact_store); + ASSERT_EQ(cbm_store_count_edges_by_type(artifact_store, project, "SIMILAR_TO"), + similarity_edges); + ASSERT_TRUE(project_has_index_mode(artifact_store, project, "moderate")); + cbm_store_close(artifact_store); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + ASSERT_EQ(cbm_store_count_edges_by_type(store, project, "SIMILAR_TO"), similarity_edges); + ASSERT_TRUE(project_has_index_mode(store, project, "moderate")); + cbm_store_close(store); + + free(project); + th_rmtree(tmpdir); + PASS(); +} + +TEST(incremental_mode_downgrade_preserves_similarity_for_changed_file) { + char tmpdir[256]; + char dbpath[512]; + ASSERT_EQ(setup_mode_upgrade_repo(tmpdir, sizeof(tmpdir), dbpath, sizeof(dbpath)), 0); + + cbm_pipeline_t *pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_MODERATE); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + char *project = strdup(cbm_pipeline_project_name(pipeline)); + cbm_pipeline_free(pipeline); + + cbm_store_t *store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + int similarity_edges = cbm_store_count_edges_by_type(store, project, "SIMILAR_TO"); + ASSERT_GT(similarity_edges, 0); + ASSERT_TRUE(project_has_index_mode(store, project, "moderate")); + cbm_store_close(store); + + FILE *changed = fopen(TH_PATH(tmpdir, "user_validator.go"), "a"); + ASSERT_NOT_NULL(changed); + ASSERT_GT(fprintf(changed, "\n// changed before fast reindex\n"), 0); + fclose(changed); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + ASSERT_EQ(cbm_store_count_edges_by_type(store, project, "SIMILAR_TO"), similarity_edges); + ASSERT_TRUE(project_has_index_mode(store, project, "moderate")); + cbm_store_close(store); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_MODERATE); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + ASSERT_EQ(cbm_store_count_edges_by_type(store, project, "SIMILAR_TO"), similarity_edges); + ASSERT_TRUE(project_has_index_mode(store, project, "moderate")); + cbm_store_close(store); + + free(project); + th_rmtree(tmpdir); + PASS(); +} + +TEST(incremental_mode_downgrade_preserves_full_extraction_for_changed_file) { + char tmpdir[256]; + char dbpath[512]; + ASSERT_EQ(setup_mode_upgrade_repo(tmpdir, sizeof(tmpdir), dbpath, sizeof(dbpath)), 0); + ASSERT_EQ(th_write_file(TH_PATH(tmpdir, "macros.c"), + "#define REVIEW_LIMIT 42\n" + "int review_value(void) { return REVIEW_LIMIT; }\n"), + 0); + + cbm_pipeline_t *pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_MODERATE); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + char *project = strdup(cbm_pipeline_project_name(pipeline)); + cbm_pipeline_free(pipeline); + + cbm_store_t *store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + ASSERT_EQ(project_count_nodes_by_label(store, project, "Macro"), 0); + ASSERT_TRUE(project_has_index_mode(store, project, "moderate")); + cbm_store_close(store); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + int macro_nodes = project_count_nodes_by_label(store, project, "Macro"); + ASSERT_GT(macro_nodes, 0); + ASSERT_TRUE(project_has_index_mode(store, project, "full")); + cbm_store_close(store); + + FILE *changed = fopen(TH_PATH(tmpdir, "macros.c"), "a"); + ASSERT_NOT_NULL(changed); + ASSERT_GT(fprintf(changed, "\n/* changed before moderate reindex */\n"), 0); + fclose(changed); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_MODERATE); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + ASSERT_EQ(project_count_nodes_by_label(store, project, "Macro"), macro_nodes); + ASSERT_TRUE(project_has_index_mode(store, project, "full")); + cbm_store_close(store); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + ASSERT_EQ(project_count_nodes_by_label(store, project, "Macro"), macro_nodes); + ASSERT_TRUE(project_has_index_mode(store, project, "full")); + cbm_store_close(store); + + free(project); + th_rmtree(tmpdir); + PASS(); +} + +TEST(incremental_noop_downgrade_honors_explicit_persistence) { + char tmpdir[256]; + char dbpath[512]; + ASSERT_EQ(setup_mode_upgrade_repo(tmpdir, sizeof(tmpdir), dbpath, sizeof(dbpath)), 0); + + cbm_pipeline_t *pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + ASSERT_FALSE(cbm_artifact_exists(tmpdir)); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + cbm_pipeline_set_persistence(pipeline, true); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + ASSERT_TRUE(cbm_artifact_exists(tmpdir)); + + th_rmtree(tmpdir); + PASS(); +} + +TEST(incremental_changed_file_propagates_explicit_persistence_failure) { + char tmpdir[256]; + char dbpath[512]; + ASSERT_EQ(setup_mode_upgrade_repo(tmpdir, sizeof(tmpdir), dbpath, sizeof(dbpath)), 0); + + cbm_pipeline_t *pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + FILE *changed = fopen(TH_PATH(tmpdir, "user_validator.go"), "a"); + ASSERT_NOT_NULL(changed); + ASSERT_GT(fprintf(changed, "\n// force incremental artifact export\n"), 0); + fclose(changed); + + char artifact_dir[512]; + snprintf(artifact_dir, sizeof(artifact_dir), "%s/.codebase-memory", tmpdir); + cbm_mkdir_p(artifact_dir, 0755); + char artifact_path[512]; + snprintf(artifact_path, sizeof(artifact_path), "%s/graph.db.zst", artifact_dir); + cbm_mkdir_p(artifact_path, 0755); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + cbm_pipeline_set_persistence(pipeline, true); + int rc = cbm_pipeline_run(pipeline); + cbm_pipeline_free(pipeline); + + ASSERT_NEQ(rc, 0); + + th_rmtree(tmpdir); + PASS(); +} + +/* Reproduce the destructive route from PR #1090 review: a transient SQLite + * error while reading Project.index_mode must preserve the existing DB rather + * than masquerade as missing legacy metadata and trigger a full rebuild. */ +extern int (*cbm_store_find_node_properties_step_hook_for_test)(sqlite3_stmt *); + +static int interrupt_node_properties_step(sqlite3_stmt *stmt) { + (void)stmt; + return SQLITE_INTERRUPT; +} + +TEST(incremental_mode_metadata_operational_error_preserves_db) { + char tmpdir[256]; + char dbpath[512]; + ASSERT_EQ(setup_mode_upgrade_repo(tmpdir, sizeof(tmpdir), dbpath, sizeof(dbpath)), 0); + + cbm_pipeline_t *pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + char *project = strdup(cbm_pipeline_project_name(pipeline)); + ASSERT_NOT_NULL(project); + cbm_pipeline_free(pipeline); + + cbm_store_t *store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + char sentinel_qn[512]; + snprintf(sentinel_qn, sizeof(sentinel_qn), "%s.metadata_error_sentinel", project); + cbm_node_t sentinel = {.project = project, + .label = "Function", + .name = "metadata_error_sentinel", + .qualified_name = sentinel_qn, + .file_path = "sentinel.go", + .properties_json = "{}"}; + ASSERT_GT(cbm_store_upsert_node(store, &sentinel), 0); + + const char *adr_text = "## PURPOSE\nPreserve on metadata read error."; + ASSERT_EQ(cbm_store_adr_store(store, project, adr_text), CBM_STORE_OK); + cbm_file_hash_t *hashes = NULL; + int hash_count = 0; + ASSERT_EQ(cbm_store_get_file_hashes(store, project, &hashes, &hash_count), CBM_STORE_OK); + ASSERT_GT(hash_count, 0); + cbm_store_free_file_hashes(hashes, hash_count); + cbm_store_close(store); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + cbm_store_find_node_properties_step_hook_for_test = interrupt_node_properties_step; + int pipeline_rc = cbm_pipeline_run(pipeline); + cbm_store_find_node_properties_step_hook_for_test = NULL; + cbm_pipeline_free(pipeline); + + store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + cbm_node_t sentinel_after = {0}; + int sentinel_rc = cbm_store_find_node_by_qn(store, project, sentinel_qn, &sentinel_after); + if (sentinel_rc == CBM_STORE_OK) { + cbm_node_free_fields(&sentinel_after); + } + + hashes = NULL; + hash_count = 0; + int hashes_rc = cbm_store_get_file_hashes(store, project, &hashes, &hash_count); + cbm_store_free_file_hashes(hashes, hash_count); + + cbm_adr_t adr = {0}; + int adr_rc = cbm_store_adr_get(store, project, &adr); + bool adr_preserved = + adr_rc == CBM_STORE_OK && adr.content && strcmp(adr.content, adr_text) == 0; + if (adr_rc == CBM_STORE_OK) { + cbm_store_adr_free(&adr); + } + bool metadata_preserved = project_has_index_mode(store, project, "fast"); + cbm_store_close(store); + + free(project); + th_rmtree(tmpdir); + + ASSERT_EQ(sentinel_rc, CBM_STORE_OK); + ASSERT_EQ(pipeline_rc, CBM_PIPELINE_ABORT_PRESERVE_DB); + ASSERT_EQ(hashes_rc, CBM_STORE_OK); + ASSERT_GT(hash_count, 0); + ASSERT_TRUE(adr_preserved); + ASSERT_TRUE(metadata_preserved); + PASS(); +} + +TEST(incremental_missing_mode_metadata_forces_reindex) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_mode_legacy_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmpdir)); + + char dbpath[512]; + snprintf(dbpath, sizeof(dbpath), "%s/test.db", tmpdir); + ASSERT_EQ(th_write_file(TH_PATH(tmpdir, "main.go"), + "package main\n\nfunc main() { println(\"hello\") }\n"), + 0); + + cbm_pipeline_t *pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + char *project = strdup(cbm_pipeline_project_name(pipeline)); + cbm_pipeline_free(pipeline); + + cbm_store_t *store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + cbm_node_t project_node = {0}; + ASSERT_EQ(cbm_store_find_node_by_qn(store, project, project, &project_node), CBM_STORE_OK); + cbm_node_t legacy_project_node = project_node; + legacy_project_node.properties_json = "{}"; + ASSERT_GT(cbm_store_upsert_node(store, &legacy_project_node), 0); + cbm_node_free_fields(&project_node); + + char sentinel_qn[512]; + snprintf(sentinel_qn, sizeof(sentinel_qn), "%s.legacy_sentinel", project); + cbm_node_t sentinel = {.project = project, + .label = "Function", + .name = "legacy_sentinel", + .qualified_name = sentinel_qn, + .file_path = "legacy.go", + .properties_json = "{}"}; + ASSERT_GT(cbm_store_upsert_node(store, &sentinel), 0); + cbm_store_close(store); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + cbm_node_t sentinel_after = {0}; + ASSERT_EQ(cbm_store_find_node_by_qn(store, project, sentinel_qn, &sentinel_after), + CBM_STORE_NOT_FOUND); + ASSERT_TRUE(project_has_index_mode(store, project, "fast")); + cbm_store_close(store); + + free(project); + th_rmtree(tmpdir); + PASS(); +} + +TEST(incremental_nul_suffixed_mode_metadata_forces_reindex) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_mode_nul_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmpdir)); + + char dbpath[512]; + snprintf(dbpath, sizeof(dbpath), "%s/test.db", tmpdir); + ASSERT_EQ(th_write_file(TH_PATH(tmpdir, "main.go"), + "package main\n\nfunc main() { println(\"hello\") }\n"), + 0); + + cbm_pipeline_t *pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + char *project = strdup(cbm_pipeline_project_name(pipeline)); + cbm_pipeline_free(pipeline); + + cbm_store_t *store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + static const char corrupt_properties[] = "{\"index_mode\":\"full\"}\0garbage"; + sqlite3_stmt *update = NULL; + ASSERT_EQ(sqlite3_prepare_v2(cbm_store_get_db(store), + "UPDATE nodes SET properties = ?1 " + "WHERE project = ?2 AND qualified_name = ?3", + -1, &update, NULL), + SQLITE_OK); + ASSERT_EQ(sqlite3_bind_text(update, 1, corrupt_properties, + (int)sizeof(corrupt_properties) - 1, SQLITE_STATIC), + SQLITE_OK); + ASSERT_EQ(sqlite3_bind_text(update, 2, project, -1, SQLITE_STATIC), SQLITE_OK); + ASSERT_EQ(sqlite3_bind_text(update, 3, project, -1, SQLITE_STATIC), SQLITE_OK); + ASSERT_EQ(sqlite3_step(update), SQLITE_DONE); + ASSERT_EQ(sqlite3_finalize(update), SQLITE_OK); + + char sentinel_qn[512]; + snprintf(sentinel_qn, sizeof(sentinel_qn), "%s.nul_mode_sentinel", project); + cbm_node_t sentinel = {.project = project, + .label = "Function", + .name = "nul_mode_sentinel", + .qualified_name = sentinel_qn, + .file_path = "nul-mode.go", + .properties_json = "{}"}; + ASSERT_GT(cbm_store_upsert_node(store, &sentinel), 0); + cbm_store_close(store); + + pipeline = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); + ASSERT_NOT_NULL(pipeline); + ASSERT_EQ(cbm_pipeline_run(pipeline), 0); + cbm_pipeline_free(pipeline); + + store = cbm_store_open_path(dbpath); + ASSERT_NOT_NULL(store); + cbm_node_t sentinel_after = {0}; + ASSERT_EQ(cbm_store_find_node_by_qn(store, project, sentinel_qn, &sentinel_after), + CBM_STORE_NOT_FOUND); + ASSERT_TRUE(project_has_index_mode(store, project, "fast")); + cbm_store_close(store); + + free(project); + th_rmtree(tmpdir); + PASS(); +} + TEST(incremental_fast_preserves_mode_skipped_tools_dir) { /* Regression: 2026-04-13. A fast-mode reindex after a full-mode index * was silently destroying every file under FAST_SKIP_DIRS directories @@ -5851,6 +6370,7 @@ TEST(incremental_fast_preserves_mode_skipped_tools_dir) { /* Step 1: full-mode index — both files should be present */ cbm_pipeline_t *p = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FULL); ASSERT_NOT_NULL(p); + cbm_pipeline_set_persistence(p, true); ASSERT_EQ(cbm_pipeline_run(p), 0); char *project = strdup(cbm_pipeline_project_name(p)); cbm_pipeline_free(p); @@ -5866,6 +6386,14 @@ TEST(incremental_fast_preserves_mode_skipped_tools_dir) { int total_before = cbm_store_count_nodes(s, project); cbm_store_close(s); + /* Persistence creates .gitattributes after the first discovery pass. + * Absorb that file before the downgrade so step 2 exercises the real + * zero-change metadata-only route rather than a normal incremental dump. */ + p = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + cbm_pipeline_free(p); + /* Step 2: fast-mode reindex — tools/util.go MUST survive (additive semantics) */ p = cbm_pipeline_new(tmpdir, dbpath, CBM_MODE_FAST); ASSERT_NOT_NULL(p); @@ -5885,6 +6413,44 @@ TEST(incremental_fast_preserves_mode_skipped_tools_dir) { ASSERT_EQ(tools_count_after, tools_count_before); /* same nodes, untouched */ cbm_store_free_nodes(tools_nodes_after, tools_count_after); + /* Project metadata records the stronger capabilities still present in the + * graph, while coverage records this run's requested discovery scope. */ + ASSERT_TRUE(project_has_index_mode(s, project, "full")); + cbm_coverage_meta_t coverage_meta = {0}; + ASSERT_EQ(cbm_store_coverage_meta_get(s, project, &coverage_meta), CBM_STORE_OK); + ASSERT_STR_EQ(coverage_meta.index_mode, "fast"); + cbm_store_coverage_meta_clear(&coverage_meta); + + cbm_coverage_row_t *tools_coverage = NULL; + int tools_coverage_count = 0; + ASSERT_EQ(cbm_store_coverage_get_path(s, project, "tools/util.go", &tools_coverage, + &tools_coverage_count), + CBM_STORE_OK); + ASSERT_GT(tools_coverage_count, 0); + ASSERT_STR_EQ(tools_coverage[0].kind, "not_indexed_dir"); + cbm_store_free_coverage(tools_coverage, tools_coverage_count); + + char artifact_dbpath[512]; + snprintf(artifact_dbpath, sizeof(artifact_dbpath), "%s/artifact-import.db", tmpdir); + ASSERT_EQ(cbm_artifact_import(tmpdir, artifact_dbpath), 0); + cbm_store_t *artifact_store = cbm_store_open_path(artifact_dbpath); + ASSERT_NOT_NULL(artifact_store); + cbm_coverage_meta_t artifact_coverage_meta = {0}; + ASSERT_EQ(cbm_store_coverage_meta_get(artifact_store, project, &artifact_coverage_meta), + CBM_STORE_OK); + ASSERT_STR_EQ(artifact_coverage_meta.index_mode, "fast"); + cbm_store_coverage_meta_clear(&artifact_coverage_meta); + cbm_coverage_row_t *artifact_tools_coverage = NULL; + int artifact_tools_coverage_count = 0; + ASSERT_EQ(cbm_store_coverage_get_path(artifact_store, project, "tools/util.go", + &artifact_tools_coverage, + &artifact_tools_coverage_count), + CBM_STORE_OK); + ASSERT_GT(artifact_tools_coverage_count, 0); + ASSERT_STR_EQ(artifact_tools_coverage[0].kind, "not_indexed_dir"); + cbm_store_free_coverage(artifact_tools_coverage, artifact_tools_coverage_count); + cbm_store_close(artifact_store); + /* Sanity: total node count should not have collapsed by ~the size of tools/ */ int total_after = cbm_store_count_nodes(s, project); ASSERT_GTE(total_after, total_before); /* additive — never less */ @@ -7095,6 +7661,14 @@ SUITE(pipeline) { RUN_TEST(incremental_aborts_when_previous_coverage_is_unreadable); RUN_TEST(incremental_detects_deleted_file); RUN_TEST(incremental_new_file_added); + RUN_TEST(incremental_mode_upgrade_reindexes_capabilities); + RUN_TEST(incremental_mode_downgrade_preserves_similarity_for_changed_file); + RUN_TEST(incremental_mode_downgrade_preserves_full_extraction_for_changed_file); + RUN_TEST(incremental_noop_downgrade_honors_explicit_persistence); + RUN_TEST(incremental_changed_file_propagates_explicit_persistence_failure); + RUN_TEST(incremental_mode_metadata_operational_error_preserves_db); + RUN_TEST(incremental_missing_mode_metadata_forces_reindex); + RUN_TEST(incremental_nul_suffixed_mode_metadata_forces_reindex); RUN_TEST(incremental_fast_preserves_mode_skipped_tools_dir); RUN_TEST(incremental_k8s_manifest_indexed); RUN_TEST(incremental_kustomize_module_indexed);