Skip to content

fix: rebuild index when mode adds capabilities #1090

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
astandrik wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: rebuild index when mode adds capabilities #1090

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 118 additions & 23 deletions src/pipeline/pipeline.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 6, P
#include "foundation/compat_thread.h"
#include "foundation/profile.h"
#include "foundation/mem.h"
#include "yyjson/yyjson.h"

#include <stdint.h>
#include <stdio.h>
Expand Down Expand Up @@ -153,6 +154,83 @@ static const char *itoa_buf(int val) {
return bufs[i];
}

const char *cbm_pipeline_mode_name(cbm_index_mode_t mode) {
switch (mode) {
case CBM_MODE_FULL:
return "full";
case CBM_MODE_MODERATE:
return "moderate";
case CBM_MODE_FAST:
return "fast";
default:
return "unknown";
}
}

typedef enum {
INDEX_MODE_METADATA_OK = 0,
INDEX_MODE_METADATA_MISSING_OR_INVALID,
INDEX_MODE_METADATA_ERROR,
} index_mode_metadata_status_t;

static index_mode_metadata_status_t parse_index_mode(const char *properties_json,
size_t properties_len,
cbm_index_mode_t *out_mode) {
if (!properties_json || !out_mode) {
return INDEX_MODE_METADATA_ERROR;
}

yyjson_read_err read_error = {0};
yyjson_doc *doc =
yyjson_read_opts((char *)properties_json, properties_len, 0, NULL, &read_error);
if (!doc) {
return read_error.code == YYJSON_READ_ERROR_MEMORY_ALLOCATION
? INDEX_MODE_METADATA_ERROR
: INDEX_MODE_METADATA_MISSING_OR_INVALID;
}

yyjson_val *root = yyjson_doc_get_root(doc);
yyjson_val *mode_value = yyjson_is_obj(root) ? yyjson_obj_get(root, "index_mode") : NULL;
index_mode_metadata_status_t status = INDEX_MODE_METADATA_OK;
if (yyjson_equals_str(mode_value, "full")) {
*out_mode = CBM_MODE_FULL;
} else if (yyjson_equals_str(mode_value, "moderate")) {
*out_mode = CBM_MODE_MODERATE;
} else if (yyjson_equals_str(mode_value, "fast")) {
*out_mode = CBM_MODE_FAST;
} else {
status = INDEX_MODE_METADATA_MISSING_OR_INVALID;
}

yyjson_doc_free(doc);
return status;
}

static bool index_mode_covers(cbm_index_mode_t stored_mode, cbm_index_mode_t requested_mode) {
return stored_mode >= CBM_MODE_FULL && stored_mode <= CBM_MODE_FAST &&
requested_mode >= CBM_MODE_FULL && requested_mode <= CBM_MODE_FAST &&
stored_mode <= requested_mode;
}

static index_mode_metadata_status_t read_stored_index_mode(cbm_store_t *store, const char *project,
cbm_index_mode_t *out_mode) {
char *properties_json = NULL;
size_t properties_len = 0;
int rc = cbm_store_find_node_properties_by_qn(store, project, project, &properties_json,
&properties_len);
if (rc == CBM_STORE_NOT_FOUND) {
return INDEX_MODE_METADATA_MISSING_OR_INVALID;
}
if (rc != CBM_STORE_OK) {
return INDEX_MODE_METADATA_ERROR;
}

index_mode_metadata_status_t status =
parse_index_mode(properties_json, properties_len, out_mode);
free(properties_json);
return status;
}

/* Log current + peak RSS at a pipeline phase boundary (memory profiling). */
static void log_phase_mem(const char *phase) {
enum { PL_BYTES_PER_MB = 1024 * 1024 };
Expand Down Expand Up @@ -448,7 +526,11 @@ static int pass_structure(cbm_pipeline_t *p, const cbm_file_info_t *files, int f
cbm_log_info("pass.start", "pass", "structure", "files", itoa_buf(file_count));

/* Project node */
cbm_gbuf_upsert_node(p->gbuf, "Project", p->project_name, p->project_name, NULL, 0, 0, "{}");
char project_props[CBM_SZ_64];
snprintf(project_props, sizeof(project_props), "{\"index_mode\":\"%s\"}",
cbm_pipeline_mode_name(p->mode));
cbm_gbuf_upsert_node(p->gbuf, "Project", p->project_name, p->project_name, NULL, 0, 0,
project_props);
const char *branch_qn = p->branch_qn ? p->branch_qn : p->project_name;
const char *branch_name = p->git_ctx.branch ? p->git_ctx.branch : "working-tree";
char branch_props[CBM_SZ_2K];
Expand Down Expand Up @@ -1116,17 +1198,37 @@ static int try_incremental_or_delete_db(cbm_pipeline_t *p, cbm_file_info_t *file
if (check_store && cbm_store_check_integrity(check_store)) {
cbm_file_hash_t *hashes = NULL;
int hash_count = 0;
cbm_index_mode_t stored_mode = CBM_MODE_FAST;
index_mode_metadata_status_t stored_mode_status =
read_stored_index_mode(check_store, p->project_name, &stored_mode);
if (stored_mode_status == INDEX_MODE_METADATA_ERROR) {
cbm_log_error("pipeline.route", "path", "metadata_read_error", "stored_mode", "error",
"requested_mode", cbm_pipeline_mode_name(p->mode), "action",
"preserve_db");
cbm_store_close(check_store);
free(db_path);
return CBM_PIPELINE_ABORT_PRESERVE_DB;
}
bool stored_mode_known = stored_mode_status == INDEX_MODE_METADATA_OK;
cbm_store_get_file_hashes(check_store, p->project_name, &hashes, &hash_count);
cbm_store_free_file_hashes(hashes, hash_count);
cbm_store_close(check_store);
if (hash_count > 0 && file_count <= hash_count + (hash_count / PAIR_LEN)) {
bool mode_covered = stored_mode_known && index_mode_covers(stored_mode, p->mode);
if (hash_count > 0 && mode_covered && file_count <= hash_count + (hash_count / PAIR_LEN)) {
cbm_log_info("pipeline.route", "path", "incremental", "stored_hashes",
itoa_buf(hash_count));
int rc = cbm_pipeline_run_incremental(p, db_path, files, file_count);
itoa_buf(hash_count), "stored_mode", cbm_pipeline_mode_name(stored_mode),
"requested_mode", cbm_pipeline_mode_name(p->mode), "effective_mode",
cbm_pipeline_mode_name(stored_mode));
int rc = cbm_pipeline_run_incremental(p, db_path, files, file_count, stored_mode,
p->persistence);
free(db_path);
return rc;
}
if (hash_count > 0) {
if (hash_count > 0 && !mode_covered) {
cbm_log_info("pipeline.route", "path", "mode_upgrade_reindex", "stored_mode",
stored_mode_known ? cbm_pipeline_mode_name(stored_mode) : "unknown",
"requested_mode", cbm_pipeline_mode_name(p->mode));
} else if (hash_count > 0) {
cbm_log_info("pipeline.route", "path", "mode_change_reindex", "stored_hashes",
itoa_buf(hash_count), "discovered", itoa_buf(file_count));
}
Expand Down Expand Up @@ -1173,19 +1275,6 @@ static int64_t stat_mtime_ns(const struct stat *fst) {
#endif
}

static const char *pipeline_mode_name(cbm_index_mode_t mode) {
switch (mode) {
case CBM_MODE_FULL:
return "full";
case CBM_MODE_MODERATE:
return "moderate";
case CBM_MODE_FAST:
return "fast";
default:
return "unknown";
}
}

/* Dump graph to SQLite and persist file hashes for incremental indexing. */
static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_info_t *files, int file_count,
struct timespec *t) {
Expand Down Expand Up @@ -1334,7 +1423,7 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_info_t *fil
: (p->ignored_total > p->ignored_count ? "truncated" : "complete");
cbm_coverage_meta_t coverage_meta = {
.generation = have_project_info ? project_info.indexed_at : NULL,
.index_mode = pipeline_mode_name(p->mode),
.index_mode = cbm_pipeline_mode_name(p->mode),
.recording_status = recording_status,
.ignored_files_stored = p->ignored_count,
.ignored_files_total = p->ignored_total,
Expand Down Expand Up @@ -1377,16 +1466,22 @@ static int dump_and_persist_hashes(cbm_pipeline_t *p, const cbm_file_info_t *fil
free(p->saved_adr);
p->saved_adr = NULL;

/* Export persistent artifact if enabled */
if (p->persistence) {
/* Export when persistence is explicitly enabled, or refresh an artifact
* created by an earlier run so a full rebuild cannot leave it stale. */
bool persistence_required = p->persistence;
if (persistence_required || cbm_artifact_exists(p->repo_path)) {
CBM_PROF_START(t_art);
int arc = cbm_artifact_export(db_path, p->repo_path, p->project_name, CBM_ARTIFACT_BEST);
CBM_PROF_END("persist", "6_artifact_export", t_art);
if (arc != 0) {
const char *err = cbm_artifact_export_last_error();
cbm_log_error("pipeline.err", "phase", "artifact_export", "err", err ? err : "unknown");
/* A failed persistence export intentionally fails the run; this used to be ignored. */
return arc;
/* An explicitly requested persistence export is part of the run's
* contract. Refreshing a pre-existing artifact remains best-effort,
* matching the incremental pipeline. */
if (persistence_required) {
return arc;
}
}
}

Expand Down
Loading
Loading