diff --git a/Makefile.cbm b/Makefile.cbm index 43ba5ffca..44ccb7473 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -174,7 +174,7 @@ STORE_SRCS = src/store/store.c CYPHER_SRCS = src/cypher/cypher.c # MCP server module (new) -MCP_SRCS = src/mcp/mcp.c src/mcp/index_supervisor.c +MCP_SRCS = src/mcp/mcp.c src/mcp/index_supervisor.c src/mcp/compact_out.c # Discover module (new) DISCOVER_SRCS = \ diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index 6d9675f13..d44691aee 100755 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -174,7 +174,9 @@ PROJECT=$(echo "$RESULT" | python3 -c "import json,sys; d=json.loads(sys.stdin.r if ! SEARCH=$(cli search_graph --project "$PROJECT" --name-pattern compute); then echo "FAIL: search_graph (flag form) exited non-zero"; cat "$CLI_STDERR"; exit 1 fi -TOTAL=$(echo "$SEARCH" | python3 -c "import json,sys; d=json.loads(sys.stdin.read()); print(d.get('total',0))" 2>/dev/null || echo "0") +# search_graph default output is TOON (key: value scalars + header/rows tables) +TOTAL=$(echo "$SEARCH" | sed -n 's/^total: //p' | head -1) +TOTAL=${TOTAL:-0} if [ "$TOTAL" -lt 1 ]; then echo "FAIL: search_graph for 'compute' returned 0 results" exit 1 @@ -185,7 +187,9 @@ echo "OK: search_graph found $TOTAL result(s) for 'compute'" if ! TRACE=$(cli trace_path --project "$PROJECT" --function-name compute --direction inbound --depth 1); then echo "FAIL: trace_path (flag form) exited non-zero"; cat "$CLI_STDERR"; exit 1 fi -CALLERS=$(echo "$TRACE" | python3 -c "import json,sys; d=json.loads(sys.stdin.read()); print(len(d.get('callers',[])))" 2>/dev/null || echo "0") +# trace_path default output is TOON: the callers[N]{...} header carries the count +CALLERS=$(echo "$TRACE" | sed -n 's/^callers\[\([0-9]*\)\].*/\1/p' | head -1) +CALLERS=${CALLERS:-0} if [ "$CALLERS" -lt 1 ]; then echo "FAIL: trace_path found 0 callers for 'compute'" exit 1 @@ -207,7 +211,8 @@ echo "OK: schema has $LABELS node labels" if ! FOLDERS=$(cli search_graph --project "$PROJECT" --label Folder); then echo "FAIL: search_graph --label Folder exited non-zero"; cat "$CLI_STDERR"; exit 1 fi -FOLDER_COUNT=$(echo "$FOLDERS" | python3 -c "import json,sys; d=json.loads(sys.stdin.read()); print(d.get('total',0))" 2>/dev/null || echo "0") +FOLDER_COUNT=$(echo "$FOLDERS" | sed -n 's/^total: //p' | head -1) +FOLDER_COUNT=${FOLDER_COUNT:-0} if [ "$FOLDER_COUNT" -lt 2 ]; then echo "FAIL: expected at least 2 Folder nodes (src, src/pkg), got $FOLDER_COUNT" exit 1 @@ -217,7 +222,8 @@ echo "OK: $FOLDER_COUNT Folder nodes (init.py didn't clobber them)" # 3d-cypher: query_graph Cypher capabilities # #238 WITH DISTINCT — all functions share label "Function" → collapses to 1 row. CYPHER_WD=$(cli query_graph --project "$PROJECT" --query "MATCH (f:Function) WITH DISTINCT f.label AS lbl RETURN lbl") -WD_ROWS=$(echo "$CYPHER_WD" | python3 -c "import json,sys; d=json.loads(sys.stdin.read()); print(len(d.get('rows',[])))" 2>/dev/null || echo "0") +WD_ROWS=$(echo "$CYPHER_WD" | sed -n 's/^total: //p' | head -1) +WD_ROWS=${WD_ROWS:-0} if [ "$WD_ROWS" -lt 1 ]; then echo "FAIL: query_graph WITH DISTINCT returned 0 rows" echo "$CYPHER_WD" @@ -227,7 +233,8 @@ echo "OK: query_graph WITH DISTINCT returned $WD_ROWS row(s)" # #241 WHERE label test — f:Function is true for every Function node. CYPHER_LBL=$(cli query_graph --project "$PROJECT" --query "MATCH (f:Function) WHERE f:Function RETURN f.name") -LBL_ROWS=$(echo "$CYPHER_LBL" | python3 -c "import json,sys; d=json.loads(sys.stdin.read()); print(len(d.get('rows',[])))" 2>/dev/null || echo "0") +LBL_ROWS=$(echo "$CYPHER_LBL" | sed -n 's/^total: //p' | head -1) +LBL_ROWS=${LBL_ROWS:-0} if [ "$LBL_ROWS" -lt 1 ]; then echo "FAIL: query_graph WHERE label-test returned 0 rows" echo "$CYPHER_LBL" @@ -237,7 +244,8 @@ echo "OK: query_graph WHERE f:Function returned $LBL_ROWS row(s)" # #242 label alternation — (n:Function|Module) seeds either label. CYPHER_ALT=$(cli query_graph --project "$PROJECT" --query "MATCH (n:Function|Module) RETURN n.name") -ALT_ROWS=$(echo "$CYPHER_ALT" | python3 -c "import json,sys; d=json.loads(sys.stdin.read()); print(len(d.get('rows',[])))" 2>/dev/null || echo "0") +ALT_ROWS=$(echo "$CYPHER_ALT" | sed -n 's/^total: //p' | head -1) +ALT_ROWS=${ALT_ROWS:-0} if [ "$ALT_ROWS" -lt 1 ]; then echo "FAIL: query_graph label alternation returned 0 rows" echo "$CYPHER_ALT" @@ -247,7 +255,8 @@ echo "OK: query_graph (n:Function|Module) returned $ALT_ROWS row(s)" # #239 count(DISTINCT) — must parse and return a single aggregate row. CYPHER_CD=$(cli query_graph --project "$PROJECT" --query "MATCH (f:Function) RETURN count(DISTINCT f.label)") -CD_ROWS=$(echo "$CYPHER_CD" | python3 -c "import json,sys; d=json.loads(sys.stdin.read()); print(len(d.get('rows',[])))" 2>/dev/null || echo "0") +CD_ROWS=$(echo "$CYPHER_CD" | sed -n 's/^total: //p' | head -1) +CD_ROWS=${CD_ROWS:-0} if [ "$CD_ROWS" -ne 1 ]; then echo "FAIL: query_graph count(DISTINCT) expected 1 row, got $CD_ROWS" echo "$CYPHER_CD" @@ -261,7 +270,7 @@ cyp_first_cell() { # argv token, so string-literal args (e.g. replace(f.name,"a","A")) and Cypher # metacharacters {}|=~<>" need no JSON escaping. cli query_graph --project "$PROJECT" --query "$1" | - python3 -c "import json,sys; d=json.loads(sys.stdin.read()); rows=d.get('rows',[]); print(rows[0][0] if rows and rows[0] else '')" 2>/dev/null || echo "" + sed -n '/^rows\[/{n;p;}' | sed 's/^ //' | sed 's/^"//;s/"$//;s/\\"/"/g' } # labels(n) → JSON list like ["Function"] @@ -316,7 +325,8 @@ echo "OK: query_graph coalesce(f.nonesuch, f.name) = $COALV" # EXISTS { } pattern predicate (edge-type-specific existence) CYPHER_EX=$(cli query_graph --project "$PROJECT" --query "MATCH (f:Function) WHERE EXISTS { (f)-[:CALLS]->() } RETURN f.name") -EX_ROWS=$(echo "$CYPHER_EX" | python3 -c "import json,sys; d=json.loads(sys.stdin.read()); print(len(d.get('rows',[])))" 2>/dev/null || echo "0") +EX_ROWS=$(echo "$CYPHER_EX" | sed -n 's/^total: //p' | head -1) +EX_ROWS=${EX_ROWS:-0} if [ "$EX_ROWS" -lt 1 ]; then echo "FAIL: query_graph EXISTS{} predicate returned 0 rows"; echo "$CYPHER_EX"; exit 1 fi @@ -324,7 +334,8 @@ echo "OK: query_graph EXISTS { (f)-[:CALLS]->() } returned $EX_ROWS row(s)" # =~ regex match in WHERE CYPHER_RX=$(cli query_graph --project "$PROJECT" --query 'MATCH (f:Function) WHERE f.name =~ ".+" RETURN f.name') -RX_ROWS=$(echo "$CYPHER_RX" | python3 -c "import json,sys; d=json.loads(sys.stdin.read()); print(len(d.get('rows',[])))" 2>/dev/null || echo "0") +RX_ROWS=$(echo "$CYPHER_RX" | sed -n 's/^total: //p' | head -1) +RX_ROWS=${RX_ROWS:-0} if [ "$RX_ROWS" -lt 1 ]; then echo "FAIL: query_graph WHERE =~ regex returned 0 rows"; echo "$CYPHER_RX"; exit 1 fi @@ -347,7 +358,7 @@ LEFTV=$(cyp_first_cell 'MATCH (f:Function) RETURN left(f.name, 3) AS l LIMIT 1') # NOT EXISTS dead-code query (functions with no caller) CYPHER_NX=$(cli query_graph --project "$PROJECT" --query "MATCH (f:Function) WHERE NOT EXISTS { (f)<-[:CALLS]-() } RETURN f.name") -NX_OK=$(echo "$CYPHER_NX" | python3 -c "import json,sys; d=json.loads(sys.stdin.read()); print('rows' in d)" 2>/dev/null || echo "False") +NX_OK=$(echo "$CYPHER_NX" | grep -qE '^rows\[[0-9]+\]\{' && echo "True" || echo "False") [ "$NX_OK" = "True" ] && echo "OK: query_graph NOT EXISTS dead-code query executed" || { echo "FAIL: NOT EXISTS query"; echo "$CYPHER_NX" | head -c 300; exit 1; } # CASE expression in RETURN @@ -370,7 +381,9 @@ esac if ! ARCH=$(cli get_architecture --project "$PROJECT" --aspects clusters); then echo "FAIL: get_architecture (flag form) exited non-zero"; cat "$CLI_STDERR"; exit 1 fi -NCLUST=$(echo "$ARCH" | python3 -c "import json,sys; d=json.loads(sys.stdin.read()); print(len(d.get('clusters',[])))" 2>/dev/null || echo "0") +# get_architecture default output is TOON: clusters[N]{...} header carries the count +NCLUST=$(echo "$ARCH" | sed -n 's/^clusters\[\([0-9]*\)\].*/\1/p' | head -1) +NCLUST=${NCLUST:-0} if [ "$NCLUST" -lt 1 ]; then echo "FAIL: get_architecture returned 0 community clusters"; echo "$ARCH" | head -c 400; exit 1 fi @@ -378,11 +391,23 @@ echo "OK: get_architecture returned $NCLUST community cluster(s)" # 3g: search_code — basic search reports elapsed_ms + matches SC=$(cli search_code --project "$PROJECT" --pattern cbm_ --mode compact --limit 5) -echo "$SC" | python3 -c "import json,sys; d=json.loads(sys.stdin.read()); assert 'elapsed_ms' in d; print('OK: search_code elapsed_ms='+str(d['elapsed_ms'])+' total_grep_matches='+str(d.get('total_grep_matches')))" 2>/dev/null || { echo "FAIL: search_code basic / no elapsed_ms"; echo "$SC" | head -c 400; exit 1; } +# compact mode emits TOON scalars: `elapsed_ms: N` + `total_grep_matches: N` +SC_ELAPSED=$(echo "$SC" | sed -n 's/^elapsed_ms: //p' | head -1) +SC_GREPM=$(echo "$SC" | sed -n 's/^total_grep_matches: //p' | head -1) +if [ -n "$SC_ELAPSED" ]; then + echo "OK: search_code elapsed_ms=$SC_ELAPSED total_grep_matches=${SC_GREPM:-0}" +else + echo "FAIL: search_code basic / no elapsed_ms"; echo "$SC" | head -c 400; exit 1 +fi # 3g: search_code — literal '|' under regex=false must surface a warning (#282) SCW=$(cli search_code --project "$PROJECT" --pattern "cbm_init|cbm_nope" --regex false --limit 5) -echo "$SCW" | python3 -c "import json,sys; d=json.loads(sys.stdin.read()); w=' '.join(d.get('warnings',[])); assert 'regex=true' in w; print('OK: search_code literal-| warning surfaced')" 2>/dev/null || { echo "FAIL: search_code literal-| warning missing"; echo "$SCW" | head -c 400; exit 1; } +# TOON scalar `warning: ... regex=true ...` +if echo "$SCW" | grep -q "regex=true"; then + echo "OK: search_code literal-| warning surfaced" +else + echo "FAIL: search_code literal-| warning missing"; echo "$SCW" | head -c 400; exit 1 +fi # 3g: search_code — '&' in file_pattern accepted, not rejected as invalid (#272) SCA=$(cli search_code --project "$PROJECT" --pattern cbm_ --file-pattern "*R&D*.c" --limit 5) @@ -396,36 +421,39 @@ echo "=== Phase 3h: CLI input-mode guards (flags / stdin / --args-file / --help # Small helper: assert its stdin is a JSON object (exit non-zero otherwise). assert_json_obj() { python3 -c "import json,sys; d=json.loads(sys.stdin.read()); sys.exit(0 if isinstance(d,dict) else 1)" 2>/dev/null; } +# search_graph emits TOON by default: a results/semantic table header proves +# the tool parsed its typed flags and produced a well-formed response. +assert_toon_table() { grep -qE '^(results|semantic)\[[0-9]+\]\{'; } -# B1: INTEGER flag — --limit is schema-typed integer; must parse to valid JSON. +# B1: INTEGER flag — --limit is schema-typed integer; must parse and answer. if ! IM_INT=$(cli search_graph --project "$PROJECT" --name-pattern compute --limit 5); then echo "FAIL B1: search_graph --limit 5 exited non-zero"; cat "$CLI_STDERR"; exit 1 fi -if echo "$IM_INT" | assert_json_obj; then - echo "OK B1: INTEGER flag (--limit 5) parsed → valid JSON" +if echo "$IM_INT" | assert_toon_table; then + echo "OK B1: INTEGER flag (--limit 5) parsed → TOON results table" else - echo "FAIL B1: --limit 5 did not produce valid JSON"; echo "$IM_INT" | head -c 300; exit 1 + echo "FAIL B1: --limit 5 did not produce a TOON results table"; echo "$IM_INT" | head -c 300; exit 1 fi # B2: BOOLEAN bare flag — --exclude-entry-points with no value → true; must succeed. if ! IM_BOOL=$(cli search_graph --project "$PROJECT" --exclude-entry-points); then echo "FAIL B2: search_graph --exclude-entry-points exited non-zero"; cat "$CLI_STDERR"; exit 1 fi -if echo "$IM_BOOL" | assert_json_obj; then +if echo "$IM_BOOL" | assert_toon_table; then echo "OK B2: BOOLEAN bare flag (--exclude-entry-points) → success" else - echo "FAIL B2: --exclude-entry-points did not produce valid JSON"; echo "$IM_BOOL" | head -c 300; exit 1 + echo "FAIL B2: --exclude-entry-points did not produce a TOON results table"; echo "$IM_BOOL" | head -c 300; exit 1 fi # B3: ARRAY flag — repeated --semantic-query accumulates into a JSON array. -# semantic_results may be empty (index-mode dependent); only assert valid JSON. +# Semantic-only calls emit ONLY the semantic table (may be empty, header stays). if ! IM_ARR=$(cli search_graph --project "$PROJECT" --semantic-query send --semantic-query publish); then echo "FAIL B3: search_graph repeated --semantic-query exited non-zero"; cat "$CLI_STDERR"; exit 1 fi -if echo "$IM_ARR" | assert_json_obj; then - echo "OK B3: ARRAY flag (repeated --semantic-query) → valid JSON" +if echo "$IM_ARR" | grep -qE '^semantic\[[0-9]+\]\{'; then + echo "OK B3: ARRAY flag (repeated --semantic-query) → semantic TOON table" else - echo "FAIL B3: repeated --semantic-query did not produce valid JSON"; echo "$IM_ARR" | head -c 300; exit 1 + echo "FAIL B3: repeated --semantic-query did not produce a semantic table"; echo "$IM_ARR" | head -c 300; exit 1 fi # B4: STDIN — piped JSON resolves; this path must NOT emit a deprecation warning. diff --git a/src/cli/hook_augment.c b/src/cli/hook_augment.c index 35ff15f78..fb602239e 100644 --- a/src/cli/hook_augment.c +++ b/src/cli/hook_augment.c @@ -146,6 +146,9 @@ static char *ha_build_args(const char *project, const char *token) { yyjson_mut_obj_add_str(doc, root, "project", project); yyjson_mut_obj_add_str(doc, root, "name_pattern", name_pattern); yyjson_mut_obj_add_int(doc, root, "limit", HA_RESULT_LIMIT); + /* Programmatic consumer: search_graph defaults to TOON text, but + * ha_format_context parses the inner payload as JSON ("results"). */ + yyjson_mut_obj_add_str(doc, root, "format", "json"); char *out = yyjson_mut_write(doc, 0, NULL); yyjson_mut_doc_free(doc); diff --git a/src/foundation/constants.h b/src/foundation/constants.h index 977183d63..dd673e15e 100644 --- a/src/foundation/constants.h +++ b/src/foundation/constants.h @@ -70,11 +70,11 @@ enum { /* ── Default pagination limits ───────────────────────────────── */ /* Default page size for search_graph and the underlying store-layer search. - * Chosen so a typical broad query (e.g. file_pattern="**" on a 12k-node - * project) stays well within MCP tool-result size budgets. Callers that - * want more results paginate via offset+limit; the response always carries - * 'total' and 'has_more' so agents can detect truncation. */ -enum { CBM_DEFAULT_SEARCH_LIMIT = 200 }; + * Responses land in an LLM agent's context window, so the default favors a + * cheap first page (~50 TOON rows ≈ 1.5K tokens) over raw coverage; the + * response always carries 'total' and 'has_more', and agents page via + * offset+limit or narrow with label/file_pattern when has_more is true. */ +enum { CBM_DEFAULT_SEARCH_LIMIT = 50 }; /* ── Time conversion factors ─────────────────────────────────── */ #define CBM_NSEC_PER_SEC 1000000000ULL diff --git a/src/mcp/compact_out.c b/src/mcp/compact_out.c new file mode 100644 index 000000000..db05aa65b --- /dev/null +++ b/src/mcp/compact_out.c @@ -0,0 +1,230 @@ +/* compact_out.c — TOON emission helpers. See compact_out.h for the contract. */ +#include "mcp/compact_out.h" + +#include +#include +#include +#include + +enum { SB_INITIAL_CAP = 1024 }; + +void cbm_sb_init(cbm_sb_t *sb) { + sb->buf = NULL; + sb->len = 0; + sb->cap = 0; + sb->oom = false; +} + +static bool sb_reserve(cbm_sb_t *sb, size_t extra) { + if (sb->oom) { + return false; + } + if (sb->len + extra + 1 <= sb->cap) { + return true; + } + size_t ncap = sb->cap ? sb->cap : SB_INITIAL_CAP; + while (ncap < sb->len + extra + 1) { + ncap *= 2; + } + char *nbuf = (char *)realloc(sb->buf, ncap); + if (!nbuf) { + sb->oom = true; + return false; + } + sb->buf = nbuf; + sb->cap = ncap; + return true; +} + +void cbm_sb_append_n(cbm_sb_t *sb, const char *s, size_t n) { + if (!s || n == 0 || !sb_reserve(sb, n)) { + return; + } + memcpy(sb->buf + sb->len, s, n); + sb->len += n; + sb->buf[sb->len] = '\0'; +} + +void cbm_sb_append(cbm_sb_t *sb, const char *s) { + if (s) { + cbm_sb_append_n(sb, s, strlen(s)); + } +} + +char *cbm_sb_finish(cbm_sb_t *sb) { + if (sb->oom) { + free(sb->buf); + cbm_sb_init(sb); + return NULL; + } + if (!sb->buf) { + /* Empty but valid: return an owned empty string. */ + char *empty = (char *)calloc(1, 1); + return empty; + } + char *out = sb->buf; + cbm_sb_init(sb); + return out; +} + +void cbm_sb_free(cbm_sb_t *sb) { + free(sb->buf); + cbm_sb_init(sb); +} + +/* ── TOON quoting ───────────────────────────────────────────────── */ + +/* True when `s` parses as an integer or real literal (sign, digits, one dot). */ +static bool looks_numeric(const char *s) { + if (!s || !*s) { + return false; + } + const char *p = s; + if (*p == '-' || *p == '+') { + p++; + } + bool digit = false; + bool dot = false; + for (; *p; p++) { + if (isdigit((unsigned char)*p)) { + digit = true; + } else if (*p == '.' && !dot) { + dot = true; + } else { + return false; + } + } + return digit; +} + +static bool needs_quotes(const char *s) { + if (!s || !*s) { + return true; /* empty cell must be visible as "" */ + } + if (isspace((unsigned char)s[0]) || isspace((unsigned char)s[strlen(s) - 1])) { + return true; + } + for (const char *p = s; *p; p++) { + if (*p == ',' || *p == '"' || *p == '\n' || *p == '\r') { + return true; + } + } + if (strcmp(s, "true") == 0 || strcmp(s, "false") == 0 || strcmp(s, "null") == 0) { + return true; + } + return looks_numeric(s); +} + +static void append_quoted(cbm_sb_t *sb, const char *s) { + cbm_sb_append_n(sb, "\"", 1); + for (const char *p = s ? s : ""; *p; p++) { + switch (*p) { + case '"': + cbm_sb_append_n(sb, "\\\"", 2); + break; + case '\\': + cbm_sb_append_n(sb, "\\\\", 2); + break; + case '\n': + cbm_sb_append_n(sb, "\\n", 2); + break; + case '\r': + cbm_sb_append_n(sb, "\\r", 2); + break; + default: + cbm_sb_append_n(sb, p, 1); + break; + } + } + cbm_sb_append_n(sb, "\"", 1); +} + +static void append_value(cbm_sb_t *sb, const char *s) { + if (needs_quotes(s)) { + append_quoted(sb, s); + } else { + cbm_sb_append(sb, s); + } +} + +/* ── Scalars ────────────────────────────────────────────────────── */ + +void cbm_toon_scalar_str(cbm_sb_t *sb, const char *key, const char *val) { + cbm_sb_append(sb, key); + cbm_sb_append_n(sb, ": ", 2); + append_value(sb, val); + cbm_sb_append_n(sb, "\n", 1); +} + +void cbm_toon_scalar_int(cbm_sb_t *sb, const char *key, long long v) { + char num[32]; + snprintf(num, sizeof(num), "%lld", v); + cbm_sb_append(sb, key); + cbm_sb_append_n(sb, ": ", 2); + cbm_sb_append(sb, num); + cbm_sb_append_n(sb, "\n", 1); +} + +void cbm_toon_scalar_bool(cbm_sb_t *sb, const char *key, bool v) { + cbm_sb_append(sb, key); + cbm_sb_append_n(sb, ": ", 2); + cbm_sb_append(sb, v ? "true" : "false"); + cbm_sb_append_n(sb, "\n", 1); +} + +/* ── Tables ─────────────────────────────────────────────────────── */ + +void cbm_toon_table_header(cbm_sb_t *sb, const char *key, int n, const char *const *cols, + int ncols) { + char num[32]; + snprintf(num, sizeof(num), "[%d]{", n); + cbm_sb_append(sb, key); + cbm_sb_append(sb, num); + for (int i = 0; i < ncols; i++) { + if (i > 0) { + cbm_sb_append_n(sb, ",", 1); + } + cbm_sb_append(sb, cols[i]); + } + cbm_sb_append_n(sb, "}:\n", 3); +} + +void cbm_toon_row_begin(cbm_sb_t *sb) { + cbm_sb_append_n(sb, " ", 2); +} + +void cbm_toon_cell_str(cbm_sb_t *sb, const char *val, bool first) { + if (!first) { + cbm_sb_append_n(sb, ",", 1); + } + append_value(sb, val ? val : ""); +} + +void cbm_toon_cell_int(cbm_sb_t *sb, long long v, bool first) { + char num[32]; + snprintf(num, sizeof(num), "%lld", v); + if (!first) { + cbm_sb_append_n(sb, ",", 1); + } + cbm_sb_append(sb, num); +} + +void cbm_toon_cell_real(cbm_sb_t *sb, double v, bool first) { + char num[48]; + snprintf(num, sizeof(num), "%.4g", v); + if (!first) { + cbm_sb_append_n(sb, ",", 1); + } + cbm_sb_append(sb, num); +} + +void cbm_toon_cell_bool(cbm_sb_t *sb, bool v, bool first) { + if (!first) { + cbm_sb_append_n(sb, ",", 1); + } + cbm_sb_append(sb, v ? "true" : "false"); +} + +void cbm_toon_row_end(cbm_sb_t *sb) { + cbm_sb_append_n(sb, "\n", 1); +} diff --git a/src/mcp/compact_out.h b/src/mcp/compact_out.h new file mode 100644 index 000000000..9d9fdedda --- /dev/null +++ b/src/mcp/compact_out.h @@ -0,0 +1,54 @@ +/* compact_out.h — TOON (Token-Oriented Object Notation) emission helpers. + * + * Tool responses are consumed by LLM agents, where every byte is context + * tokens. TOON encodes the same data as JSON but declares tabular fields + * once in a header and streams rows line by line, cutting 40-60% of tokens + * on homogeneous result sets at equal-or-better retrieval accuracy + * (toonformat.dev/guide/benchmarks). Emitters here cover the subset we + * emit: scalar key-value lines and flat tables with explicit [N] lengths. + * + * Quoting: a cell/scalar is double-quoted iff it is empty, has leading or + * trailing whitespace, contains a comma, quote, newline, or CR, or would + * read as a non-string literal (true/false/null/number). Quotes and + * backslashes are escaped JSON-style; newlines become \n. + */ +#ifndef CBM_MCP_COMPACT_OUT_H +#define CBM_MCP_COMPACT_OUT_H + +#include +#include + +/* Minimal growing string buffer (OOM-safe: sticky flag, finish returns NULL). */ +typedef struct { + char *buf; + size_t len; + size_t cap; + bool oom; +} cbm_sb_t; + +void cbm_sb_init(cbm_sb_t *sb); +void cbm_sb_append_n(cbm_sb_t *sb, const char *s, size_t n); +void cbm_sb_append(cbm_sb_t *sb, const char *s); +/* Returns the heap buffer (caller frees) and resets sb. NULL on OOM. */ +char *cbm_sb_finish(cbm_sb_t *sb); +void cbm_sb_free(cbm_sb_t *sb); + +/* `key: value` scalar lines (top-level, no indent). */ +void cbm_toon_scalar_str(cbm_sb_t *sb, const char *key, const char *val); +void cbm_toon_scalar_int(cbm_sb_t *sb, const char *key, long long v); +void cbm_toon_scalar_bool(cbm_sb_t *sb, const char *key, bool v); + +/* `key[n]{col1,col2,...}:` table header; rows follow at 2-space indent. */ +void cbm_toon_table_header(cbm_sb_t *sb, const char *key, int n, const char *const *cols, + int ncols); + +/* Row cells: call row_begin, then cell_* per column (first=true for the + * first cell), then row_end. Empty/NULL strings emit as empty cells. */ +void cbm_toon_row_begin(cbm_sb_t *sb); +void cbm_toon_cell_str(cbm_sb_t *sb, const char *val, bool first); +void cbm_toon_cell_int(cbm_sb_t *sb, long long v, bool first); +void cbm_toon_cell_real(cbm_sb_t *sb, double v, bool first); +void cbm_toon_cell_bool(cbm_sb_t *sb, bool v, bool first); +void cbm_toon_row_end(cbm_sb_t *sb); + +#endif /* CBM_MCP_COMPACT_OUT_H */ diff --git a/src/mcp/mcp.c b/src/mcp/mcp.c index a0d8e2a91..84cf007e7 100644 --- a/src/mcp/mcp.c +++ b/src/mcp/mcp.c @@ -55,6 +55,7 @@ enum { #include "foundation/log.h" #include "foundation/limits.h" #include "mcp/index_supervisor.h" +#include "mcp/compact_out.h" #include "foundation/str_util.h" #include "foundation/dump_verify.h" #include "foundation/compat_regex.h" @@ -352,11 +353,17 @@ static const tool_def_t TOOLS[] = { "(2) name_pattern='.*regex.*' for exact pattern matching; (3) semantic_query=[...] for " "vector cosine search that bridges vocabulary (finds 'publish' when you search 'send'). " "The three modes are independent and can be combined in a single call. " - "PAGINATION: results are capped at limit (default 200) — broader queries are silently " - "truncated. The response always includes 'total' (full match count before limit) and " - "'has_more' (true when total > offset+returned). Detect truncation with has_more, then " - "page by re-calling with offset=offset+limit until has_more is false. Narrow first via " - "label/file_pattern/min_degree before paginating large result sets.", + "RESPONSE: compact TOON tables by default — `results[N]{qn,label,file,lines,in,out}:` " + "header then one row per hit. in/out = TOTAL degree across ALL edge types (DEFINES, " + "USAGE, CALLS, ...), NOT caller/callee counts — use trace_path for callers. Add per-node " + "property columns via " + "fields (e.g. [\"complexity\",\"signature\",\"docstring\"]); pass format=\"json\" for " + "legacy verbose objects (include_connected always uses JSON). " + "PAGINATION: results are capped at limit (default 50). The response always includes " + "'total' (full match count before limit) and 'has_more' (true when total > " + "offset+returned). Detect truncation with has_more, then page by re-calling with " + "offset=offset+limit until has_more is false. Narrow first via label/file_pattern/" + "min_degree before paginating large result sets.", "{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"}," "\"query\":{\"type\":\"string\",\"description\":\"Natural-language or keyword full-text " "search using BM25 ranking. Tokens are split on whitespace; camelCase identifiers are " @@ -374,11 +381,18 @@ static const tool_def_t TOOLS[] = { "Each keyword is scored independently via per-keyword min-cosine; results reflect functions " "that score well on ALL keywords. Requires moderate/full index mode. Results appear in the " "'semantic_results' field (separate from 'results').\"},\"limit\":{\"type\":" - "\"integer\",\"description\":\"Max results per call. Default 200. Response carries " + "\"integer\",\"description\":\"Max results per call. Default 50. Response carries " "'total' (full match count) and 'has_more' (true if truncated) so callers can " "detect the limit and paginate.\"},\"offset\":{\"type\":\"integer\",\"default\":0," "\"description\":\"Skip the first N matching nodes. Combine with 'limit' to page: " - "increment offset by limit and re-call while has_more is true.\"}}," + "increment offset by limit and re-call while has_more is true.\"}," + "\"format\":{\"type\":\"string\",\"enum\":[\"toon\",\"json\"],\"default\":\"toon\"," + "\"description\":\"Response encoding. toon (default): compact header+rows tables, " + "~60% fewer tokens. json: legacy verbose per-node objects.\"}," + "\"fields\":{\"type\":\"array\",\"items\":{\"type\":\"string\"},\"description\":" + "\"Extra per-node property columns for toon output, e.g. complexity, cognitive, " + "signature, docstring, return_type, is_test, lines(int). Missing values emit as " + "empty cells.\"}}," "\"required\":[\"project\"]}"}, {"query_graph", "Query graph", @@ -418,7 +432,10 @@ static const tool_def_t TOOLS[] = { {"trace_path", "Trace path", "Trace paths through the code graph. Modes: calls (callers/callees), data_flow (value " "propagation with args at each hop), cross_service (through HTTP/async Route nodes). " - "Use INSTEAD OF grep for callers, dependencies, impact analysis, or data flow tracing.", + "Use INSTEAD OF grep for callers, dependencies, impact analysis, or data flow tracing. " + "RESPONSE: compact TOON tables — `callees[N]{qn,hop}:`/`callers[N]{qn,hop}:` headers then " + "one row per reached node (hop = BFS distance); risk/test/args columns appear when the " + "matching flags are set. Pass format=\"json\" for legacy verbose objects.", "{\"type\":\"object\",\"properties\":{\"function_name\":{\"type\":\"string\"},\"project\":{" "\"type\":\"string\"},\"direction\":{\"type\":\"string\",\"enum\":[\"inbound\",\"outbound\"," "\"both\"],\"default\":\"both\"},\"depth\":{\"type\":\"integer\",\"default\":3},\"mode\":{" @@ -433,8 +450,11 @@ static const tool_def_t TOOLS[] = { "\"description\":\"Add risk classification (CRITICAL/HIGH/MEDIUM/LOW) based on hop distance" "\"},\"include_tests\":{\"type\":\"boolean\",\"default\":false," "\"description\":\"Include test files in results. When false (default), test files are " - "filtered out. When true, test nodes are included with is_test=true marker." - "\"}},\"required\":[\"function_name\",\"project\"]}"}, + "filtered out. When true, test nodes are included with a test column/marker.\"}," + "\"format\":{\"type\":\"string\",\"enum\":[\"toon\",\"json\"],\"default\":\"toon\"," + "\"description\":\"Response encoding. toon (default): compact header+rows tables. " + "json: legacy verbose per-hop objects.\"}}," + "\"required\":[\"function_name\",\"project\"]}"}, {"get_code_snippet", "Get code snippet", "Read source code for a function/class/symbol. IMPORTANT: First call search_graph to find the " @@ -454,12 +474,14 @@ static const tool_def_t TOOLS[] = { "\"project\"]}"}, {"get_architecture", "Get architecture", - "Get high-level architecture overview — packages, services, dependencies, and project " - "structure at a glance. Includes 'clusters': Leiden community detection over the call/import " - "graph, surfacing the de-facto modules (each with a label, member count, cohesion score, " - "representative top_nodes, and the packages/edge_types that bind it) — use these to grasp " - "the real architectural seams, which often cut across the folder layout. Optional path scopes " - "analysis to nodes under that directory prefix (file_path).", + "Get high-level architecture overview. DEFAULT (no aspects) is a compact summary — " + "overview counts, languages, packages, entry_points; request more via aspects:[...] " + "(structure, dependencies, routes, hotspots, boundaries, layers, clusters, file_tree) or " + "[\"all\"]. 'clusters' runs Leiden community detection over the call/import graph, " + "surfacing the de-facto modules (label, member count, cohesion score, representative " + "top_nodes, binding packages/edge_types) — the real architectural seams, which often cut " + "across the folder layout. Optional path scopes analysis to nodes under that directory " + "prefix (file_path).", /* The aspects enum mirrors VALID_ASPECTS (see aspect_is_valid) — update both together. */ "{\"type\":\"object\",\"properties\":{\"project\":{\"type\":\"string\"},\"path\":{\"type\":" "\"string\",\"description\":\"Optional directory prefix to scope architecture (e.g. " @@ -474,7 +496,9 @@ static const tool_def_t TOOLS[] = { "Graph-augmented code search. Finds text patterns via grep, then enriches results with " "the knowledge graph: deduplicates matches into containing functions, ranks by structural " "importance (definitions first, popular functions next, tests last). " - "Modes: compact (default, signatures only — token efficient), full (with source), " + "Modes: compact (default, signatures only — token efficient), full (source capped at a " + "60-line window around the first match per hit; source_truncated marks the cut — use " + "get_code_snippet for the complete symbol), " "files (just file paths). Use path_filter regex to scope results. " "TRUNCATION: enriched results are capped at limit (default 10). Response carries " "'total_grep_matches' (raw grep hit count) and 'total_results' (deduplicated function " @@ -1400,7 +1424,18 @@ static void build_project_json_entry(yyjson_mut_doc *doc, yyjson_mut_val *arr, c yyjson_mut_val *p = yyjson_mut_obj(doc); yyjson_mut_obj_add_strcpy(doc, p, "name", project_name); yyjson_mut_obj_add_strcpy(doc, p, "root_path", root_path_buf); - add_git_context_json(doc, p, root_path_buf[0] ? root_path_buf : NULL); + /* Listing stays lean: only the branch (the one git fact that + * disambiguates same-repo projects). The 12-field git block — mostly + * null for non-git roots — cost ~10KB across a full cache and is one + * index_status call away for the project you actually care about. */ + if (root_path_buf[0]) { + cbm_git_context_t gctx = {0}; + (void)cbm_git_context_resolve(root_path_buf, &gctx); + if (gctx.is_git && gctx.branch) { + yyjson_mut_obj_add_strcpy(doc, p, "branch", gctx.branch); + } + cbm_git_context_free(&gctx); + } yyjson_mut_obj_add_int(doc, p, "nodes", nodes); yyjson_mut_obj_add_int(doc, p, "edges", edges); yyjson_mut_obj_add_int(doc, p, "size_bytes", size_bytes); @@ -1610,7 +1645,7 @@ enum { BM25_MIN_BUF = 2, /* minimum buffer size: at least NUL + one char */ BM25_SEP_RESERVE = 1, BM25_QUERY_BUF = 1024, - BM25_DEFAULT_LIMIT = 100, + BM25_DEFAULT_LIMIT = 50, BM25_COL_ID = 0, BM25_COL_LABEL = 1, BM25_COL_NAME = 2, @@ -1708,7 +1743,7 @@ static char *bm25_file_pattern_like(const char *file_pattern) { * Returns NULL if FTS5 is unavailable or the query produced no usable tokens, * in which case the caller falls back to the regex-based search path. */ static char *bm25_search(cbm_store_t *store, const char *project, const char *query, - const char *file_pattern, int limit, int offset) { + const char *file_pattern, int limit, int offset, bool toon) { sqlite3 *db = cbm_store_get_db(store); if (!db) { return NULL; @@ -1804,6 +1839,47 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu } } + if (toon) { + /* TOON: rows are buffered first because the table header carries the + * row count, which sqlite only yields by stepping to completion. */ + cbm_sb_t rows; + cbm_sb_init(&rows); + int emitted = 0; + while (sqlite3_step(stmt) == SQLITE_ROW) { + char lines[CBM_SZ_32]; + int sl = sqlite3_column_int(stmt, BM25_COL_START); + int el = sqlite3_column_int(stmt, BM25_COL_END); + if (sl > 0) { + snprintf(lines, sizeof(lines), "%d-%d", sl, el > sl ? el : sl); + } else { + lines[0] = '\0'; + } + cbm_toon_row_begin(&rows); + cbm_toon_cell_str(&rows, (const char *)sqlite3_column_text(stmt, BM25_COL_QN), true); + cbm_toon_cell_str(&rows, (const char *)sqlite3_column_text(stmt, BM25_COL_LABEL), + false); + cbm_toon_cell_str(&rows, (const char *)sqlite3_column_text(stmt, BM25_COL_FILE), false); + cbm_toon_cell_str(&rows, lines, false); + cbm_toon_cell_real(&rows, sqlite3_column_double(stmt, BM25_COL_RANK), false); + cbm_toon_row_end(&rows); + emitted++; + } + sqlite3_finalize(stmt); + free(file_like); + + cbm_sb_t sb; + cbm_sb_init(&sb); + cbm_toon_scalar_int(&sb, "total", total); + cbm_toon_scalar_str(&sb, "search_mode", "bm25"); + static const char *const cols[] = {"qn", "label", "file", "lines", "rank"}; + cbm_toon_table_header(&sb, "results", emitted, cols, 5); + char *rows_text = cbm_sb_finish(&rows); + cbm_sb_append(&sb, rows_text ? rows_text : ""); + free(rows_text); + cbm_toon_scalar_bool(&sb, "has_more", total > offset + emitted); + return cbm_sb_finish(&sb); + } + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); yyjson_mut_val *root = yyjson_mut_obj(doc); yyjson_mut_doc_set_root(doc, root); @@ -1920,15 +1996,25 @@ static void emit_semantic_results(yyjson_mut_doc *doc, yyjson_mut_val *root, yyjson_mut_obj_add_val(doc, root, "semantic_results", sem_results); } -/* Append the semantic_query vector-search results onto the doc. Returns - * true if semantic_query was provided as a non-array (type error — caller - * should surface to the user). */ -static bool run_semantic_query(yyjson_mut_doc *doc, yyjson_mut_val *root, const char *args, - cbm_store_t *store, const char *project, int limit) { +/* Run the semantic_query vector search from raw args. Sets *out_vresults / + * *out_vcount (caller frees via cbm_store_free_vector_results when vcount>0). + * Returns true if semantic_query was provided as a non-array (type error — + * caller should surface to the user). */ +static bool run_semantic_query_core(const char *args, cbm_store_t *store, const char *project, + int limit, cbm_vector_result_t **out_vresults, int *out_vcount, + bool *out_present) { enum { MAX_KW_SEARCH = 32 }; + *out_vresults = NULL; + *out_vcount = 0; + if (out_present) { + *out_present = false; + } yyjson_doc *args_doc = yyjson_read(args, strlen(args), 0); yyjson_val *args_root = args_doc ? yyjson_doc_get_root(args_doc) : NULL; yyjson_val *sq_val = args_root ? yyjson_obj_get(args_root, "semantic_query") : NULL; + if (out_present && sq_val) { + *out_present = true; + } bool type_error = false; if (sq_val && !yyjson_is_arr(sq_val)) { type_error = true; @@ -1941,8 +2027,8 @@ static bool run_semantic_query(yyjson_mut_doc *doc, yyjson_mut_val *root, const if (cbm_store_vector_search(store, project, keywords, ki, sem_limit, &vresults, &vcount) == CBM_STORE_OK && vcount > 0) { - emit_semantic_results(doc, root, vresults, vcount); - cbm_store_free_vector_results(vresults, vcount); + *out_vresults = vresults; + *out_vcount = vcount; } } if (args_doc) { @@ -1951,6 +2037,145 @@ static bool run_semantic_query(yyjson_mut_doc *doc, yyjson_mut_val *root, const return type_error; } +/* Append the semantic_query vector-search results onto the doc. Returns + * true if semantic_query was provided as a non-array (type error — caller + * should surface to the user). */ +static bool run_semantic_query(yyjson_mut_doc *doc, yyjson_mut_val *root, const char *args, + cbm_store_t *store, const char *project, int limit) { + cbm_vector_result_t *vresults = NULL; + int vcount = 0; + bool type_error = + run_semantic_query_core(args, store, project, limit, &vresults, &vcount, NULL); + if (vcount > 0) { + emit_semantic_results(doc, root, vresults, vcount); + cbm_store_free_vector_results(vresults, vcount); + } + return type_error; +} + +/* ── TOON output for search_graph ─────────────────────────────────── + * Default response encoding: header+rows tables (compact_out.h). The + * verbose per-node JSON objects remain available via format:"json" and + * are forced for include_connected=true (nested neighbor lists). */ + +enum { SG_MAX_EXTRA_FIELDS = 12 }; + +/* Internal-only node properties never emitted to agents: similarity / + * semantic pipeline intermediates (minhash fingerprint, structural profile, + * body-token bag). They dominate payload size and carry zero agent value. */ +static bool sg_field_blocked(const char *f) { + return strcmp(f, "fp") == 0 || strcmp(f, "sp") == 0 || strcmp(f, "bt") == 0; +} + +/* Parse the `fields` argument (array of property names) into out[] as + * pointers owned by the returned doc (caller frees the doc after emission). + * Blocked internal fields are silently dropped. */ +static int sg_parse_fields(const char *args, const char *out[], int max_out, + yyjson_doc **out_owner) { + *out_owner = NULL; + yyjson_doc *args_doc = yyjson_read(args, strlen(args), 0); + yyjson_val *args_root = args_doc ? yyjson_doc_get_root(args_doc) : NULL; + yyjson_val *fv = args_root ? yyjson_obj_get(args_root, "fields") : NULL; + if (!fv || !yyjson_is_arr(fv)) { + if (args_doc) { + yyjson_doc_free(args_doc); + } + return 0; + } + int n = 0; + size_t idx = 0; + size_t max = 0; + yyjson_val *item; + yyjson_arr_foreach(fv, idx, max, item) { + const char *s = yyjson_get_str(item); + if (s && s[0] && !sg_field_blocked(s) && n < max_out) { + out[n++] = s; + } + } + if (n == 0) { + yyjson_doc_free(args_doc); + return 0; + } + *out_owner = args_doc; + return n; +} + +/* Append one row's extra-field cells, pulled from the node's properties. */ +static void sg_toon_extra_cells(cbm_sb_t *sb, const char *props_json, const char *const *fields, + int nfields) { + yyjson_doc *pd = + (props_json && props_json[0]) ? yyjson_read(props_json, strlen(props_json), 0) : NULL; + yyjson_val *pr = pd ? yyjson_doc_get_root(pd) : NULL; + for (int f = 0; f < nfields; f++) { + yyjson_val *v = (pr && yyjson_is_obj(pr)) ? yyjson_obj_get(pr, fields[f]) : NULL; + if (v && yyjson_is_str(v)) { + cbm_toon_cell_str(sb, yyjson_get_str(v), false); + } else if (v && yyjson_is_bool(v)) { + cbm_toon_cell_bool(sb, yyjson_get_bool(v), false); + } else if (v && yyjson_is_int(v)) { + cbm_toon_cell_int(sb, yyjson_get_int(v), false); + } else if (v && yyjson_is_real(v)) { + cbm_toon_cell_real(sb, yyjson_get_real(v), false); + } else { + cbm_toon_cell_str(sb, "", false); + } + } + if (pd) { + yyjson_doc_free(pd); + } +} + +/* "start-end" line range, or empty when the node carries no line info. */ +static void sg_lines_str(char *out, size_t sz, int start, int end) { + if (start > 0) { + snprintf(out, sz, "%d-%d", start, end > start ? end : start); + } else { + out[0] = '\0'; + } +} + +/* Emit the regex-path search results as a TOON table. */ +static void emit_search_results_toon(cbm_sb_t *sb, const cbm_search_output_t *out, int offset, + const char *const *fields, int nfields) { + cbm_toon_scalar_int(sb, "total", out->total); + const char *cols[6 + SG_MAX_EXTRA_FIELDS] = {"qn", "label", "file", "lines", "in", "out"}; + int ncols = 6; + for (int f = 0; f < nfields; f++) { + cols[ncols++] = fields[f]; + } + cbm_toon_table_header(sb, "results", out->count, cols, ncols); + for (int i = 0; i < out->count; i++) { + const cbm_search_result_t *sr = &out->results[i]; + char lines[CBM_SZ_32]; + sg_lines_str(lines, sizeof(lines), sr->node.start_line, sr->node.end_line); + cbm_toon_row_begin(sb); + cbm_toon_cell_str(sb, sr->node.qualified_name, true); + cbm_toon_cell_str(sb, sr->node.label, false); + cbm_toon_cell_str(sb, sr->node.file_path, false); + cbm_toon_cell_str(sb, lines, false); + cbm_toon_cell_int(sb, sr->in_degree, false); + cbm_toon_cell_int(sb, sr->out_degree, false); + sg_toon_extra_cells(sb, sr->node.properties_json, fields, nfields); + cbm_toon_row_end(sb); + } + cbm_toon_scalar_bool(sb, "has_more", out->total > offset + out->count); +} + +/* Emit semantic vector-search results as a TOON table. */ +static void emit_semantic_results_toon(cbm_sb_t *sb, const cbm_vector_result_t *vresults, + int vcount) { + static const char *const cols[] = {"qn", "label", "file", "score"}; + cbm_toon_table_header(sb, "semantic", vcount, cols, 4); + for (int v = 0; v < vcount; v++) { + cbm_toon_row_begin(sb); + cbm_toon_cell_str(sb, vresults[v].qualified_name, true); + cbm_toon_cell_str(sb, vresults[v].label, false); + cbm_toon_cell_str(sb, vresults[v].file_path, false); + cbm_toon_cell_real(sb, vresults[v].score, false); + cbm_toon_row_end(sb); + } +} + static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { char *project = get_project_arg(args); cbm_store_t *store = resolve_store(srv, project); @@ -1962,6 +2187,16 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { return not_indexed; } + /* Response encoding: TOON tables by default (compact, header+rows). + * format:"json" restores the legacy verbose per-node objects; nested + * neighbor lists (include_connected) need them, so they force JSON. */ + char *format_arg = cbm_mcp_get_string_arg(args, "format"); + bool legacy_json = format_arg && strcmp(format_arg, "json") == 0; + free(format_arg); + if (cbm_mcp_get_bool_arg(args, "include_connected")) { + legacy_json = true; + } + /* BM25 path: if `query` is set, run FTS5 full-text search with ranking * and return early. The regex/vector path below is untouched for all * other callers. If FTS5 is unavailable or the query is empty after @@ -1971,7 +2206,8 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { int q_limit = cbm_mcp_get_int_arg(args, "limit", BM25_DEFAULT_LIMIT); int q_offset = cbm_mcp_get_int_arg(args, "offset", 0); char *q_file_pattern = cbm_mcp_get_string_arg(args, "file_pattern"); - char *bm25_json = bm25_search(store, project, query, q_file_pattern, q_limit, q_offset); + char *bm25_json = + bm25_search(store, project, query, q_file_pattern, q_limit, q_offset, !legacy_json); free(q_file_pattern); if (bm25_json) { free(query); @@ -2020,6 +2256,93 @@ static char *handle_search_graph(cbm_mcp_server_t *srv, const char *args) { .max_degree = max_degree, }; + if (!legacy_json) { + const char *fields[SG_MAX_EXTRA_FIELDS]; + yyjson_doc *fields_owner = NULL; + int nfields = sg_parse_fields(args, fields, SG_MAX_EXTRA_FIELDS, &fields_owner); + + cbm_vector_result_t *vresults = NULL; + int vcount = 0; + bool sq_present = false; + bool sq_type_error = + run_semantic_query_core(args, store, project, limit, &vresults, &vcount, &sq_present); + if (!sq_type_error) { + /* Semantic-only calls get semantic results only: the legacy + * behavior also ran the UNFILTERED regex search and prepended + * up to `limit` unrelated enriched nodes to the response. */ + bool has_filters = label || name_pattern || qn_pattern || file_pattern || + relationship || exclude_entry_points || + min_degree != CBM_NOT_FOUND || max_degree != CBM_NOT_FOUND; + bool semantic_only = sq_present && !has_filters; + + cbm_sb_t sb; + cbm_sb_init(&sb); + cbm_search_output_t tout = {0}; + if (!semantic_only) { + cbm_store_search(store, ¶ms, &tout); + emit_search_results_toon(&sb, &tout, offset, fields, nfields); + if (tout.total == 0) { + if (name_pattern && label) { + cbm_toon_scalar_str(&sb, "hint", + "No results. Try removing the label filter or " + "broadening the name_pattern regex."); + } else if (name_pattern) { + cbm_toon_scalar_str( + &sb, "hint", + "No nodes match this pattern. Check spelling or try a broader regex."); + } else if (label) { + cbm_toon_scalar_str(&sb, "hint", + "No nodes with this label. Available labels: " + "Function, Method, Class, Interface, Route, " + "Variable, Module, Package, File, Folder."); + } + } + } + if (vcount > 0) { + emit_semantic_results_toon(&sb, vresults, vcount); + } else if (semantic_only) { + static const char *const sem_cols[] = {"qn", "label", "file", "score"}; + cbm_toon_table_header(&sb, "semantic", 0, sem_cols, 4); + cbm_toon_scalar_str(&sb, "hint", + "No semantic matches. semantic_query needs a moderate/full " + "index; try broader or fewer keywords."); + } + if (vcount > 0) { + cbm_store_free_vector_results(vresults, vcount); + } + if (fields_owner) { + yyjson_doc_free(fields_owner); + } + cbm_store_search_free(&tout); + free(project); + free(label); + free(name_pattern); + free(qn_pattern); + free(file_pattern); + free(relationship); + char *text = cbm_sb_finish(&sb); + char *result = cbm_mcp_text_result(text ? text : "out of memory", text == NULL); + free(text); + return result; + } + /* semantic_query type error: fall through to the shared error text. */ + if (fields_owner) { + yyjson_doc_free(fields_owner); + } + free(project); + free(label); + free(name_pattern); + free(qn_pattern); + free(file_pattern); + free(relationship); + return cbm_mcp_text_result( + "semantic_query must be an array of keyword strings, e.g. " + "[\"send\",\"pubsub\",\"publish\"] — not a single string. Split your query " + "into individual keywords; each is scored independently via per-keyword " + "min-cosine.", + true); + } + cbm_search_output_t out = {0}; cbm_store_search(store, ¶ms, &out); @@ -2151,43 +2474,71 @@ static char *handle_query_graph(cbm_mcp_server_t *srv, const char *args) { return resp; } - yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); - yyjson_mut_val *root = yyjson_mut_obj(doc); - yyjson_mut_doc_set_root(doc, root); - - /* columns */ - yyjson_mut_val *cols = yyjson_mut_arr(doc); - for (int i = 0; i < result.col_count; i++) { - yyjson_mut_arr_add_str(doc, cols, result.columns[i]); - } - yyjson_mut_obj_add_val(doc, root, "columns", cols); + /* Response encoding: TOON table by default (the columns double as the + * table header); format:"json" restores the legacy columns/rows arrays. */ + char *qg_format = cbm_mcp_get_string_arg(args, "format"); + bool qg_legacy_json = qg_format && strcmp(qg_format, "json") == 0; + free(qg_format); + + char *json = NULL; + if (!qg_legacy_json) { + cbm_sb_t sb; + cbm_sb_init(&sb); + cbm_toon_table_header(&sb, "rows", result.row_count, (const char *const *)result.columns, + result.col_count); + for (int r = 0; r < result.row_count; r++) { + cbm_toon_row_begin(&sb); + for (int c = 0; c < result.col_count; c++) { + cbm_toon_cell_str(&sb, result.rows[r][c], c == 0); + } + cbm_toon_row_end(&sb); + } + cbm_toon_scalar_int(&sb, "total", result.row_count); + if (result.row_count == 0) { + cbm_toon_scalar_str(&sb, "hint", + "Query returned no results. Use get_graph_schema() to see " + "available labels and edge types."); + } + json = cbm_sb_finish(&sb); + } else { + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = yyjson_mut_obj(doc); + yyjson_mut_doc_set_root(doc, root); + + /* columns */ + yyjson_mut_val *cols = yyjson_mut_arr(doc); + for (int i = 0; i < result.col_count; i++) { + yyjson_mut_arr_add_str(doc, cols, result.columns[i]); + } + yyjson_mut_obj_add_val(doc, root, "columns", cols); + + /* rows */ + yyjson_mut_val *rows = yyjson_mut_arr(doc); + for (int r = 0; r < result.row_count; r++) { + yyjson_mut_val *row = yyjson_mut_arr(doc); + for (int c = 0; c < result.col_count; c++) { + yyjson_mut_arr_add_str(doc, row, result.rows[r][c]); + } + yyjson_mut_arr_add_val(rows, row); + } + yyjson_mut_obj_add_val(doc, root, "rows", rows); + yyjson_mut_obj_add_int(doc, root, "total", result.row_count); - /* rows */ - yyjson_mut_val *rows = yyjson_mut_arr(doc); - for (int r = 0; r < result.row_count; r++) { - yyjson_mut_val *row = yyjson_mut_arr(doc); - for (int c = 0; c < result.col_count; c++) { - yyjson_mut_arr_add_str(doc, row, result.rows[r][c]); + if (result.row_count == 0) { + yyjson_mut_obj_add_str( + doc, root, "hint", + "Query returned no results. Use get_graph_schema() to see available labels and " + "edge types."); } - yyjson_mut_arr_add_val(rows, row); - } - yyjson_mut_obj_add_val(doc, root, "rows", rows); - yyjson_mut_obj_add_int(doc, root, "total", result.row_count); - if (result.row_count == 0) { - yyjson_mut_obj_add_str( - doc, root, "hint", - "Query returned no results. Use get_graph_schema() to see available labels and " - "edge types."); + json = yy_doc_to_str(doc); + yyjson_mut_doc_free(doc); } - - char *json = yy_doc_to_str(doc); - yyjson_mut_doc_free(doc); cbm_cypher_result_free(&result); free(query); free(project); - char *res = cbm_mcp_text_result(json, false); + char *res = cbm_mcp_text_result(json ? json : "out of memory", json == NULL); free(json); return res; } @@ -2483,6 +2834,21 @@ static void append_cross_repo_summary(yyjson_mut_doc *doc, yyjson_mut_val *root, } } +/* Join a string list into buf with ';' separators (keeps TOON cells + * comma-free so they need no quoting). Truncates silently at sz. */ +static void arch_join_list(char *buf, size_t sz, const char **items, int n) { + size_t pos = 0; + buf[0] = '\0'; + for (int i = 0; i < n; i++) { + const char *s = items[i] ? items[i] : ""; + int w = snprintf(buf + pos, sz - pos, "%s%s", i > 0 ? ";" : "", s); + if (w < 0 || (size_t)w >= sz - pos) { + break; + } + pos += (size_t)w; + } +} + static char *handle_get_architecture(cbm_mcp_server_t *srv, const char *args) { char *project = get_project_arg(args); char *scope_path = cbm_mcp_get_string_arg(args, "path"); @@ -2557,6 +2923,20 @@ static char *handle_get_architecture(cbm_mcp_server_t *srv, const char *args) { } } + /* Default (no aspects) = compact summary. The old default rendered ALL + * aspects including the full file_tree — ~94KB (~23K tokens) on a + * mid-size repo, a context bomb for the LLM consumers. Explicit + * aspects (or ["all"]) keep full access to every section. */ + bool default_summary = false; + if (aspects_strs_count == 0) { + /* NOT "overview" — that means everything-except-file_tree. Totals and + * node_labels/edge_types counts are always emitted alongside. */ + aspects_strs[aspects_strs_count++] = "languages"; + aspects_strs[aspects_strs_count++] = "packages"; + aspects_strs[aspects_strs_count++] = "entry_points"; + default_summary = true; + } + cbm_schema_info_t schema = {0}; /* Counts-only: this handler renders label/type counts but never property * keys, and full key discovery json_each-scans every row (seconds-to- @@ -2564,15 +2944,223 @@ static char *handle_get_architecture(cbm_mcp_server_t *srv, const char *args) { cbm_store_get_schema_counts_scoped(store, project, scope_path, &schema); cbm_architecture_info_t arch = {0}; - cbm_store_get_architecture(store, project, scope_path, - aspects_strs_count > 0 ? aspects_strs : NULL, aspects_strs_count, - &arch); + cbm_store_get_architecture(store, project, scope_path, aspects_strs, aspects_strs_count, &arch); int node_count = cbm_store_count_nodes_scoped(store, project, scope_path); int edge_count = cbm_store_count_edges_scoped(store, project, scope_path); char norm_path[CBM_SZ_512]; bool path_scoped = cbm_store_normalize_arch_path(scope_path, norm_path, sizeof(norm_path)); + /* Response encoding: TOON tables by default; format:"json" restores the + * legacy per-item objects. */ + char *arch_format = cbm_mcp_get_string_arg(args, "format"); + bool arch_legacy_json = arch_format && strcmp(arch_format, "json") == 0; + free(arch_format); + + if (!arch_legacy_json) { + cbm_sb_t sb; + cbm_sb_init(&sb); + if (project) { + cbm_toon_scalar_str(&sb, "project", project); + } + if (default_summary) { + cbm_toon_scalar_str(&sb, "aspects_hint", + "Summary view (default). More on request via aspects:[...] — " + "structure, dependencies, routes, hotspots, boundaries, layers, " + "clusters, file_tree — or [\"all\"] for everything."); + } + if (path_scoped) { + cbm_toon_scalar_str(&sb, "path", norm_path); + cbm_toon_scalar_int(&sb, "root_total_nodes", cbm_store_count_nodes(store, project)); + cbm_toon_scalar_int(&sb, "root_total_edges", cbm_store_count_edges(store, project)); + cbm_toon_scalar_int(&sb, "scoped_total_nodes", node_count); + cbm_toon_scalar_int(&sb, "scoped_total_edges", edge_count); + } + cbm_toon_scalar_int(&sb, "total_nodes", node_count); + cbm_toon_scalar_int(&sb, "total_edges", edge_count); + + if (aspect_wanted(aspects_doc, aspects_arr, "structure") && schema.node_label_count > 0) { + static const char *const lcols[] = {"label", "count"}; + cbm_toon_table_header(&sb, "node_labels", schema.node_label_count, lcols, 2); + for (int i = 0; i < schema.node_label_count; i++) { + cbm_toon_row_begin(&sb); + cbm_toon_cell_str(&sb, schema.node_labels[i].label, true); + cbm_toon_cell_int(&sb, schema.node_labels[i].count, false); + cbm_toon_row_end(&sb); + } + } + if (aspect_wanted(aspects_doc, aspects_arr, "dependencies") && schema.edge_type_count > 0) { + static const char *const tcols[] = {"type", "count"}; + cbm_toon_table_header(&sb, "edge_types", schema.edge_type_count, tcols, 2); + for (int i = 0; i < schema.edge_type_count; i++) { + cbm_toon_row_begin(&sb); + cbm_toon_cell_str(&sb, schema.edge_types[i].type, true); + cbm_toon_cell_int(&sb, schema.edge_types[i].count, false); + cbm_toon_row_end(&sb); + } + } + if (aspect_wanted(aspects_doc, aspects_arr, "routes") && schema.rel_pattern_count > 0) { + static const char *const pcols[] = {"pattern"}; + cbm_toon_table_header(&sb, "relationship_patterns", schema.rel_pattern_count, pcols, 1); + for (int i = 0; i < schema.rel_pattern_count; i++) { + cbm_toon_row_begin(&sb); + cbm_toon_cell_str(&sb, schema.rel_patterns[i], true); + cbm_toon_row_end(&sb); + } + } + if (arch.language_count > 0) { + static const char *const gcols[] = {"language", "files"}; + cbm_toon_table_header(&sb, "languages", arch.language_count, gcols, 2); + for (int i = 0; i < arch.language_count; i++) { + cbm_toon_row_begin(&sb); + cbm_toon_cell_str(&sb, arch.languages[i].language, true); + cbm_toon_cell_int(&sb, arch.languages[i].file_count, false); + cbm_toon_row_end(&sb); + } + } + if (arch.package_count > 0) { + static const char *const kcols[] = {"name", "nodes", "fan_in", "fan_out"}; + cbm_toon_table_header(&sb, "packages", arch.package_count, kcols, 4); + for (int i = 0; i < arch.package_count; i++) { + cbm_toon_row_begin(&sb); + cbm_toon_cell_str(&sb, arch.packages[i].name, true); + cbm_toon_cell_int(&sb, arch.packages[i].node_count, false); + cbm_toon_cell_int(&sb, arch.packages[i].fan_in, false); + cbm_toon_cell_int(&sb, arch.packages[i].fan_out, false); + cbm_toon_row_end(&sb); + } + } + if (arch.entry_point_count > 0) { + /* qn only — `name` is its last segment. */ + static const char *const ecols[] = {"qn", "file"}; + cbm_toon_table_header(&sb, "entry_points", arch.entry_point_count, ecols, 2); + for (int i = 0; i < arch.entry_point_count; i++) { + cbm_toon_row_begin(&sb); + cbm_toon_cell_str(&sb, arch.entry_points[i].qualified_name, true); + cbm_toon_cell_str(&sb, arch.entry_points[i].file, false); + cbm_toon_row_end(&sb); + } + } + if (arch.route_count > 0) { + static const char *const rcols[] = {"method", "path", "handler"}; + cbm_toon_table_header(&sb, "routes", arch.route_count, rcols, 3); + for (int i = 0; i < arch.route_count; i++) { + cbm_toon_row_begin(&sb); + cbm_toon_cell_str(&sb, arch.routes[i].method, true); + cbm_toon_cell_str(&sb, arch.routes[i].path, false); + cbm_toon_cell_str(&sb, arch.routes[i].handler, false); + cbm_toon_row_end(&sb); + } + } + if (arch.hotspot_count > 0) { + static const char *const hcols[] = {"qn", "fan_in"}; + cbm_toon_table_header(&sb, "hotspots", arch.hotspot_count, hcols, 2); + for (int i = 0; i < arch.hotspot_count; i++) { + cbm_toon_row_begin(&sb); + cbm_toon_cell_str(&sb, arch.hotspots[i].qualified_name, true); + cbm_toon_cell_int(&sb, arch.hotspots[i].fan_in, false); + cbm_toon_row_end(&sb); + } + } + if (arch.boundary_count > 0) { + static const char *const bcols[] = {"from", "to", "calls"}; + cbm_toon_table_header(&sb, "boundaries", arch.boundary_count, bcols, 3); + for (int i = 0; i < arch.boundary_count; i++) { + cbm_toon_row_begin(&sb); + cbm_toon_cell_str(&sb, arch.boundaries[i].from, true); + cbm_toon_cell_str(&sb, arch.boundaries[i].to, false); + cbm_toon_cell_int(&sb, arch.boundaries[i].call_count, false); + cbm_toon_row_end(&sb); + } + } + if (arch.service_count > 0) { + static const char *const scols[] = {"from", "to", "type", "count"}; + cbm_toon_table_header(&sb, "services", arch.service_count, scols, 4); + for (int i = 0; i < arch.service_count; i++) { + cbm_toon_row_begin(&sb); + cbm_toon_cell_str(&sb, arch.services[i].from, true); + cbm_toon_cell_str(&sb, arch.services[i].to, false); + cbm_toon_cell_str(&sb, arch.services[i].type, false); + cbm_toon_cell_int(&sb, arch.services[i].count, false); + cbm_toon_row_end(&sb); + } + } + if (arch.layer_count > 0) { + static const char *const ycols[] = {"name", "layer", "reason"}; + cbm_toon_table_header(&sb, "layers", arch.layer_count, ycols, 3); + for (int i = 0; i < arch.layer_count; i++) { + cbm_toon_row_begin(&sb); + cbm_toon_cell_str(&sb, arch.layers[i].name, true); + cbm_toon_cell_str(&sb, arch.layers[i].layer, false); + cbm_toon_cell_str(&sb, arch.layers[i].reason, false); + cbm_toon_row_end(&sb); + } + } + if (arch.cluster_count > 0) { + /* Nested lists become ';'-joined cells. */ + static const char *const ccols[] = {"id", "label", "members", "cohesion", + "top_nodes", "packages", "edge_types"}; + cbm_toon_table_header(&sb, "clusters", arch.cluster_count, ccols, 7); + for (int i = 0; i < arch.cluster_count; i++) { + const cbm_cluster_info_t *c = &arch.clusters[i]; + char joined[CBM_SZ_1K]; + cbm_toon_row_begin(&sb); + cbm_toon_cell_int(&sb, c->id, true); + cbm_toon_cell_str(&sb, c->label, false); + cbm_toon_cell_int(&sb, c->members, false); + cbm_toon_cell_real(&sb, c->cohesion, false); + arch_join_list(joined, sizeof(joined), c->top_nodes, c->top_node_count); + cbm_toon_cell_str(&sb, joined, false); + arch_join_list(joined, sizeof(joined), c->packages, c->package_count); + cbm_toon_cell_str(&sb, joined, false); + arch_join_list(joined, sizeof(joined), c->edge_types, c->edge_type_count); + cbm_toon_cell_str(&sb, joined, false); + cbm_toon_row_end(&sb); + } + } + if (arch.file_tree_count > 0) { + static const char *const fcols[] = {"path", "type", "children"}; + cbm_toon_table_header(&sb, "file_tree", arch.file_tree_count, fcols, 3); + for (int i = 0; i < arch.file_tree_count; i++) { + cbm_toon_row_begin(&sb); + cbm_toon_cell_str(&sb, arch.file_tree[i].path, true); + cbm_toon_cell_str(&sb, arch.file_tree[i].type, false); + cbm_toon_cell_int(&sb, arch.file_tree[i].children, false); + cbm_toon_row_end(&sb); + } + } + /* Cross-repo edge summary (mirrors append_cross_repo_summary). */ + { + static const char *const cross_types[] = {"CROSS_HTTP_CALLS", "CROSS_ASYNC_CALLS", + "CROSS_CHANNEL", "CROSS_GRPC_CALLS", + "CROSS_GRAPHQL_CALLS", "CROSS_TRPC_CALLS"}; + int cross_total = 0; + for (int t = 0; t < (int)(sizeof(cross_types) / sizeof(cross_types[0])); t++) { + for (int i = 0; i < schema.edge_type_count; i++) { + if (strcmp(schema.edge_types[i].type, cross_types[t]) == 0) { + cross_total += schema.edge_types[i].count; + break; + } + } + } + if (cross_total > 0) { + cbm_toon_scalar_int(&sb, "cross_repo_links_total", cross_total); + } + } + + cbm_store_architecture_free(&arch); + cbm_store_schema_free(&schema); + if (aspects_doc) { + yyjson_doc_free(aspects_doc); + } + free(project); + free(scope_path); + char *text = cbm_sb_finish(&sb); + char *result = cbm_mcp_text_result(text ? text : "out of memory", text == NULL); + free(text); + return result; + } + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); yyjson_mut_val *root = yyjson_mut_obj(doc); yyjson_mut_doc_set_root(doc, root); @@ -2580,6 +3168,12 @@ static char *handle_get_architecture(cbm_mcp_server_t *srv, const char *args) { if (project) { yyjson_mut_obj_add_str(doc, root, "project", project); } + if (default_summary) { + yyjson_mut_obj_add_str(doc, root, "aspects_hint", + "Summary view (default). More on request via aspects:[...] — " + "structure, dependencies, routes, hotspots, boundaries, layers, " + "clusters, file_tree — or [\"all\"] for everything."); + } if (path_scoped) { yyjson_mut_obj_add_str(doc, root, "path", norm_path); int root_nodes = cbm_store_count_nodes(store, project); @@ -2957,6 +3551,61 @@ static yyjson_mut_val *bfs_to_json_array(yyjson_mut_doc *doc, cbm_traverse_resul return arr; } +/* TOON table for one trace direction: callees[N]{qn,hop,...} with optional + * risk / test / args columns. `name` is omitted (it is the qn's last + * segment); the per-item JSON key envelope was 84% of the legacy payload. */ +static void bfs_to_toon_table(cbm_sb_t *sb, const char *key, cbm_traverse_result_t *tr, + bool risk_labels, bool include_tests, bool data_flow) { + int visible = 0; + for (int i = 0; i < tr->visited_count; i++) { + if (!include_tests && is_test_file(tr->visited[i].node.file_path)) { + continue; + } + visible++; + } + const char *cols[5] = {"qn", "hop"}; + int ncols = 2; + if (risk_labels) { + cols[ncols++] = "risk"; + } + if (include_tests) { + cols[ncols++] = "test"; + } + if (data_flow) { + cols[ncols++] = "args"; + } + cbm_toon_table_header(sb, key, visible, cols, ncols); + for (int i = 0; i < tr->visited_count; i++) { + const char *fp = tr->visited[i].node.file_path; + bool test = is_test_file(fp); + if (!include_tests && test) { + continue; + } + cbm_toon_row_begin(sb); + cbm_toon_cell_str(sb, tr->visited[i].node.qualified_name, true); + cbm_toon_cell_int(sb, tr->visited[i].hop, false); + if (risk_labels) { + cbm_toon_cell_str(sb, cbm_risk_label(cbm_hop_to_risk(tr->visited[i].hop)), false); + } + if (include_tests) { + cbm_toon_cell_bool(sb, test, false); + } + if (data_flow) { + size_t alen = 0; + const char *ea = bfs_edge_args_for_hop(tr, tr->visited[i].node.id, &alen); + if (ea && alen > 0 && alen < CBM_SZ_1K) { + char abuf[CBM_SZ_1K]; + memcpy(abuf, ea, alen); + abuf[alen] = '\0'; + cbm_toon_cell_str(sb, abuf, false); + } else { + cbm_toon_cell_str(sb, "", false); + } + } + cbm_toon_row_end(sb); + } +} + static char *snippet_suggestions(const char *input, cbm_node_t *nodes, int count); /* Rank a candidate for name resolution. The label tier (callable > class-like > @@ -3211,15 +3860,11 @@ static char *handle_trace_call_path(cbm_mcp_server_t *srv, const char *args) { return result; } - yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); - yyjson_mut_val *root = yyjson_mut_obj(doc); - yyjson_mut_doc_set_root(doc, root); - - yyjson_mut_obj_add_str(doc, root, "function", func_name); - yyjson_mut_obj_add_str(doc, root, "direction", direction); - if (mode) { - yyjson_mut_obj_add_str(doc, root, "mode", mode); - } + /* Response encoding: TOON tables by default; format:"json" restores the + * legacy verbose per-hop objects. */ + char *trace_format = cbm_mcp_get_string_arg(args, "format"); + bool trace_legacy_json = trace_format && strcmp(trace_format, "json") == 0; + free(trace_format); /* Edge types: explicit > mode-based > default */ const char *edge_types[MCP_COL_16]; @@ -3227,8 +3872,8 @@ static char *handle_trace_call_path(cbm_mcp_server_t *srv, const char *args) { yyjson_doc *et_doc_keep = resolve_trace_edge_types(args, mode, edge_types, &edge_type_count); /* Run BFS for each requested direction. - * IMPORTANT: yyjson_mut_obj_add_str borrows pointers — we must keep - * traversal results alive until after yy_doc_to_str serialization. */ + * IMPORTANT: emitters borrow node-string pointers — traversal results + * must stay alive until after serialization. */ bool do_outbound = strcmp(direction, "outbound") == 0 || strcmp(direction, "both") == 0; bool do_inbound = strcmp(direction, "inbound") == 0 || strcmp(direction, "both") == 0; @@ -3242,22 +3887,52 @@ static char *handle_trace_call_path(cbm_mcp_server_t *srv, const char *args) { if (do_outbound) { bfs_union_same_name(store, nodes, node_count, "outbound", edge_types, edge_type_count, depth, &tr_out); - yyjson_mut_obj_add_val( - doc, root, "callees", - bfs_to_json_array(doc, &tr_out, risk_labels, include_tests, data_flow)); } - if (do_inbound) { bfs_union_same_name(store, nodes, node_count, "inbound", edge_types, edge_type_count, depth, &tr_in); - yyjson_mut_obj_add_val( - doc, root, "callers", - bfs_to_json_array(doc, &tr_in, risk_labels, include_tests, data_flow)); } - /* Serialize BEFORE freeing traversal results (yyjson borrows strings) */ - char *json = yy_doc_to_str(doc); - yyjson_mut_doc_free(doc); + char *json = NULL; + if (!trace_legacy_json) { + cbm_sb_t sb; + cbm_sb_init(&sb); + cbm_toon_scalar_str(&sb, "function", func_name); + cbm_toon_scalar_str(&sb, "direction", direction); + if (mode) { + cbm_toon_scalar_str(&sb, "mode", mode); + } + if (do_outbound) { + bfs_to_toon_table(&sb, "callees", &tr_out, risk_labels, include_tests, data_flow); + } + if (do_inbound) { + bfs_to_toon_table(&sb, "callers", &tr_in, risk_labels, include_tests, data_flow); + } + json = cbm_sb_finish(&sb); + } else { + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = yyjson_mut_obj(doc); + yyjson_mut_doc_set_root(doc, root); + + yyjson_mut_obj_add_str(doc, root, "function", func_name); + yyjson_mut_obj_add_str(doc, root, "direction", direction); + if (mode) { + yyjson_mut_obj_add_str(doc, root, "mode", mode); + } + if (do_outbound) { + yyjson_mut_obj_add_val( + doc, root, "callees", + bfs_to_json_array(doc, &tr_out, risk_labels, include_tests, data_flow)); + } + if (do_inbound) { + yyjson_mut_obj_add_val( + doc, root, "callers", + bfs_to_json_array(doc, &tr_in, risk_labels, include_tests, data_flow)); + } + /* Serialize BEFORE freeing traversal results (yyjson borrows strings) */ + json = yy_doc_to_str(doc); + yyjson_mut_doc_free(doc); + } /* Now safe to free traversal data */ if (do_outbound) { @@ -4588,8 +5263,11 @@ static char *build_snippet_response(cbm_mcp_server_t *srv, cbm_node_t *node, yyjson_mut_obj_add_str(doc, root_obj, "match_method", match_method); } - /* Enrich with node properties (freed AFTER serialization — zero-copy). */ - yyjson_doc *props_doc = enrich_node_properties(doc, root_obj, node->properties_json); + /* No property-blob enrichment: the verbatim source IS the payload here — + * signature/docstring are literally in it, and the similarity internals + * (fp/sp/bt) plus metric fields were 41% of the response for zero agent + * value. Metrics stay reachable via search_graph fields=[...]. */ + yyjson_doc *props_doc = NULL; /* Caller/callee counts — store already resolved by calling handler */ cbm_store_t *store = srv->store; @@ -4930,11 +5608,33 @@ static void attach_result_source(yyjson_mut_doc *doc, yyjson_mut_val *item, sear } if (mode == MODE_FULL) { - char *source = read_file_lines(abs_path, r->start_line, r->end_line); + /* Cap each hit's source at a match-anchored window: uncapped + * whole-symbol dumps ran to 5.7KB × N hits (142KB responses). The + * complete symbol stays one get_code_snippet call away; + * source_start/source_truncated make the cut explicit. */ + enum { SC_FULL_MAX_LINES = 60, SC_FULL_LEAD = 5 }; + int s = r->start_line; + int e = r->end_line; + bool truncated = false; + if (e - s + 1 > SC_FULL_MAX_LINES) { + if (r->match_count > 0 && r->match_lines[0] - SC_FULL_LEAD > s) { + s = r->match_lines[0] - SC_FULL_LEAD; + } + e = s + SC_FULL_MAX_LINES - 1; + if (e > r->end_line) { + e = r->end_line; + } + truncated = true; + } + char *source = read_file_lines(abs_path, s, e); if (source) { sanitize_ascii(source); yyjson_mut_obj_add_strcpy(doc, item, "source", source); free(source); + if (truncated) { + yyjson_mut_obj_add_int(doc, item, "source_start", s); + yyjson_mut_obj_add_bool(doc, item, "source_truncated", true); + } } } else if (context_lines > 0 && r->match_count > 0) { int ctx_start = r->match_lines[0] - context_lines; @@ -4953,11 +5653,10 @@ static void attach_result_source(yyjson_mut_doc *doc, yyjson_mut_val *item, sear } /* Build directory distribution object from search results (top-level dir → count). */ -static yyjson_mut_val *build_dir_distribution(yyjson_mut_doc *doc, search_result_t *sr, - int sr_count) { - yyjson_mut_val *dirs = yyjson_mut_obj(doc); - char dir_names[CBM_SZ_64][CBM_SZ_128]; - int dir_counts[CBM_SZ_64]; +/* Aggregate hits by top-level directory. Shared by the JSON object and the + * TOON table emission. Returns the number of distinct directories. */ +static int aggregate_search_dirs(search_result_t *sr, int sr_count, char dir_names[][CBM_SZ_128], + int *dir_counts, int max_dirs) { int dir_n = 0; for (int di = 0; di < sr_count; di++) { char top[CBM_SZ_128] = ""; @@ -4981,12 +5680,21 @@ static yyjson_mut_val *build_dir_distribution(yyjson_mut_doc *doc, search_result } if (found >= 0) { dir_counts[found]++; - } else if (dir_n < CBM_SZ_64) { - snprintf(dir_names[dir_n], sizeof(dir_names[0]), "%s", top); + } else if (dir_n < max_dirs) { + snprintf(dir_names[dir_n], CBM_SZ_128, "%s", top); dir_counts[dir_n] = SKIP_ONE; dir_n++; } } + return dir_n; +} + +static yyjson_mut_val *build_dir_distribution(yyjson_mut_doc *doc, search_result_t *sr, + int sr_count) { + yyjson_mut_val *dirs = yyjson_mut_obj(doc); + char dir_names[CBM_SZ_64][CBM_SZ_128]; + int dir_counts[CBM_SZ_64]; + int dir_n = aggregate_search_dirs(sr, sr_count, dir_names, dir_counts, CBM_SZ_64); for (int d = 0; d < dir_n; d++) { yyjson_mut_val *key = yyjson_mut_strcpy(doc, dir_names[d]); yyjson_mut_val *val = yyjson_mut_int(doc, dir_counts[d]); @@ -4995,6 +5703,97 @@ static yyjson_mut_val *build_dir_distribution(yyjson_mut_doc *doc, search_result return dirs; } +/* TOON emission for compact-mode search results: one row per hit + * (qn/label/file/lines/matches/degrees — `node` dropped, it duplicates the + * qn's last segment), a raw[] table for uncorrelated matches, a dirs[] + * distribution table, and the summary scalars. */ +static char *assemble_search_output_toon(search_result_t *sr, int sr_count, grep_match_t *raw, + int raw_count, int gm_count, int limit, + bool warn_literal_pipe, uint64_t elapsed_ms) { + enum { MAX_RAW = 20, SEARCH_SLOW_MS = 5000 }; + cbm_sb_t sb; + cbm_sb_init(&sb); + + int output_count = sr_count < limit ? sr_count : limit; + static const char *const cols[] = {"qn", "label", "file", "lines", "matches", "in", "out"}; + cbm_toon_table_header(&sb, "results", output_count, cols, 7); + for (int ri = 0; ri < output_count; ri++) { + search_result_t *r = &sr[ri]; + char lines[CBM_SZ_32]; + if (r->start_line > 0) { + snprintf(lines, sizeof(lines), "%d-%d", r->start_line, + r->end_line > r->start_line ? r->end_line : r->start_line); + } else { + lines[0] = '\0'; + } + /* match line numbers ';'-joined (no comma → no cell quoting) */ + char matches[CBM_SZ_256]; + size_t mpos = 0; + matches[0] = '\0'; + for (int j = 0; j < r->match_count && mpos + 12 < sizeof(matches); j++) { + int n = snprintf(matches + mpos, sizeof(matches) - mpos, "%s%d", j > 0 ? ";" : "", + r->match_lines[j]); + if (n < 0) { + break; + } + mpos += (size_t)n; + } + cbm_toon_row_begin(&sb); + cbm_toon_cell_str(&sb, r->qualified_name, true); + cbm_toon_cell_str(&sb, r->label, false); + cbm_toon_cell_str(&sb, r->file, false); + cbm_toon_cell_str(&sb, lines, false); + cbm_toon_cell_str(&sb, matches, false); + cbm_toon_cell_int(&sb, r->in_degree, false); + cbm_toon_cell_int(&sb, r->out_degree, false); + cbm_toon_row_end(&sb); + } + + int raw_output = raw_count < MAX_RAW ? raw_count : MAX_RAW; + if (raw_output > 0) { + static const char *const rcols[] = {"file", "line", "content"}; + cbm_toon_table_header(&sb, "raw", raw_output, rcols, 3); + for (int ri = 0; ri < raw_output; ri++) { + cbm_toon_row_begin(&sb); + cbm_toon_cell_str(&sb, raw[ri].file, true); + cbm_toon_cell_int(&sb, raw[ri].line, false); + cbm_toon_cell_str(&sb, raw[ri].content, false); + cbm_toon_row_end(&sb); + } + } + + char dir_names[CBM_SZ_64][CBM_SZ_128]; + int dir_counts[CBM_SZ_64]; + int dir_n = aggregate_search_dirs(sr, sr_count, dir_names, dir_counts, CBM_SZ_64); + if (dir_n > 0) { + static const char *const dcols[] = {"dir", "hits"}; + cbm_toon_table_header(&sb, "dirs", dir_n, dcols, 2); + for (int d = 0; d < dir_n; d++) { + cbm_toon_row_begin(&sb); + cbm_toon_cell_str(&sb, dir_names[d], true); + cbm_toon_cell_int(&sb, dir_counts[d], false); + cbm_toon_row_end(&sb); + } + } + + cbm_toon_scalar_int(&sb, "total_grep_matches", gm_count); + cbm_toon_scalar_int(&sb, "total_results", sr_count); + cbm_toon_scalar_int(&sb, "raw_match_count", raw_count); + cbm_toon_scalar_int(&sb, "elapsed_ms", (long long)elapsed_ms); + if (warn_literal_pipe) { + cbm_toon_scalar_str(&sb, "warning", + "pattern contains '|' but regex=false, so it is matched literally " + "(not as alternation). Pass regex=true for 'foo|bar' to mean " + "'foo OR bar'."); + } + if (elapsed_ms >= SEARCH_SLOW_MS) { + cbm_toon_scalar_str(&sb, "warning_slow", + "search was slow; narrow file_pattern/path_filter or use a more " + "specific pattern"); + } + return cbm_sb_finish(&sb); +} + /* Phase 4: assemble JSON output from search results */ static char *assemble_search_output(search_result_t *sr, int sr_count, grep_match_t *raw, int raw_count, int gm_count, int limit, int mode, @@ -5655,9 +6454,25 @@ static char *handle_search_code(cbm_mcp_server_t *srv, const char *args) { /* ── Phase 4: Context assembly (extracted helper) ─────────── */ - char *result = - assemble_search_output(sr, sr_count, raw, raw_count, gm_count, limit, mode, context_lines, - root_path, pat_has_pipe && !use_regex, cbm_now_ms() - search_t0); + /* compact mode (default) emits TOON tables; format:"json" restores the + * legacy per-hit objects. full/files keep their JSON shapes (full is + * source-block-heavy; files is already minimal). */ + char *sc_format = cbm_mcp_get_string_arg(args, "format"); + bool sc_legacy_json = sc_format && strcmp(sc_format, "json") == 0; + free(sc_format); + + char *result = NULL; + if (mode == 0 && !sc_legacy_json) { + char *toon_text = + assemble_search_output_toon(sr, sr_count, raw, raw_count, gm_count, limit, + pat_has_pipe && !use_regex, cbm_now_ms() - search_t0); + result = cbm_mcp_text_result(toon_text ? toon_text : "out of memory", toon_text == NULL); + free(toon_text); + } else { + result = assemble_search_output(sr, sr_count, raw, raw_count, gm_count, limit, mode, + context_lines, root_path, pat_has_pipe && !use_regex, + cbm_now_ms() - search_t0); + } free(gm); free(sr); free(raw); diff --git a/tests/test_incremental.c b/tests/test_incremental.c index 7883a9cc9..ae425b936 100644 --- a/tests/test_incremental.c +++ b/tests/test_incremental.c @@ -127,7 +127,9 @@ static char *call_tool(const char *tool, const char *args_fmt, ...) { return cbm_mcp_handle_tool(g_srv, tool, args); } -/* Parse integer from JSON response (handles nested MCP envelope) */ +/* Parse integer from a tool response (handles the nested MCP envelope). + * Matches the JSON forms "key":N / \"key\":N and the TOON scalar `key: N` + * (search_graph default output since the compact-output change). */ static int count_in_response(const char *resp, const char *key) { if (!resp) return -1; @@ -138,6 +140,10 @@ static int count_in_response(const char *resp, const char *key) { return atoi(p + strlen(pattern)); snprintf(pattern, sizeof(pattern), "\\\"%s\\\":", key); p = strstr(resp, pattern); + if (p) + return atoi(p + strlen(pattern)); + snprintf(pattern, sizeof(pattern), "%s: ", key); + p = strstr(resp, pattern); if (p) return atoi(p + strlen(pattern)); return -1; @@ -928,6 +934,14 @@ static int resp_has_key(const char *resp, const char *key) { if (strstr(resp, pattern) != NULL) return 1; snprintf(pattern, sizeof(pattern), "\\\"%s\\\"", key); + if (strstr(resp, pattern) != NULL) + return 1; + /* TOON scalar form (`key: value`), the search_graph default output. */ + snprintf(pattern, sizeof(pattern), "%s: ", key); + if (strstr(resp, pattern) != NULL) + return 1; + /* TOON table-header form (`key[N]{...}:`), used by trace_path. */ + snprintf(pattern, sizeof(pattern), "%s[", key); return strstr(resp, pattern) != NULL; } diff --git a/tests/test_mcp.c b/tests/test_mcp.c index c072088c0..9c3cadfda 100644 --- a/tests/test_mcp.c +++ b/tests/test_mcp.c @@ -739,15 +739,16 @@ static void cleanup_snippet_dir(const char *tmp_dir); static char *extract_text_content(const char *mcp_result); TEST(tool_search_graph_includes_node_properties) { - /* search_graph results must surface each node's properties_json - * payload so callers don't have to round-trip through get_code_snippet - * just to read them. The setup_snippet_server inserts HandleRequest - * with a signature/return_type/is_exported property blob; this test - * pins that those keys reach the MCP response. */ + /* Node properties are OPT-IN columns in the default TOON output: the + * default row is qn/label/file/lines/degrees only, `fields` adds the + * requested property columns, and format:"json" restores the legacy + * verbose objects with the full property blob. The setup_snippet_server + * inserts HandleRequest with a signature/return_type/is_exported blob. */ char tmp[256]; cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp)); ASSERT_NOT_NULL(srv); + /* Default TOON: compact table, no property spill. */ char *resp = cbm_mcp_server_handle( srv, "{\"jsonrpc\":\"2.0\",\"id\":42,\"method\":\"tools/call\"," "\"params\":{\"name\":\"search_graph\"," @@ -756,8 +757,38 @@ TEST(tool_search_graph_includes_node_properties) { ASSERT_NOT_NULL(resp); char *inner = extract_text_content(resp); ASSERT_NOT_NULL(inner); - /* Properties from HandleRequest's properties_json must appear. */ - ASSERT_NOT_NULL(strstr(inner, "signature")); + ASSERT_NOT_NULL(strstr(inner, "results[")); /* TOON table header */ + ASSERT_NOT_NULL(strstr(inner, "{qn,label,file,lines,in,out}")); + ASSERT_NOT_NULL(strstr(inner, "HandleRequest")); + ASSERT_NULL(strstr(inner, "func HandleRequest")); /* signature not spilled */ + ASSERT_NULL(strstr(inner, "is_exported")); + free(inner); + free(resp); + + /* fields:["signature"] adds the column + values. */ + resp = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":43,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"search_graph\"," + "\"arguments\":{\"project\":\"test-project\",\"label\":\"Function\"," + "\"name_pattern\":\"HandleRequest\",\"fields\":[\"signature\"],\"limit\":5}}}"); + ASSERT_NOT_NULL(resp); + inner = extract_text_content(resp); + ASSERT_NOT_NULL(inner); + ASSERT_NOT_NULL(strstr(inner, "{qn,label,file,lines,in,out,signature}")); + ASSERT_NOT_NULL(strstr(inner, "func HandleRequest")); + free(inner); + free(resp); + + /* format:"json" keeps the legacy verbose objects intact. */ + resp = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":44,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"search_graph\"," + "\"arguments\":{\"project\":\"test-project\",\"label\":\"Function\"," + "\"name_pattern\":\"HandleRequest\",\"format\":\"json\",\"limit\":5}}}"); + ASSERT_NOT_NULL(resp); + inner = extract_text_content(resp); + ASSERT_NOT_NULL(inner); + ASSERT_NOT_NULL(strstr(inner, "\"signature\"")); ASSERT_NOT_NULL(strstr(inner, "func HandleRequest")); ASSERT_NOT_NULL(strstr(inner, "is_exported")); free(inner); @@ -768,6 +799,96 @@ TEST(tool_search_graph_includes_node_properties) { PASS(); } +TEST(tool_output_byte_budgets) { + /* GUARD: absolute byte ceilings on default tool outputs. Re-bloat (e.g. + * a property blob sneaking back into row emission — the fp field alone + * is ~450B/hit) blows these ceilings immediately. The numbers are + * generous vs the measured compact outputs (search hit rows ≈ 90B) but + * far below the legacy verbose sizes (≈1.5KB/hit). */ + char tmp[256]; + cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp)); + ASSERT_NOT_NULL(srv); + + /* search_graph: 1-hit search must stay under 600B. */ + char *resp = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":46,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"search_graph\"," + "\"arguments\":{\"project\":\"test-project\",\"label\":\"Function\"," + "\"name_pattern\":\"HandleRequest\",\"limit\":5}}}"); + ASSERT_NOT_NULL(resp); + char *inner = extract_text_content(resp); + ASSERT_NOT_NULL(inner); + ASSERT_NOT_NULL(strstr(inner, "HandleRequest")); /* non-vacuous: hit present */ + ASSERT_LT((int)strlen(inner), 600); + free(inner); + free(resp); + + /* trace_path: single-hop trace on the fixture must stay under 800B. */ + resp = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":47,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"trace_call_path\"," + "\"arguments\":{\"project\":\"test-project\",\"function_name\":\"HandleRequest\"," + "\"direction\":\"both\",\"depth\":2}}}"); + ASSERT_NOT_NULL(resp); + inner = extract_text_content(resp); + ASSERT_NOT_NULL(inner); + ASSERT_NOT_NULL(strstr(inner, "callees[")); + ASSERT_LT((int)strlen(inner), 800); + free(inner); + free(resp); + + cbm_mcp_server_free(srv); + cleanup_snippet_dir(tmp); + PASS(); +} + +TEST(tool_search_graph_toon_never_leaks_internal_fields) { + /* The similarity/semantic pipeline intermediates (fp minhash hex, sp + * structural profile, bt body-token bag) dominated the legacy payload + * (~45%) and carry zero agent value. GUARD: they never appear in TOON + * output — not by default and not even when explicitly requested via + * fields (blocklist). */ + char tmp[256]; + cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp)); + ASSERT_NOT_NULL(srv); + cbm_store_t *st = cbm_mcp_server_store(srv); + ASSERT_NOT_NULL(st); + + /* A node whose properties carry the internal fields with sentinels. */ + cbm_node_t n = {0}; + n.project = "test-project"; + n.label = "Function"; + n.name = "fpCarrier"; + n.qualified_name = "test-project.src.fpCarrier"; + n.file_path = "src/fp.go"; + n.start_line = 1; + n.end_line = 2; + n.properties_json = "{\"fp\":\"FPSENTINEL00\",\"sp\":\"SPSENTINEL00\"," + "\"bt\":\"BTSENTINEL00\",\"complexity\":7}"; + ASSERT_GT(cbm_store_upsert_node(st, &n), 0); + + char *resp = cbm_mcp_server_handle( + srv, "{\"jsonrpc\":\"2.0\",\"id\":45,\"method\":\"tools/call\"," + "\"params\":{\"name\":\"search_graph\"," + "\"arguments\":{\"project\":\"test-project\",\"name_pattern\":\"fpCarrier\"," + "\"fields\":[\"fp\",\"sp\",\"bt\",\"complexity\"],\"limit\":5}}}"); + ASSERT_NOT_NULL(resp); + char *inner = extract_text_content(resp); + ASSERT_NOT_NULL(inner); + ASSERT_NOT_NULL(strstr(inner, "fpCarrier")); + ASSERT_NULL(strstr(inner, "FPSENTINEL00")); + ASSERT_NULL(strstr(inner, "SPSENTINEL00")); + ASSERT_NULL(strstr(inner, "BTSENTINEL00")); + /* Non-blocked requested field still comes through. */ + ASSERT_NOT_NULL(strstr(inner, "complexity")); + free(inner); + free(resp); + + cbm_mcp_server_free(srv); + cleanup_snippet_dir(tmp); + PASS(); +} + TEST(tool_search_graph_query_honors_file_pattern_issue552) { cbm_mcp_server_t *srv = cbm_mcp_server_new(NULL); ASSERT_NOT_NULL(srv); @@ -814,8 +935,8 @@ TEST(tool_search_graph_query_honors_file_pattern_issue552) { ASSERT_NOT_NULL(resp); char *inner = extract_text_content(resp); ASSERT_NOT_NULL(inner); - ASSERT_NOT_NULL(strstr(inner, "\"search_mode\":\"bm25\"")); - ASSERT_NOT_NULL(strstr(inner, "\"file_path\":\"src/lib/status.c\"")); + ASSERT_NOT_NULL(strstr(inner, "search_mode: bm25")); + ASSERT_NOT_NULL(strstr(inner, "src/lib/status.c")); ASSERT_NULL(strstr(inner, "src/components/status.c")); free(inner); @@ -1057,9 +1178,10 @@ TEST(tool_trace_call_path_depth_clamped) { char *inner = extract_text_content(resp); ASSERT_NOT_NULL(inner); - /* Reached within the ceiling (proves the traversal ran) but clamped at 15. */ - ASSERT_NOT_NULL(strstr(inner, "\"n15\"")); - ASSERT_NULL(strstr(inner, "\"n16\"")); + /* Reached within the ceiling (proves the traversal ran) but clamped at 15. + * TOON rows carry bare QNs, so match the names unquoted. */ + ASSERT_NOT_NULL(strstr(inner, "n15")); + ASSERT_NULL(strstr(inner, "n16")); free(inner); free(resp); @@ -1281,7 +1403,7 @@ TEST(tool_get_architecture_emits_populated_sections) { * those existed before #281. The "entry_points" array only appears * when cbm_store_get_architecture is actually called and its result * is serialized — which is exactly what #281 wires up. */ - ASSERT_NOT_NULL(strstr(inner, "\"entry_points\"")); + ASSERT_NOT_NULL(strstr(inner, "entry_points[")); ASSERT_NOT_NULL(strstr(inner, "main")); free(inner); @@ -1334,8 +1456,8 @@ TEST(tool_get_architecture_overview_compact_subset_pr560) { ASSERT_NOT_NULL(resp_all); char *inner_all = extract_text_content(resp_all); ASSERT_NOT_NULL(inner_all); - ASSERT_NOT_NULL(strstr(inner_all, "\"entry_points\"")); - ASSERT_NOT_NULL(strstr(inner_all, "\"file_tree\"")); + ASSERT_NOT_NULL(strstr(inner_all, "entry_points[")); + ASSERT_NOT_NULL(strstr(inner_all, "file_tree[")); free(inner_all); free(resp_all); @@ -1348,9 +1470,9 @@ TEST(tool_get_architecture_overview_compact_subset_pr560) { ASSERT_NOT_NULL(resp); char *inner = extract_text_content(resp); ASSERT_NOT_NULL(inner); - ASSERT_NOT_NULL(strstr(inner, "\"entry_points\"")); - ASSERT_NOT_NULL(strstr(inner, "\"node_labels\"")); - ASSERT_NULL(strstr(inner, "\"file_tree\"")); + ASSERT_NOT_NULL(strstr(inner, "entry_points[")); + ASSERT_NOT_NULL(strstr(inner, "node_labels[")); + ASSERT_NULL(strstr(inner, "file_tree[")); free(inner); free(resp); @@ -1431,7 +1553,7 @@ TEST(tool_get_architecture_accepts_project_name_alias_issue640) { /* RED before the alias: inner is the "project not found" error. * GREEN after: the alias resolves and architecture sections surface. */ ASSERT_NULL(strstr(inner, "project not found")); - ASSERT_NOT_NULL(strstr(inner, "\"entry_points\"")); + ASSERT_NOT_NULL(strstr(inner, "entry_points[")); free(inner); free(resp); @@ -1539,18 +1661,23 @@ TEST(tool_get_architecture_path_scoping) { ASSERT_NOT_NULL(strstr(inner_scoped, "root_total_nodes")); ASSERT_NOT_NULL(strstr(inner_scoped, "scoped_total_nodes")); - ASSERT_NOT_NULL(strstr(inner_scoped, "\"path\"")); + ASSERT_NOT_NULL(strstr(inner_scoped, "path: ")); ASSERT_NOT_NULL(strstr(inner_scoped, "hoa")); ASSERT_NULL(strstr(inner_scoped, "Django")); int root_nodes = 0; int scoped_nodes = 0; - const char *rt = strstr(inner_scoped, "\"root_total_nodes\":"); - const char *stn = strstr(inner_scoped, "\"scoped_total_nodes\":"); + /* TOON scalar form (`key: N`) with JSON fallback for format:"json". */ + const char *rt = strstr(inner_scoped, "root_total_nodes: "); + const char *stn = strstr(inner_scoped, "scoped_total_nodes: "); if (rt) { + sscanf(rt, "root_total_nodes: %d", &root_nodes); + } else if ((rt = strstr(inner_scoped, "\"root_total_nodes\":")) != NULL) { sscanf(rt, "\"root_total_nodes\":%d", &root_nodes); } if (stn) { + sscanf(stn, "scoped_total_nodes: %d", &scoped_nodes); + } else if ((stn = strstr(inner_scoped, "\"scoped_total_nodes\":")) != NULL) { sscanf(stn, "\"scoped_total_nodes\":%d", &scoped_nodes); } ASSERT_TRUE(root_nodes > scoped_nodes); @@ -1753,6 +1880,9 @@ TEST(search_code_scoped_path_with_spaces_issue687) { const char *g = strstr(inner, "\"total_grep_matches\":"); if (g) { sscanf(g, "\"total_grep_matches\":%d", &grep_matches); + } else if ((g = strstr(inner, "total_grep_matches: ")) != NULL) { + /* TOON scalar form — the search_code compact default. */ + sscanf(g, "total_grep_matches: %d", &grep_matches); } ASSERT_TRUE(grep_matches > 0); @@ -1826,6 +1956,9 @@ TEST(search_code_scoped_path_with_cjk_root_issue903) { const char *g = strstr(inner, "\"total_grep_matches\":"); if (g) { sscanf(g, "\"total_grep_matches\":%d", &grep_matches); + } else if ((g = strstr(inner, "total_grep_matches: ")) != NULL) { + /* TOON scalar form — the search_code compact default. */ + sscanf(g, "total_grep_matches: %d", &grep_matches); } ASSERT_TRUE(grep_matches > 0); @@ -1944,6 +2077,9 @@ TEST(search_code_path_filter_prefilter_keeps_matches) { const char *g = strstr(inner, "\"total_grep_matches\":"); if (g) { sscanf(g, "\"total_grep_matches\":%d", &grep_matches); + } else if ((g = strstr(inner, "total_grep_matches: ")) != NULL) { + /* TOON scalar form — the search_code compact default. */ + sscanf(g, "total_grep_matches: %d", &grep_matches); } ASSERT_EQ(grep_matches, 1); @@ -1982,12 +2118,17 @@ TEST(search_code_path_filter_matches_nothing) { const char *g = strstr(inner, "\"total_grep_matches\":"); if (g) { sscanf(g, "\"total_grep_matches\":%d", &grep_matches); + } else if ((g = strstr(inner, "total_grep_matches: ")) != NULL) { + /* TOON scalar form — the search_code compact default. */ + sscanf(g, "total_grep_matches: %d", &grep_matches); } ASSERT_EQ(grep_matches, 0); int results = -1; const char *r = strstr(inner, "\"total_results\":"); if (r) { sscanf(r, "\"total_results\":%d", &results); + } else if ((r = strstr(inner, "total_results: ")) != NULL) { + sscanf(r, "total_results: %d", &results); } ASSERT_EQ(results, 0); ASSERT_TRUE(strstr(inner, "handler.go") == NULL); @@ -2046,7 +2187,7 @@ TEST(search_code_literal_pipe_warns_issue282) { "\"arguments\":{\"pattern\":\"HandleRequest|Nope\"," "\"regex\":false,\"project\":\"test-project\"}}}"); ASSERT_NOT_NULL(resp); - ASSERT_NOT_NULL(strstr(resp, "warnings")); /* surfaced, not silent */ + ASSERT_NOT_NULL(strstr(resp, "warning")); /* surfaced, not silent */ ASSERT_NOT_NULL(strstr(resp, "regex=true")); /* the hint names the fix */ ASSERT_NOT_NULL(strstr(resp, "elapsed_ms")); /* timing is reported */ free(resp); @@ -2938,9 +3079,11 @@ TEST(snippet_exact_qn) { ASSERT_NOT_NULL(strstr(resp, "\"source\"")); /* Exact match should NOT have match_method */ ASSERT_NULL(strstr(resp, "\"match_method\"")); - /* Enriched properties */ - ASSERT_NOT_NULL(strstr(resp, "\"signature\":\"func HandleRequest() error\"")); - ASSERT_NOT_NULL(strstr(resp, "\"return_type\":\"error\"")); + /* No property-blob spill: the source IS the payload (signature and + * docstring are literally in it); metrics live behind search_graph + * fields=[...]. */ + ASSERT_NULL(strstr(resp, "\"signature\"")); + ASSERT_NULL(strstr(resp, "\"return_type\"")); /* Caller/callee counts: 0 callers, 2 callees */ ASSERT_NOT_NULL(strstr(resp, "\"callers\":0")); ASSERT_NOT_NULL(strstr(resp, "\"callees\":2")); @@ -3082,6 +3225,11 @@ TEST(snippet_fuzzy_suggestions) { /* ── TestSnippet_EnrichedProperties ───────────────────────────── */ TEST(snippet_enriched_properties) { + /* GUARD (inverted since the compact-output change): the snippet response + * carries the verbatim source plus location/degree/coverage metadata and + * NOTHING from the node's property blob — no signature/return_type/ + * is_exported duplication, and never the fp/sp/bt similarity internals + * (41% of the legacy response). */ char tmp[256]; cbm_mcp_server_t *srv = setup_snippet_server(tmp, sizeof(tmp)); ASSERT_NOT_NULL(srv); @@ -3090,9 +3238,12 @@ TEST(snippet_enriched_properties) { call_snippet(srv, "{\"qualified_name\":\"test-project.cmd.server.main.HandleRequest\"," "\"project\":\"test-project\"}"); ASSERT_NOT_NULL(resp); - ASSERT_NOT_NULL(strstr(resp, "\"signature\"")); - ASSERT_NOT_NULL(strstr(resp, "\"return_type\"")); - ASSERT_NOT_NULL(strstr(resp, "\"is_exported\":true")); + ASSERT_NOT_NULL(strstr(resp, "\"source\"")); + ASSERT_NULL(strstr(resp, "\"signature\"")); + ASSERT_NULL(strstr(resp, "\"return_type\"")); + ASSERT_NULL(strstr(resp, "\"is_exported\"")); + ASSERT_NULL(strstr(resp, "\"fp\"")); + ASSERT_NULL(strstr(resp, "\"bt\"")); free(resp); cbm_mcp_server_free(srv); @@ -5263,6 +5414,8 @@ SUITE(mcp) { RUN_TEST(tool_unknown_tool); RUN_TEST(tool_search_graph_basic); RUN_TEST(tool_search_graph_includes_node_properties); + RUN_TEST(tool_search_graph_toon_never_leaks_internal_fields); + RUN_TEST(tool_output_byte_budgets); RUN_TEST(tool_search_graph_query_honors_file_pattern_issue552); RUN_TEST(tool_query_graph_basic); RUN_TEST(tool_index_status_no_project); diff --git a/tests/windows/test_non_ascii_path.py b/tests/windows/test_non_ascii_path.py index 896209f80..4635a3665 100644 --- a/tests/windows/test_non_ascii_path.py +++ b/tests/windows/test_non_ascii_path.py @@ -88,12 +88,14 @@ def index_and_count(binary, repo, cache): out = {"name": p.get("name"), "nodes": p.get("nodes"), "edges": p.get("edges")} # Definition-level counts prove the parser ran (not just discovery). - # query_graph returns {"columns":[...],"rows":[[""]],...}. + # query_graph defaults to TOON text; this scripted consumer requests + # format="json" ({"columns":[...],"rows":[[""]],...}) explicitly. name = p.get("name") defs = 0 for label in ("Function", "Class", "Method"): q = "MATCH (n:%s) RETURN count(n)" % label - r = s.call_tool("query_graph", {"query": q, "project": name}, + r = s.call_tool("query_graph", + {"query": q, "project": name, "format": "json"}, timeout=60) t, _ = s.tool_text(r) try: