From d67cc71185b71850880876f3f34d8f6528bd0e21 Mon Sep 17 00:00:00 2001 From: Arjun Kumar Date: Thu, 28 May 2026 23:08:50 +0800 Subject: [PATCH 1/2] feat: slash commands and goal halt-gate --- examples/goal-test/cli.tsx | 158 ++------- src/core/agent-ctx.ts | 53 ++- src/core/agent.ts | 71 +++- src/core/halt-gate.ts | 143 ++++++++ src/core/types.ts | 10 +- src/index.ts | 1 + src/jsx/components/goal.tsx | 83 +++++ src/jsx/components/index.ts | 1 + src/jsx/components/skills.tsx | 165 +++++++++- src/jsx/render.ts | 20 +- src/jsx/runtime.ts | 40 ++- test/agentctx/core/agent-run-contract.test.ts | 306 ++++++++++++++++++ test/agentctx/core/goal-component.test.ts | 33 ++ 13 files changed, 932 insertions(+), 152 deletions(-) create mode 100644 src/core/halt-gate.ts create mode 100644 src/jsx/components/goal.tsx create mode 100644 test/agentctx/core/agent-run-contract.test.ts create mode 100644 test/agentctx/core/goal-component.test.ts diff --git a/examples/goal-test/cli.tsx b/examples/goal-test/cli.tsx index 73146eb..a177701 100644 --- a/examples/goal-test/cli.tsx +++ b/examples/goal-test/cli.tsx @@ -1,16 +1,6 @@ -// Userspace emulation of Claude Code's /goal: -// -// 1. Set a natural-language condition. -// 2. Run the agent. -// 3. When the agent halts, call a SEPARATE `infer` (the judge) to -// evaluate whether the condition holds against the transcript. -// 4. If the judge says ok=false, re-prompt the agent with the judge's -// reason. Loop until ok=true (or iteration budget is exhausted). -// -// No runtime modification: this is pure userspace orchestration around -// `agent.run` + `agent.events` + a second `infer` call. The judge is -// the same model family but isolated — it never sees the agent's tools, -// only the transcript. +// End-to-end test of the `` component + `/goal` slash command + +// halt-gate fiber. This replaces the prior userspace orchestration — +// the runtime now owns the loop. We just set the goal and run. // // Run: // infisical run --silent -- npx tsx cli.tsx @@ -26,114 +16,19 @@ import { import { Agent, Block, + Goal, Messages, Workspace, } from "@flamecast/agentjsx/components" -import type { Event, InferFn } from "@flamecast/agentjsx" - -// --- Config --------------------------------------------------------------- const GOAL = "The assistant has greeted the user in pirate speak (e.g. 'Ahoy')." const INITIAL_PROMPT = "Greet me." -const MAX_ITERATIONS = 4 - -// --- Pretty-printing ------------------------------------------------------ const DIM = (s: string) => `\x1b[2m${s}\x1b[0m` const BLUE = (s: string) => `\x1b[34m${s}\x1b[0m` const GREEN = (s: string) => `\x1b[32m${s}\x1b[0m` -const RED = (s: string) => `\x1b[31m${s}\x1b[0m` const YELLOW = (s: string) => `\x1b[33m${s}\x1b[0m` -// --- Judge ---------------------------------------------------------------- - -interface Verdict { - ok: boolean - reason: string -} - -function buildTranscript(events: ReadonlyArray): string { - const lines: string[] = [] - for (const e of events) { - if (e.type === "user.message") { - lines.push(`[user] ${typeof e.content === "string" ? e.content : JSON.stringify(e.content)}`) - } else if (e.type === "assistant.message") { - if (e.content) lines.push(`[assistant] ${e.content}`) - } else if (e.type === "tool.call.started") { - lines.push(`[tool.call] ${e.tool_name}`) - } else if (e.type === "tool.result") { - lines.push(`[tool.result] ${e.content.slice(0, 400)}`) - } - } - return lines.join("\n") -} - -async function judge( - infer: InferFn, - condition: string, - transcript: string, -): Promise { - const system = [ - "You are evaluating a hook condition in agentjsx.", - "Judge whether the user-provided condition is met against the transcript below.", - 'Respond with a JSON object EXACTLY of the shape {"ok": boolean, "reason": string}.', - "Always include a reason. Quote specific text from the transcript when possible.", - "If there is no clear evidence, return ok: false with reason \"insufficient evidence\".", - ].join("\n") - - const userMsg = `CONDITION:\n${condition}\n\nTRANSCRIPT:\n${transcript}\n\nRespond with JSON only.` - - const res = await infer({ - system, - messages: [{ role: "user", content: userMsg }], - tools: [], - }) - - const raw = res.content.trim() - // Strip ```json fences if the model adds them - const cleaned = raw.replace(/^```(?:json)?\s*/i, "").replace(/```$/i, "").trim() - try { - const parsed = JSON.parse(cleaned) - if (typeof parsed?.ok === "boolean" && typeof parsed?.reason === "string") { - return parsed - } - return { ok: false, reason: `judge returned malformed JSON: ${raw}` } - } catch { - return { ok: false, reason: `judge returned non-JSON: ${raw}` } - } -} - -// --- Agent loop ----------------------------------------------------------- - -async function drainTurn(agent: ReturnType): Promise { - const startLen = (await agent.events()).length - let printed = startLen - while (true) { - await new Promise((r) => setTimeout(r, 100)) - const events = await agent.events() - for (let i = printed; i < events.length; i++) { - const e = events[i]! - if (e.type === "assistant.message" && e.content) { - console.log(`${GREEN("agent")} ${e.content}`) - } else if (e.type === "tool.call.started") { - console.log(DIM(` ${YELLOW("→")} ${e.tool_name}`)) - } else if (e.type === "tool.result") { - const snippet = e.content.length > 80 ? `${e.content.slice(0, 80)}…` : e.content - console.log(DIM(` ${YELLOW("←")} ${snippet}`)) - } else if (e.type === "assistant.halted") { - console.log(DIM(` ${YELLOW("!")} halted: ${e.reason}`)) - } - } - printed = events.length - const last = events[events.length - 1] - const isTerminal = - (last?.type === "assistant.message" && !last.tool_calls?.length) || - last?.type === "assistant.halted" || - last?.type === "inference.failed" - if (isTerminal) break - } -} - async function main(): Promise { const apiKey = process.env.AI_GATEWAY_API_KEY if (!apiKey) { @@ -155,6 +50,7 @@ async function main(): Promise { You are a friendly assistant. Respond briefly to the user. + , @@ -164,29 +60,41 @@ async function main(): Promise { try { console.log(DIM(`goal: ${GOAL}`)) console.log("") + + await agent.run(`/goal ${GOAL}`) + console.log(`${BLUE("you")} /goal …`) + console.log(`${BLUE("you")} ${INITIAL_PROMPT}`) await agent.run(INITIAL_PROMPT) - await drainTurn(agent) - for (let i = 1; i <= MAX_ITERATIONS; i++) { + // Stream events. The halt-gate fiber will reprompt automatically + // on assistant.halted if the predicate fails; we just observe. + let printed = (await agent.events()).length + const startTs = Date.now() + while (Date.now() - startTs < 60_000) { + await new Promise((r) => setTimeout(r, 200)) const events = await agent.events() - const transcript = buildTranscript(events) - console.log("") - console.log(DIM(` judging iteration ${i}/${MAX_ITERATIONS}…`)) - const verdict = await judge(infer, GOAL, transcript) - if (verdict.ok) { - console.log(GREEN(` ✓ goal met: ${verdict.reason}`)) - return + for (let i = printed; i < events.length; i++) { + const e = events[i]! + if (e.type === "assistant.message" && e.content) { + console.log(`${GREEN("agent")} ${e.content}`) + } else if (e.type === "user.message") { + const c = typeof e.content === "string" ? e.content : JSON.stringify(e.content) + if (i > printed - 1) console.log(`${DIM(YELLOW("reprompt"))} ${c}`) + } else if (e.type === "assistant.halted") { + console.log(DIM(` ${YELLOW("!")} halted: ${e.reason}`)) + } } - console.log(RED(` ✗ not met: ${verdict.reason}`)) - const reprompt = `[goal: ${GOAL}]: ${verdict.reason}` - console.log(`${BLUE("you")} ${reprompt}`) - await agent.run(reprompt) - await drainTurn(agent) + printed = events.length + const last = events[events.length - 1] + // Terminal when last event is an unanswered halt (gate decided + // the predicate is satisfied) OR a clean assistant.message + // with no further tool calls and no reprompt queued. + if (last?.type === "assistant.halted") break } + console.log("") - console.log(RED(`exhausted ${MAX_ITERATIONS} iterations without meeting goal`)) - process.exit(2) + console.log(GREEN("done")) } finally { await agent.dispose() } diff --git a/src/core/agent-ctx.ts b/src/core/agent-ctx.ts index 9c0d681..5d5eedc 100644 --- a/src/core/agent-ctx.ts +++ b/src/core/agent-ctx.ts @@ -24,6 +24,7 @@ import type { TextDelta, Tool, } from "./types"; +import type { Command, HaltPredicate } from "../jsx/runtime"; const TEXT_DELTA_CAPACITY = 256; @@ -135,6 +136,25 @@ export interface AgentCtxService { // subscribed to `events.changes` — those must call `render` (below) // to avoid the FRP glitch. readonly rendered: SubscriptionRef.SubscriptionRef; + // Commands emitted by the current JSX projection. Populated each + // render pass from `Rendered.commands`. The slash-command router in + // `agent.run` reads this to dispatch `/ ...` inputs to a + // registered handler. Empty when no `contextFn` is configured or no + // `emitCommand` appears in the tree. + readonly commands: SubscriptionRef.SubscriptionRef>; + // Halt-predicate registry, written by command handlers via their + // CommandRuntime. SubscriptionRef rather than plain Ref because future + // consumers (UI surfacing active goals, the halt-gate fiber in a + // follow-up PR) will subscribe to changes. + readonly haltPredicates: SubscriptionRef.SubscriptionRef< + ReadonlyMap + >; + readonly registerHaltPredicate: ( + name: string, + fn: HaltPredicate, + ) => Effect.Effect; + readonly clearHaltPredicate: (name: string) => Effect.Effect; + readonly getHaltPredicates: Effect.Effect>; // Synchronous render from primary sources (events + ambients + // transforms + tools). Runs the shaper chain AND the terminal // adapter, so the result is the same ProviderContext shape the @@ -196,6 +216,10 @@ export const make = ( tools: [], }; const rendered = yield* SubscriptionRef.make(emptyContext); + const commands = yield* SubscriptionRef.make>([]); + const haltPredicates = yield* SubscriptionRef.make< + ReadonlyMap + >(new Map()); // Explicit re-render trigger for extension-owned reactive state. // Bumped by `invalidate`; merged into the render driver below. const invalidateRef = yield* SubscriptionRef.make(0); @@ -377,9 +401,10 @@ export const make = ( rendered = renderedExit.value; } else { yield* reportError("context", renderedExit.error); - rendered = { fragments: [], tools: [] }; + rendered = { fragments: [], tools: [], commands: [] }; } yield* reconcileContextTools(rendered.tools); + yield* SubscriptionRef.set(commands, [...rendered.commands]); fragments = [...rendered.fragments]; } else if (renderer) { const eventsArr = Chunk.toReadonlyArray(currentEvents); @@ -455,6 +480,27 @@ export const make = ( (n) => n + 1, ); + const registerHaltPredicate = ( + name: string, + fn: HaltPredicate, + ): Effect.Effect => + SubscriptionRef.update(haltPredicates, (current) => { + const next = new Map(current); + next.set(name, fn); + return next; + }); + const clearHaltPredicate = (name: string): Effect.Effect => + SubscriptionRef.update(haltPredicates, (current) => { + if (!current.has(name)) return current; + const next = new Map(current); + next.delete(name); + return next; + }); + const getHaltPredicates: Effect.Effect> = + SubscriptionRef.get(haltPredicates).pipe( + Effect.map((m) => new Map(m) as ReadonlyMap), + ); + const deltasHub = yield* PubSub.sliding(TEXT_DELTA_CAPACITY); const textDeltas: Stream.Stream = Stream.fromPubSub(deltasHub); const emitTextDelta = (delta: TextDelta): Effect.Effect => @@ -467,6 +513,11 @@ export const make = ( transforms, errors, rendered, + commands, + haltPredicates, + registerHaltPredicate, + clearHaltPredicate, + getHaltPredicates, render, addTool, addAmbient, diff --git a/src/core/agent.ts b/src/core/agent.ts index 9a2bd6c..5f15887 100644 --- a/src/core/agent.ts +++ b/src/core/agent.ts @@ -7,7 +7,9 @@ import { Stream, SubscriptionRef, } from "effect"; +import type { CommandRuntime, HaltPredicate } from "../jsx/runtime"; import { AgentCtx, type AgentErrorEntry, type Renderer } from "./agent-ctx"; +import { runHaltGate } from "./halt-gate"; import { runInference } from "./inference"; import { validateProviderContext } from "./validate"; import { PendingSends } from "./pending-sends"; @@ -158,6 +160,10 @@ export const createAgent = ( : opts.validate ?? validateProviderContext, }); yield* runToolExecution({ concurrency: opts.toolConcurrency ?? 8 }); + // Halt-gate supervisor: runs predicates on each `assistant.halted` and + // appends a synthetic user.message that re-prompts the model when a + // goal is unmet. No-op when no predicates are registered. + yield* runHaltGate(opts.infer); return ctx; }); @@ -270,15 +276,70 @@ export const createAgentRuntime = (opts: AgentOptions): Agent => { withCtx((ctx) => ctx.events.snapshot.pipe(Effect.map(lastResult))); const run = (input: unknown): Promise => { - const body: Effect.Effect = Effect.gen(function* () { + const SLASH_RE = /^\/([a-zA-Z_][\w-]*)(?:\s+([\s\S]*))?$/; + const body: Effect.Effect< + Promise | null, + never, + AgentCtx | PendingSends + > = Effect.gen(function* () { const ctx = yield* AgentCtx; const pending = yield* PendingSends; + + // Slash-command router. Only intercepts when input is a string + // matching `/(...)`. Looks up the name in the current JSX + // projection's command list; if matched, runs the handler instead + // of appending a user.message / triggering inference. Unknown + // commands fall through with a warning so the model still gets + // the literal text (lets the model explain that the slash command + // isn't registered rather than silently dropping the input). + if (typeof input === "string") { + const m = input.match(SLASH_RE); + if (m) { + const name = m[1]!; + const args = m[2] ?? ""; + const cmds = yield* SubscriptionRef.get(ctx.commands); + const cmd = cmds.find((c) => c.name === name); + if (cmd) { + // Build the handler-facing CommandRuntime. Each method + // bridges out of the handler's JS-promise world back into + // Effect via runtime.runPromise. `appendUserMessage` goes + // through ctx.events.append so the log remains the single + // source of truth (no side-channel writes — principle 1 + // in src/CLAUDE.md). + const cmdRuntime: CommandRuntime = { + appendUserMessage: (text: string) => { + void runtime.runPromise( + ctx.events.append({ type: "user.message", content: text }), + ); + }, + registerHaltPredicate: (n: string, fn: HaltPredicate) => { + void runtime.runPromise(ctx.registerHaltPredicate(n, fn)); + }, + clearHaltPredicate: (n: string) => { + void runtime.runPromise(ctx.clearHaltPredicate(n)); + }, + }; + // Run the handler outside the Effect fiber — handlers are + // user JS that may return a Promise. Return the promise so + // the outer `run` awaits it before resolving. + const result = cmd.handler({ args, runtime: cmdRuntime }); + return result instanceof Promise ? result : Promise.resolve(); + } + // Unknown command — warn once and fall through to the normal + // user.message path. + console.warn( + `[agentjsx] received slash input "/${name}" but no command with that name is registered; passing through as user.message`, + ); + } + } + const evs = yield* ctx.events.snapshot; if (toolsInFlight(evs)) { yield* pending.push(input); - return; + return null; } yield* ctx.events.append({ type: "user.message", content: input }); + return null; }); // Ensure the agent has finished building before sending — otherwise // seeded extensions could still be installing and a user message @@ -288,7 +349,11 @@ export const createAgentRuntime = (opts: AgentOptions): Agent => { // resolves only after the append (or pending-sends push) has landed, // so callers that `await send` can safely subscribe to `until` // without a stale-replay race. - return built.then(() => runtime.runPromise(body)); + return built.then(() => + runtime.runPromise(body).then((handlerPromise) => + handlerPromise ? handlerPromise : undefined, + ), + ); }; const until = (predicate: (snapshot: AgentSnapshot) => T | null): Promise => { diff --git a/src/core/halt-gate.ts b/src/core/halt-gate.ts new file mode 100644 index 0000000..a453af9 --- /dev/null +++ b/src/core/halt-gate.ts @@ -0,0 +1,143 @@ +import { Chunk, Effect, Ref, Stream } from "effect"; +import { AgentCtx } from "./agent-ctx"; +import type { Event, InferFn } from "./types"; + +// Halt-gate supervisor. Watches the event log for "about to return +// control to the user" states, runs all registered halt predicates, +// and — if any returns `ok: false` — appends a synthetic `user.message` +// whose content concatenates the failing reasons. The existing inference +// loop sees the new user.message and triggers another turn, re-prompting +// the model toward the unmet goal. +// +// Two trigger shapes count as "about to halt": +// 1. An explicit `assistant.halted` event (e.g. from `maxSteps`). +// 2. A *natural* terminal — the last event is an `assistant.message` +// with no `tool_calls` AND no other in-flight tool calls earlier +// in the turn. This is how `/goal` matches Claude Code's Stop hook +// semantics: predicates gate every point where the agent would +// otherwise return to the user, not just forced halts. +// +// In both cases the gate dedupes by the triggering event's `seq` so each +// terminal is judged at most once even if the log churns. +// +// Source of truth: the gate writes a real `user.message` event through +// `ctx.events.append` so the log remains the single durable record +// (principle 1 in src/CLAUDE.md). No side-channel re-prompts. +// +// Concurrency: forked with `Effect.forkScoped`, so the supervisor dies +// with the enclosing agent scope. The fiber subscribes to +// `ctx.events.changes` and is the SOLE consumer that maintains +// "last seen halt seq" state — no other fiber needs that view. +export const runHaltGate = ( + infer: InferFn, +): Effect.Effect => + Effect.gen(function* () { + const ctx = yield* AgentCtx; + // Per-seq dedupe. Once a halted seq has been judged (predicates run, + // either reprompt appended or halt left to stand), we never reprocess + // that seq — even if the log changes shape later. Set lives only + // inside this fiber; no cross-fiber sharing. + const seenRef = yield* Ref.make>(new Set()); + + // Detect a "natural terminal": last event is `assistant.message` + // with no tool_calls, and no `tool.call.started` after the most + // recent `user.message` remains unmatched. Mirrors the inference + // loop's idle condition. + const findNaturalTerminalSeq = ( + arr: ReadonlyArray, + ): number | null => { + const last = arr[arr.length - 1]; + if (!last || last.type !== "assistant.message") return null; + if (last.tool_calls && last.tool_calls.length > 0) return null; + // Walk back to the last user.message; any tool.call.started since + // then must have a matching tool.result. + const started = new Set(); + const finished = new Set(); + for (let i = arr.length - 1; i >= 0; i--) { + const e = arr[i]!; + if (e.type === "user.message") break; + if (e.type === "tool.call.started") started.add(e.tool_call_id); + else if (e.type === "tool.result") finished.add(e.tool_call_id); + } + for (const id of started) { + if (!finished.has(id)) return null; + } + return last.seq; + }; + + const step = (events: Chunk.Chunk): Effect.Effect => + Effect.gen(function* () { + const arr = Chunk.toReadonlyArray(events); + const seen = yield* Ref.get(seenRef); + // Trigger 1: most recent unjudged `assistant.halted` (iterate + // from tail so the common "just appended" case is O(1)). + let triggerSeq: number | null = null; + for (let i = arr.length - 1; i >= 0; i--) { + const e = arr[i]!; + if (e.type !== "assistant.halted") continue; + if (seen.has(e.seq)) break; + triggerSeq = e.seq; + break; + } + // Trigger 2: natural terminal (assistant.message with no pending + // tool work). Only used if no halted trigger fired. + if (triggerSeq === null) { + const naturalSeq = findNaturalTerminalSeq(arr); + if (naturalSeq !== null && !seen.has(naturalSeq)) { + triggerSeq = naturalSeq; + } + } + if (triggerSeq === null) return; + // Mark this seq judged BEFORE running predicates so a predicate + // that itself triggers more events (e.g. by calling infer) can't + // re-enter this step for the same trigger. + const haltedSeq = triggerSeq; + yield* Ref.update(seenRef, (s) => { + const next = new Set(s); + next.add(haltedSeq); + return next; + }); + + const predicates = yield* ctx.getHaltPredicates; + if (predicates.size === 0) return; // halt stands + + const snapshot: ReadonlyArray = arr; + const entries = Array.from(predicates.entries()); + // Run predicates in parallel. Predicate throws are caught and + // mapped to `{ ok: false, reason: "predicate threw: ..." }` so a + // misbehaving goal can't kill the supervisor. + const results = yield* Effect.all( + entries.map(([name, fn]) => + Effect.tryPromise({ + try: () => fn({ events: snapshot, infer }), + catch: (err) => err, + }).pipe( + Effect.match({ + onFailure: (err) => ({ + name, + ok: false, + reason: `predicate threw: ${ + err instanceof Error ? err.message : String(err) + }`, + }), + onSuccess: (r) => ({ name, ok: r.ok, reason: r.reason }), + }), + ), + ), + { concurrency: "unbounded" }, + ); + + const failing = results.filter((r) => !r.ok); + if (failing.length === 0) return; // all goals met — halt stands + + const content = failing + .map((r) => `[goal: ${r.name}] not met: ${r.reason}`) + .join("\n"); + yield* ctx.events.append({ type: "user.message", content }); + }); + + const driver = ctx.events.changes.pipe( + Stream.mapEffect((evs) => step(evs), { concurrency: 1 }), + ); + yield* Effect.forkScoped(Stream.runDrain(driver)); + }); diff --git a/src/core/types.ts b/src/core/types.ts index ade16b9..67e96cd 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -166,13 +166,15 @@ export type Fragment = { }[keyof FragmentMap]; // Output of the JSX render walk (see `src/jsx/render.ts`). Components -// emit into one of these channels via `emitFragment` / `emitTool`. The -// runtime consumes a `Rendered` to seed `ctx.ambients` and `ctx.tools` -// at startup. Additional channels (transforms, forked effects) are -// reserved for future stages — not implemented yet. +// emit into one of these channels via `emitFragment` / `emitTool` / +// `emitCommand`. The runtime consumes a `Rendered` to seed +// `ctx.ambients`, `ctx.tools`, and slash commands at startup. +// Additional channels (transforms, forked effects) are reserved for +// future stages — not implemented yet. export interface Rendered { readonly fragments: ReadonlyArray; readonly tools: ReadonlyArray; + readonly commands: ReadonlyArray; } export interface InferResponse { diff --git a/src/index.ts b/src/index.ts index a0e8e62..889116e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -22,6 +22,7 @@ export { export { PendingSends, type PendingSendsService } from "./core/pending-sends"; export { runInference } from "./core/inference"; export { runToolExecution } from "./core/tool-exec"; +export { runHaltGate } from "./core/halt-gate"; export { isHalted, lastResult, diff --git a/src/jsx/components/goal.tsx b/src/jsx/components/goal.tsx new file mode 100644 index 0000000..cc8e664 --- /dev/null +++ b/src/jsx/components/goal.tsx @@ -0,0 +1,83 @@ +// Goal — set a stop condition judged by a separate inference call. +// +// Mount `` and the operator can type `/goal ` to +// install a halt predicate. The predicate runs whenever the agent emits +// `assistant.halted`: a side-channel inference call judges whether the +// condition holds against the transcript and returns `{ ok, reason }`. +// `/goal clear` (or `/goal` with no args) removes the predicate. +// +// v1 simplification: the system fragment only shows help text. The +// active condition isn't echoed back into the projection because +// CommandRuntime doesn't expose `appendSystem`. The model learns the +// condition the first time the predicate fails (via its reason string). + +import { emitCommand, emitFragment } from "../runtime"; +import type { Node } from "../runtime"; + +export function Goal(): Node { + return [ + emitFragment({ + tag: "core/system", + source: "goal", + content: + "(use `/goal ` to set a stop condition; `/goal clear` to remove)", + }), + emitCommand({ + name: "goal", + description: + "Set or clear a halt condition judged by a separate inference call.", + handler: async ({ args, runtime }) => { + const trimmed = args.trim(); + if (trimmed === "" || trimmed === "clear") { + runtime.clearHaltPredicate("goal"); + return; + } + const condition = trimmed; + runtime.registerHaltPredicate("goal", async ({ events, infer }) => { + const transcript = events + .filter( + (e) => e.type === "user.message" || e.type === "assistant.message", + ) + .map((e) => { + if (e.type === "user.message") { + const c = e.content; + return `[user] ${typeof c === "string" ? c : JSON.stringify(c)}`; + } + return `[assistant] ${e.content}`; + }) + .join("\n"); + const system = [ + "You are evaluating a hook condition.", + `Judge whether this condition holds against the transcript: "${condition}".`, + 'Respond with JSON exactly: {"ok": boolean, "reason": string}. Always include a reason. Quote specific text from the transcript when possible.', + ].join("\n"); + try { + const res = await infer({ + system, + messages: [{ role: "user", content: `TRANSCRIPT:\n${transcript}` }], + tools: [], + }); + const raw = res.content.trim(); + const cleaned = raw + .replace(/^```(?:json)?\s*/i, "") + .replace(/```$/i, "") + .trim(); + const parsed = JSON.parse(cleaned); + if ( + typeof parsed?.ok === "boolean" && + typeof parsed?.reason === "string" + ) { + return parsed; + } + return { ok: false, reason: `judge returned malformed JSON: ${raw}` }; + } catch (e) { + return { + ok: false, + reason: `judge error: ${e instanceof Error ? e.message : String(e)}`, + }; + } + }); + }, + }), + ]; +} diff --git a/src/jsx/components/index.ts b/src/jsx/components/index.ts index 8e80f93..f6d2ca9 100644 --- a/src/jsx/components/index.ts +++ b/src/jsx/components/index.ts @@ -11,3 +11,4 @@ export { McpServer } from "./mcp"; export { Subagent } from "./subagent"; export { Memory } from "./memory"; export { WebSearch, WebFetch } from "./web"; +export { Goal } from "./goal"; diff --git a/src/jsx/components/skills.tsx b/src/jsx/components/skills.tsx index bcba31d..7d69e0e 100644 --- a/src/jsx/components/skills.tsx +++ b/src/jsx/components/skills.tsx @@ -33,12 +33,24 @@ import { FileSystem, Path } from "@effect/platform"; import { Effect, Schema } from "effect"; import { defineTool } from "../../core/define-tool"; import type { Fragment as RenderedFragment } from "../../core/types"; -import { emitFragment, emitTool, type Element, type Node } from "../runtime"; +import { + emitCommand, + emitFragment, + emitTool, + type Element, + type Node, +} from "../runtime"; import { useRenderContext } from "../render"; +interface SkillCommand { + name: string; + prompt: string; +} + interface SkillEntry { name: string; description: string; + commands: SkillCommand[]; } interface CacheState { @@ -96,21 +108,107 @@ function parseFrontmatterFields(raw: string): Record { return out; } +// Parse the `commands:` list out of a frontmatter block. Shape: +// commands: +// - name: review +// prompt: "Review the current diff." +// - name: ship +// prompt: "Land the current work as a PR." +// Tiny indentation-aware parser; values may be quoted. Malformed +// entries (missing `name` or `prompt`) are skipped with a warning. +// Returns [] if the field is absent. Throws are caught at the call +// site so a bad `commands:` block doesn't crash the component. +function parseFrontmatterCommands(raw: string): SkillCommand[] { + const lines = raw.split(/\r?\n/); + let i = 0; + // Find the `commands:` line. + while (i < lines.length && !/^commands\s*:\s*$/.test(lines[i]!.trim())) i++; + if (i >= lines.length) return []; + i++; + const out: SkillCommand[] = []; + let current: Partial | null = null; + const stripQuotes = (v: string) => { + const t = v.trim(); + if ( + (t.startsWith('"') && t.endsWith('"')) || + (t.startsWith("'") && t.endsWith("'")) + ) { + return t.slice(1, -1); + } + return t; + }; + const flush = () => { + if (!current) return; + if ( + typeof current.name === "string" && + current.name.length > 0 && + typeof current.prompt === "string" && + current.prompt.length > 0 + ) { + out.push({ name: current.name, prompt: current.prompt }); + } else { + // eslint-disable-next-line no-console + console.warn( + `[skills] skipping malformed command entry: ${JSON.stringify(current)}`, + ); + } + current = null; + }; + for (; i < lines.length; i++) { + const raw = lines[i]!; + const trimmed = raw.trim(); + if (trimmed === "") continue; + // A non-indented line ends the commands block. + if (!/^\s/.test(raw)) break; + const itemMatch = /^\s*-\s*(.*)$/.exec(raw); + if (itemMatch) { + flush(); + current = {}; + const rest = itemMatch[1]!.trim(); + if (rest.length > 0) { + const kv = /^([a-zA-Z_][\w-]*)\s*:\s*(.*)$/.exec(rest); + if (kv) { + const k = kv[1]!; + const v = stripQuotes(kv[2]!); + if (k === "name") current.name = v; + else if (k === "prompt") current.prompt = v; + } + } + continue; + } + if (!current) continue; + const kv = /^\s+([a-zA-Z_][\w-]*)\s*:\s*(.*)$/.exec(raw); + if (!kv) continue; + const k = kv[1]!; + const v = stripQuotes(kv[2]!); + if (k === "name") current.name = v; + else if (k === "prompt") current.prompt = v; + } + flush(); + return out; +} + // Parse a skill file into its description (one-liner shown in the -// `` menu) and body (full markdown the model sees on -// `skill_lookup`/`skill_invoke`). If the file starts with a YAML -// frontmatter block (`---\n...\n---\n`), the `description` field -// supplies the menu line and the body is everything after the closing -// fence. If no frontmatter is present, falls back to the heuristic: -// description is the first non-heading, non-blank line; body is the -// full file. Backwards-compatible with plain-markdown skill files. +// `` menu), body (full markdown the model sees on +// `skill_lookup`/`skill_invoke`), and any declared slash commands. +// If the file starts with a YAML frontmatter block (`---\n...\n---\n`), +// the `description` field supplies the menu line and `commands:` (if +// present) defines slash commands the skill registers. The body is +// everything after the closing fence. If no frontmatter is present, +// falls back to the heuristic: description is the first non-heading, +// non-blank line; body is the full file; no commands. export function parseSkillFile(content: string): { description: string; body: string; + commands: SkillCommand[]; } { const m = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/.exec(content); if (!m) { - return { description: deriveDescription(content), body: content }; + return { + description: deriveDescription(content), + body: content, + commands: [], + }; } const frontmatterRaw = m[1]!; const body = m[2] ?? ""; @@ -119,7 +217,17 @@ export function parseSkillFile(content: string): { fields.description && fields.description.length > 0 ? fields.description : deriveDescription(body); - return { description, body }; + let commands: SkillCommand[] = []; + try { + commands = parseFrontmatterCommands(frontmatterRaw); + } catch (e) { + // eslint-disable-next-line no-console + console.warn( + `[skills] failed to parse commands frontmatter: ${e instanceof Error ? e.message : String(e)}`, + ); + commands = []; + } + return { description, body, commands }; } function listSkills( @@ -142,8 +250,8 @@ function listSkills( const content = yield* fs .readFileString(skillFile) .pipe(Effect.catchAll(() => Effect.succeed(""))); - const { description } = parseSkillFile(content); - out.push({ name: entry, description }); + const { description, commands } = parseSkillFile(content); + out.push({ name: entry, description, commands }); } out.sort((a, b) => a.name.localeCompare(b.name)); return out; @@ -249,7 +357,13 @@ export function Skills(props: SkillsProps): Node { } else if (state.entries.length === 0) { content = `\n(none)\n`; } else { - const lines = state.entries.map((s) => `- ${s.name}: ${s.description}`); + const lines: string[] = []; + for (const s of state.entries) { + lines.push(`- ${s.name}: ${s.description}`); + for (const c of s.commands) { + lines.push(` /${c.name}`); + } + } content = `\n${lines.join("\n")}\n`; } @@ -264,5 +378,30 @@ export function Skills(props: SkillsProps): Node { emitTool(skill_invoke), emitFragment(block), ]; + + // Emit one command per (skill, command) pair. Names are flat: collisions + // across skills are an operator concern. Description carries the skill + // name in brackets so operators can disambiguate when listing commands. + if (state.resolved) { + for (const s of state.entries) { + for (const c of s.commands) { + const oneLine = c.prompt.replace(/\s+/g, " ").trim(); + const truncated = + oneLine.length > 80 ? `${oneLine.slice(0, 77)}...` : oneLine; + const prompt = c.prompt; + emits.push( + emitCommand({ + name: c.name, + description: `[${s.name}] ${truncated}`, + handler: ({ args, runtime }) => { + runtime.appendUserMessage( + `${prompt}${args ? `\n\n${args}` : ""}`, + ); + }, + }), + ); + } + } + } return emits as Node; } diff --git a/src/jsx/render.ts b/src/jsx/render.ts index 53966bb..832da84 100644 --- a/src/jsx/render.ts +++ b/src/jsx/render.ts @@ -9,6 +9,7 @@ import type { Effect } from "effect"; import type { Event, Fragment as RenderedFragment, InferFn, Rendered, Tool } from "../core/types"; import { + type Command, type ComponentFunction, type Element, type Node, @@ -18,6 +19,7 @@ import { interface RenderCollector { fragments: RenderedFragment[]; tools: Tool[]; + commands: Command[]; } // RenderContext is the ambient state visible to function components @@ -91,7 +93,7 @@ export function useRenderContext(): RenderContext { } export function render(root: Node, context?: RenderContext): Rendered { - const collector: RenderCollector = { fragments: [], tools: [] }; + const collector: RenderCollector = { fragments: [], tools: [], commands: [] }; const previous = currentContext; // Precedence: explicit `context` arg wins (callers that thread their // own state stay in control), otherwise pick up the runtime-injected @@ -104,7 +106,11 @@ export function render(root: Node, context?: RenderContext): Rendered { } finally { currentContext = previous; } - return { fragments: collector.fragments, tools: collector.tools }; + return { + fragments: collector.fragments, + tools: collector.tools, + commands: collector.commands, + }; } // Walk a child subtree into a fresh local collector and return the @@ -124,10 +130,14 @@ export function render(root: Node, context?: RenderContext): Rendered { // the existing walker already takes its collector as an explicit // argument — see walk() below. export function renderChildren(children: Node | ReadonlyArray): Rendered { - const collector: RenderCollector = { fragments: [], tools: [] }; + const collector: RenderCollector = { fragments: [], tools: [], commands: [] }; // Array case is just a Node per the Node union; walk handles both. walk(children as Node, collector); - return { fragments: collector.fragments, tools: collector.tools }; + return { + fragments: collector.fragments, + tools: collector.tools, + commands: collector.commands, + }; } function walk(node: Node, collector: RenderCollector): void { @@ -151,6 +161,8 @@ function walk(node: Node, collector: RenderCollector): void { collector.fragments.push(value as RenderedFragment); } else if (kind === "tool") { collector.tools.push(value as Tool); + } else if (kind === "command") { + collector.commands.push(value as Command); } return; } diff --git a/src/jsx/runtime.ts b/src/jsx/runtime.ts index 8d1182e..2ec568a 100644 --- a/src/jsx/runtime.ts +++ b/src/jsx/runtime.ts @@ -76,13 +76,41 @@ export function createElement( // Modeled as an Element so the JSX tree stays homogeneous. export const EMIT_SENTINEL: ComponentFunction = () => null; -export type EmitKind = "fragment" | "tool"; +// Commands are user-initiated: the operator types `/ ` into +// `agent.run` and the runtime dispatches to a registered handler. They +// differ from tools (model-initiated) and fragments (declarative prompt +// content). A handler may register halt predicates imperatively via its +// CommandRuntime; halt predicates are not emit values themselves because +// the runtime owns their lifecycle, not the JSX walk. +export interface Command { + readonly name: string; + readonly description?: string; + readonly handler: CommandHandler; +} + +export type CommandHandler = (ctx: { + readonly args: string; + readonly runtime: CommandRuntime; +}) => Promise | void; + +export interface CommandRuntime { + appendUserMessage: (text: string) => void; + registerHaltPredicate: (name: string, fn: HaltPredicate) => void; + clearHaltPredicate: (name: string) => void; +} + +export type HaltPredicate = (ctx: { + readonly events: ReadonlyArray; + readonly infer: import("../core/types").InferFn; +}) => Promise<{ ok: boolean; reason: string }>; + +export type EmitKind = "fragment" | "tool" | "command"; export interface EmitElement extends Element { readonly type: typeof EMIT_SENTINEL; readonly props: { readonly __emit: EmitKind; - readonly value: RenderedFragment | Tool; + readonly value: RenderedFragment | Tool | Command; }; } @@ -102,6 +130,14 @@ export function emitTool(value: Tool): EmitElement { }; } +export function emitCommand(value: Command): EmitElement { + return { + type: EMIT_SENTINEL, + props: { __emit: "command", value }, + children: [], + }; +} + export function isEmitElement(node: unknown): node is EmitElement { return ( typeof node === "object" && diff --git a/test/agentctx/core/agent-run-contract.test.ts b/test/agentctx/core/agent-run-contract.test.ts new file mode 100644 index 0000000..b89f875 --- /dev/null +++ b/test/agentctx/core/agent-run-contract.test.ts @@ -0,0 +1,306 @@ +import { Effect } from "effect"; +import { describe, expect, it } from "vitest"; +import { AgentCtx, createAgentRuntime, maxSteps } from "@flamecast/agentjsx"; +import type { InferFn, ProviderContext } from "@flamecast/agentjsx"; + +// Locks the current `agent.run` behavior before a planned migration adds +// slash-command routing and halt-predicate gating. Each test here pins a +// behavior that the next PR will deliberately change; when these break, +// the regression proves the new behavior shipped. + +describe("agentctx: agent.run contract (pre-router, pre-halt-gating)", () => { + it("non-slash input is appended as a user.message and fed to infer unchanged", async () => { + let lastContext: ProviderContext | undefined; + const infer: InferFn = async (ctx) => { + lastContext = ctx; + return { content: "ok" }; + }; + + const agent = createAgentRuntime({ infer }); + try { + await agent.run("hello world"); + + await agent.until((s) => { + const last = s.events.at(-1); + return last?.type === "assistant.message" ? true : null; + }); + + expect(lastContext).toBeDefined(); + const messages = lastContext!.messages; + const lastUser = [...messages].reverse().find((m) => m.role === "user"); + expect(lastUser).toBeDefined(); + const content = lastUser!.content; + const text = + typeof content === "string" + ? content + : content.map((c) => c.text).join(""); + expect(text).toBe("hello world"); + + const events = await agent.events(); + const userIdx = events.findIndex( + (e) => e.type === "user.message" && e.content === "hello world", + ); + expect(userIdx).toBeGreaterThanOrEqual(0); + const afterUser = events.slice(userIdx + 1); + expect( + afterUser.some( + (e) => e.type === "assistant.message" && e.content === "ok", + ), + ).toBe(true); + } finally { + await agent.dispose(); + } + }); + + it("slash-prefixed input STILL goes to inference today (baseline before router)", async () => { + // Today: no router intercepts `/`. The string lands as a regular + // user.message and infer is invoked with it. When the router lands, + // this test must be updated — that update is the proof the router + // shipped. + let lastContext: ProviderContext | undefined; + let inferCalls = 0; + const infer: InferFn = async (ctx) => { + inferCalls++; + lastContext = ctx; + return { content: "ok" }; + }; + + const agent = createAgentRuntime({ infer }); + try { + await agent.run("/foo bar"); + + await agent.until((s) => { + const last = s.events.at(-1); + return last?.type === "assistant.message" ? true : null; + }); + + expect(inferCalls).toBe(1); + expect(lastContext).toBeDefined(); + const lastUser = [...lastContext!.messages] + .reverse() + .find((m) => m.role === "user"); + expect(lastUser).toBeDefined(); + const content = lastUser!.content; + const text = + typeof content === "string" + ? content + : content.map((c) => c.text).join(""); + expect(text).toBe("/foo bar"); + + const events = await agent.events(); + expect( + events.some( + (e) => e.type === "user.message" && e.content === "/foo bar", + ), + ).toBe(true); + } finally { + await agent.dispose(); + } + }); + + it("registered slash command intercepts: handler runs, original input is NOT a user.message", async () => { + // Wire an agent whose JSX projection emits one command named `ping`. + // The handler appends `pong: ` via runtime.appendUserMessage, + // which routes through ctx.events.append (log is source of truth). + // The original `/ping hello` must NOT itself land as a user.message. + let inferCalls = 0; + const infer: InferFn = async () => { + inferCalls++; + return { content: "ok-after-handler", tool_calls: undefined }; + }; + + const agent = createAgentRuntime({ + infer, + context: () => ({ + fragments: [], + tools: [], + commands: [ + { + name: "ping", + handler: ({ args, runtime }) => { + runtime.appendUserMessage(`pong: ${args}`); + }, + }, + ], + }), + }); + + try { + await agent.run("/ping hello"); + + // Wait for the appended user.message to drive an inference reply. + await agent.until((s) => { + const last = s.events.at(-1); + return last?.type === "assistant.message" && + last.content === "ok-after-handler" + ? true + : null; + }); + + const events = await agent.events(); + const userMessages = events.filter((e) => e.type === "user.message"); + // Exactly one user.message: the handler's `pong: hello`. The + // raw `/ping hello` string must NOT appear. + expect(userMessages.length).toBe(1); + expect( + userMessages.some( + (e) => e.type === "user.message" && e.content === "pong: hello", + ), + ).toBe(true); + expect( + userMessages.some( + (e) => e.type === "user.message" && e.content === "/ping hello", + ), + ).toBe(false); + expect(inferCalls).toBe(1); + } finally { + await agent.dispose(); + } + }); + + it("assistant.halted is terminal without predicates: no further inference, no auto user.message", async () => { + // Stub returns a plain assistant message with no tool calls. The + // natural terminal is `assistant.message`; once it lands, no further + // inference fires and no user.message is appended automatically. + // When halt-predicate gating ships, an *absent* predicate must + // preserve this exact behavior — this test pins the baseline. + let inferCalls = 0; + const infer: InferFn = async () => { + inferCalls++; + return { content: "done", tool_calls: undefined }; + }; + + const agent = createAgentRuntime({ infer }); + try { + await agent.run("hi"); + + await agent.until((s) => { + const last = s.events.at(-1); + return last?.type === "assistant.message" && last.content === "done" + ? true + : null; + }); + + // Settle window: if a follow-up inference or auto user.message + // were going to fire, it would happen within this window. + await new Promise((r) => setTimeout(r, 200)); + + const events = await agent.events(); + const last = events.at(-1); + expect(last?.type).toBe("assistant.message"); + + // Exactly one user.message — the one we sent. + const userMessages = events.filter((e) => e.type === "user.message"); + expect(userMessages.length).toBe(1); + expect(inferCalls).toBe(1); + } finally { + await agent.dispose(); + } + }); + + it("halt-gate reprompts when a predicate returns ok=false", async () => { + // Stub infer always returns a plain assistant message (no tool calls). + // Each call therefore produces an `assistant.halted`. With a predicate + // registered that returns ok=false, the gate must append a synthetic + // user.message whose content includes the predicate's reason — which + // re-drives inference. After clearing the predicate, the next halt + // must be terminal. + let inferCalls = 0; + const infer: InferFn = async () => { + inferCalls++; + return { content: "done", tool_calls: undefined }; + }; + + // maxSteps(1) makes the first assistant.message append a halt — the + // shape the gate is supposed to handle. Without it, content-only + // replies never produce an `assistant.halted` event. + const agent = createAgentRuntime({ infer, extensions: [maxSteps(1)] }); + try { + // Reach into AgentCtx via the runtime escape hatch to register a + // predicate without going through the slash-command path. + let predicateCalls = 0; + await agent.runtime.runPromise( + Effect.gen(function* () { + const ctx = yield* AgentCtx; + yield* ctx.registerHaltPredicate("test", () => { + predicateCalls++; + // Fail the first halt; let subsequent halts stand so the + // test reliably terminates instead of looping forever. + return Promise.resolve( + predicateCalls === 1 + ? { ok: false, reason: "not yet" } + : { ok: true, reason: "satisfied" }, + ); + }); + }), + ); + + await agent.run("hi"); + + // Wait until the gate's synthetic user.message lands in the log. + await agent.until((s) => { + const reprompt = s.events.find( + (e) => + e.type === "user.message" && + typeof e.content === "string" && + e.content.includes("not yet"), + ); + return reprompt ? true : null; + }); + + let events = await agent.events(); + const userMessages = events.filter((e) => e.type === "user.message"); + // Original "hi" plus the gate's reprompt. + expect(userMessages.length).toBeGreaterThanOrEqual(2); + expect( + userMessages.some( + (e) => + e.type === "user.message" && + typeof e.content === "string" && + e.content === "[goal: test] not met: not yet", + ), + ).toBe(true); + // Inference fired at least twice: original + reprompt-driven turn. + expect(inferCalls).toBeGreaterThanOrEqual(2); + + // Clear the predicate. The next halt should be terminal. + await agent.runtime.runPromise( + Effect.gen(function* () { + const ctx = yield* AgentCtx; + yield* ctx.clearHaltPredicate("test"); + }), + ); + + const callsBefore = inferCalls; + await agent.run("again"); + // With maxSteps(1) still active and the predicate cleared, the + // first assistant.message of the new turn drives a halt that + // must stand — the gate is a no-op now. + await agent.until((s) => { + const last = s.events.at(-1); + return last?.type === "assistant.halted" ? true : null; + }); + // Settle: if the gate were still reprompting, another inference + // and synthetic user.message would land within this window. + await new Promise((r) => setTimeout(r, 200)); + + events = await agent.events(); + const last = events.at(-1); + expect(last?.type).toBe("assistant.halted"); + // No new synthetic reprompt after clearing — only "again" added. + const newUserMessages = events + .filter((e) => e.type === "user.message") + .slice(userMessages.length); + expect(newUserMessages.length).toBe(1); + expect( + newUserMessages[0]!.type === "user.message" && + newUserMessages[0]!.content === "again", + ).toBe(true); + // Exactly one more inference for the "again" turn — no reprompt + // amplification after the predicate was cleared. + expect(inferCalls).toBe(callsBefore + 1); + } finally { + await agent.dispose(); + } + }); +}); diff --git a/test/agentctx/core/goal-component.test.ts b/test/agentctx/core/goal-component.test.ts new file mode 100644 index 0000000..7dfbadf --- /dev/null +++ b/test/agentctx/core/goal-component.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vitest"; +import { Goal } from "../../../src/jsx/components/goal"; +import { isEmitElement } from "../../../src/jsx/runtime"; +import type { Command } from "../../../src/jsx/runtime"; +import type { Fragment } from "../../../src/core/types"; + +describe("Goal component", () => { + it("returns an emitFragment and an emitCommand named 'goal'", () => { + const node = Goal(); + expect(Array.isArray(node)).toBe(true); + const items = node as ReadonlyArray; + + const emits = items.filter(isEmitElement); + expect(emits.length).toBe(2); + + const fragmentEmits = emits.filter( + (e) => (e.props as { __emit: string }).__emit === "fragment", + ); + const commandEmits = emits.filter( + (e) => (e.props as { __emit: string }).__emit === "command", + ); + expect(fragmentEmits.length).toBe(1); + expect(commandEmits.length).toBe(1); + + const fragment = (fragmentEmits[0]!.props as { value: Fragment }).value; + expect(fragment.source).toBe("goal"); + expect(fragment.tag).toBe("core/system"); + + const command = (commandEmits[0]!.props as { value: Command }).value; + expect(command.name).toBe("goal"); + expect(typeof command.handler).toBe("function"); + }); +}); From 8f89a4b6eb8a84b23932174de74c6cdd0f57d54b Mon Sep 17 00:00:00 2001 From: Arjun Kumar Date: Thu, 28 May 2026 23:10:50 +0800 Subject: [PATCH 2/2] chore: keep goal-test as userspace example --- examples/goal-test/cli.tsx | 158 +++++++++++++++++++++++++++++-------- 1 file changed, 125 insertions(+), 33 deletions(-) diff --git a/examples/goal-test/cli.tsx b/examples/goal-test/cli.tsx index a177701..73146eb 100644 --- a/examples/goal-test/cli.tsx +++ b/examples/goal-test/cli.tsx @@ -1,6 +1,16 @@ -// End-to-end test of the `` component + `/goal` slash command + -// halt-gate fiber. This replaces the prior userspace orchestration — -// the runtime now owns the loop. We just set the goal and run. +// Userspace emulation of Claude Code's /goal: +// +// 1. Set a natural-language condition. +// 2. Run the agent. +// 3. When the agent halts, call a SEPARATE `infer` (the judge) to +// evaluate whether the condition holds against the transcript. +// 4. If the judge says ok=false, re-prompt the agent with the judge's +// reason. Loop until ok=true (or iteration budget is exhausted). +// +// No runtime modification: this is pure userspace orchestration around +// `agent.run` + `agent.events` + a second `infer` call. The judge is +// the same model family but isolated — it never sees the agent's tools, +// only the transcript. // // Run: // infisical run --silent -- npx tsx cli.tsx @@ -16,19 +26,114 @@ import { import { Agent, Block, - Goal, Messages, Workspace, } from "@flamecast/agentjsx/components" +import type { Event, InferFn } from "@flamecast/agentjsx" + +// --- Config --------------------------------------------------------------- const GOAL = "The assistant has greeted the user in pirate speak (e.g. 'Ahoy')." const INITIAL_PROMPT = "Greet me." +const MAX_ITERATIONS = 4 + +// --- Pretty-printing ------------------------------------------------------ const DIM = (s: string) => `\x1b[2m${s}\x1b[0m` const BLUE = (s: string) => `\x1b[34m${s}\x1b[0m` const GREEN = (s: string) => `\x1b[32m${s}\x1b[0m` +const RED = (s: string) => `\x1b[31m${s}\x1b[0m` const YELLOW = (s: string) => `\x1b[33m${s}\x1b[0m` +// --- Judge ---------------------------------------------------------------- + +interface Verdict { + ok: boolean + reason: string +} + +function buildTranscript(events: ReadonlyArray): string { + const lines: string[] = [] + for (const e of events) { + if (e.type === "user.message") { + lines.push(`[user] ${typeof e.content === "string" ? e.content : JSON.stringify(e.content)}`) + } else if (e.type === "assistant.message") { + if (e.content) lines.push(`[assistant] ${e.content}`) + } else if (e.type === "tool.call.started") { + lines.push(`[tool.call] ${e.tool_name}`) + } else if (e.type === "tool.result") { + lines.push(`[tool.result] ${e.content.slice(0, 400)}`) + } + } + return lines.join("\n") +} + +async function judge( + infer: InferFn, + condition: string, + transcript: string, +): Promise { + const system = [ + "You are evaluating a hook condition in agentjsx.", + "Judge whether the user-provided condition is met against the transcript below.", + 'Respond with a JSON object EXACTLY of the shape {"ok": boolean, "reason": string}.', + "Always include a reason. Quote specific text from the transcript when possible.", + "If there is no clear evidence, return ok: false with reason \"insufficient evidence\".", + ].join("\n") + + const userMsg = `CONDITION:\n${condition}\n\nTRANSCRIPT:\n${transcript}\n\nRespond with JSON only.` + + const res = await infer({ + system, + messages: [{ role: "user", content: userMsg }], + tools: [], + }) + + const raw = res.content.trim() + // Strip ```json fences if the model adds them + const cleaned = raw.replace(/^```(?:json)?\s*/i, "").replace(/```$/i, "").trim() + try { + const parsed = JSON.parse(cleaned) + if (typeof parsed?.ok === "boolean" && typeof parsed?.reason === "string") { + return parsed + } + return { ok: false, reason: `judge returned malformed JSON: ${raw}` } + } catch { + return { ok: false, reason: `judge returned non-JSON: ${raw}` } + } +} + +// --- Agent loop ----------------------------------------------------------- + +async function drainTurn(agent: ReturnType): Promise { + const startLen = (await agent.events()).length + let printed = startLen + while (true) { + await new Promise((r) => setTimeout(r, 100)) + const events = await agent.events() + for (let i = printed; i < events.length; i++) { + const e = events[i]! + if (e.type === "assistant.message" && e.content) { + console.log(`${GREEN("agent")} ${e.content}`) + } else if (e.type === "tool.call.started") { + console.log(DIM(` ${YELLOW("→")} ${e.tool_name}`)) + } else if (e.type === "tool.result") { + const snippet = e.content.length > 80 ? `${e.content.slice(0, 80)}…` : e.content + console.log(DIM(` ${YELLOW("←")} ${snippet}`)) + } else if (e.type === "assistant.halted") { + console.log(DIM(` ${YELLOW("!")} halted: ${e.reason}`)) + } + } + printed = events.length + const last = events[events.length - 1] + const isTerminal = + (last?.type === "assistant.message" && !last.tool_calls?.length) || + last?.type === "assistant.halted" || + last?.type === "inference.failed" + if (isTerminal) break + } +} + async function main(): Promise { const apiKey = process.env.AI_GATEWAY_API_KEY if (!apiKey) { @@ -50,7 +155,6 @@ async function main(): Promise { You are a friendly assistant. Respond briefly to the user. - , @@ -60,41 +164,29 @@ async function main(): Promise { try { console.log(DIM(`goal: ${GOAL}`)) console.log("") - - await agent.run(`/goal ${GOAL}`) - console.log(`${BLUE("you")} /goal …`) - console.log(`${BLUE("you")} ${INITIAL_PROMPT}`) await agent.run(INITIAL_PROMPT) + await drainTurn(agent) - // Stream events. The halt-gate fiber will reprompt automatically - // on assistant.halted if the predicate fails; we just observe. - let printed = (await agent.events()).length - const startTs = Date.now() - while (Date.now() - startTs < 60_000) { - await new Promise((r) => setTimeout(r, 200)) + for (let i = 1; i <= MAX_ITERATIONS; i++) { const events = await agent.events() - for (let i = printed; i < events.length; i++) { - const e = events[i]! - if (e.type === "assistant.message" && e.content) { - console.log(`${GREEN("agent")} ${e.content}`) - } else if (e.type === "user.message") { - const c = typeof e.content === "string" ? e.content : JSON.stringify(e.content) - if (i > printed - 1) console.log(`${DIM(YELLOW("reprompt"))} ${c}`) - } else if (e.type === "assistant.halted") { - console.log(DIM(` ${YELLOW("!")} halted: ${e.reason}`)) - } + const transcript = buildTranscript(events) + console.log("") + console.log(DIM(` judging iteration ${i}/${MAX_ITERATIONS}…`)) + const verdict = await judge(infer, GOAL, transcript) + if (verdict.ok) { + console.log(GREEN(` ✓ goal met: ${verdict.reason}`)) + return } - printed = events.length - const last = events[events.length - 1] - // Terminal when last event is an unanswered halt (gate decided - // the predicate is satisfied) OR a clean assistant.message - // with no further tool calls and no reprompt queued. - if (last?.type === "assistant.halted") break + console.log(RED(` ✗ not met: ${verdict.reason}`)) + const reprompt = `[goal: ${GOAL}]: ${verdict.reason}` + console.log(`${BLUE("you")} ${reprompt}`) + await agent.run(reprompt) + await drainTurn(agent) } - console.log("") - console.log(GREEN("done")) + console.log(RED(`exhausted ${MAX_ITERATIONS} iterations without meeting goal`)) + process.exit(2) } finally { await agent.dispose() }