33These exercise ``conclave.cli.ask`` through Typer's ``CliRunner`` (no real keys,
44no network). Three concerns are pinned here:
55
6- * **Exit-code contract (#17).** A run that produces zero *usable* member answers,
7- or an Elite run whose ``decision_readiness`` is not ``ready``, exits non-zero
8- (code 1) on both the human and ``--json`` paths. Under ``--json`` the full JSON
9- payload is still emitted to stdout so a script can parse the result *and*
10- detect the failure via the exit code. Other runs with a usable answer exit 0.
6+ * **Exit-code contract (#17, extended by DSE-901).** A run that produces zero
7+ *usable* member answers, or an Elite run whose ``decision_readiness`` is not
8+ ``ready``, exits non-zero (code 1) on both the human and ``--json`` paths.
9+ Under ``--json`` the full JSON payload is still emitted to stdout so a script
10+ can parse the result *and* detect the failure via the exit code. A run with a
11+ usable answer whose judge/synthesizer step nonetheless failed
12+ (``CouncilResult.degraded``) exits a distinct code 3 rather than 0 -- see
13+ ``tests/test_modes.py`` / ``tests/test_synthesizer.py`` for the mode-level
14+ ``degraded`` behavior this exit code surfaces. Every other run with a usable
15+ answer exits 0.
1116* **Durable result output (#58).** Buffered runs can atomically persist the same
1217 complete JSON payload with user-private permissions, including before a failure exit.
1318 Invalid destinations and streaming combinations fail before council construction.
2833from conclave .config import ConclaveConfig
2934from conclave .models import CouncilResult , EliteResult , ModelAnswer
3035from conclave .verdict import CouncilVerdict
36+ from tests .conftest import make_response
3137
3238runner = CliRunner ()
3339
@@ -52,6 +58,14 @@ def patch_cli_config(monkeypatch):
5258 monkeypatch .setattr (cli , "load_config" , _config )
5359
5460
61+ def _system_text (messages ) -> str :
62+ """Return the system-role content of a message list, or '' if none."""
63+ for m in messages :
64+ if m .get ("role" ) == "system" :
65+ return m .get ("content" , "" )
66+ return ""
67+
68+
5569def test_no_members_human_exits_one (clear_keys , patch_cli_config ):
5670 """Plain (human) path: zero usable answers -> exit code 1."""
5771 result = runner .invoke (cli .app , ["ask" , "hello" ])
@@ -285,6 +299,125 @@ def test_unknown_mode_exits_two(patch_cli_config):
285299 assert result .exit_code == 2
286300
287301
302+ # --------------------------------------------------------------------------- #
303+ # Degraded-run exit code (DSE-901): members answered, judge/synthesizer failed.
304+ # --------------------------------------------------------------------------- #
305+
306+
307+ def test_adversarial_judge_failure_json_is_degraded_not_clean (
308+ monkeypatch , patch_cli_config , patch_call_model
309+ ):
310+ """DSE-901: members answer but the judge/synthesizer call fails -> degraded.
311+
312+ Reproduces the 2026-07-25 incident that filed this ticket: the Anthropic key
313+ had no credit, so the ``claude`` judge's call returned an HTTP 400 while
314+ ``grok``/``gemini`` still answered normally, and the CLI exited 0 with
315+ ``adversarial.verdict`` silently ``null``. The fix must surface
316+ ``degraded: true`` in the JSON payload and exit the new degraded code
317+ (``cli._DEGRADED_EXIT_CODE`` == 3), not the clean-pass code 0.
318+ """
319+ for var in ("XAI_API_KEY" , "GEMINI_API_KEY" , "ANTHROPIC_API_KEY" ):
320+ monkeypatch .setenv (var , "dummy-key" )
321+
322+ def handler (model , messages , ** kwargs ):
323+ if "judge of an adversarial review" in _system_text (messages ):
324+ raise RuntimeError ("HTTP 400: your credit balance is too low" )
325+ return make_response (f"answer from { model } )
326+
327+ patch_call_model (handler )
328+ result = runner .invoke (
329+ cli .app ,
330+ ["ask" , "hello" , "--council" , "grok,gemini" , "--mode" , "adversarial" , "--json" ],
331+ )
332+
333+ assert result .exit_code == cli ._DEGRADED_EXIT_CODE
334+ assert result .exit_code not in 0 , 1 , 2 )
335+ payload = json .loads (result .stdout )
336+ assert payload ["degraded" ] is True
337+ assert payload ["adversarial" ]["verdict" ] is None
338+ assert "credit balance" in payload ["adversarial" ]["verdict_error" ]
339+ assert payload ["synthesis_error" ] == payload ["adversarial" ]["verdict_error" ]
340+ # This is a PARTIAL run, not the hard "zero usable answers" failure (code 1):
341+ # the proposer/critic that didn't hit the judge path still answered.
342+ assert len (payload ["answers" ]) >= 1
343+ assert any (a ["error" ] is None and a ["answer" ] is not None for a in payload ["answers" ])
344+
345+
346+ def test_adversarial_judge_failure_human_exits_degraded_code (
347+ monkeypatch , patch_cli_config , patch_call_model
348+ ):
349+ """Same DSE-901 scenario on the human (non-``--json``) render path."""
350+ for var in ("XAI_API_KEY" , "GEMINI_API_KEY" , "ANTHROPIC_API_KEY" ):
351+ monkeypatch .setenv (var , "dummy-key" )
352+
353+ def handler (model , messages , ** kwargs ):
354+ if "judge of an adversarial review" in _system_text (messages ):
355+ raise RuntimeError ("HTTP 400: your credit balance is too low" )
356+ return make_response (f"answer from { model } )
357+
358+ patch_call_model (handler )
359+ result = runner .invoke (
360+ cli .app , ["ask" , "hello" , "--council" , "grok,gemini" , "--mode" , "adversarial" ]
361+ )
362+
363+ assert result .exit_code == cli ._DEGRADED_EXIT_CODE
364+ assert "No verdict" in result .output
365+ assert "credit balance" in result .output
366+
367+
368+ def test_adversarial_clean_pass_still_exits_zero (monkeypatch , patch_cli_config , patch_call_model ):
369+ """Regression: a fully successful adversarial run (proposer/critic/judge all
370+ succeed) is NOT degraded and keeps exiting 0 -- the new check must not fire
371+ on the existing clean-pass path."""
372+ for var in ("XAI_API_KEY" , "GEMINI_API_KEY" , "ANTHROPIC_API_KEY" ):
373+ monkeypatch .setenv (var , "dummy-key" )
374+
375+ def handler (model , messages , ** kwargs ):
376+ system = _system_text (messages )
377+ if "judge of an adversarial review" in system :
378+ return make_response ("verdict text" )
379+ if "critic on an adversarial review" in system :
380+ return make_response (f"critique from { model } )
381+ return make_response (f"proposal from { model } )
382+
383+ patch_call_model (handler )
384+ result = runner .invoke (
385+ cli .app ,
386+ ["ask" , "hello" , "--council" , "grok,gemini" , "--mode" , "adversarial" , "--json" ],
387+ )
388+
389+ assert result .exit_code == 0
390+ payload = json .loads (result .stdout )
391+ assert payload ["degraded" ] is False
392+ assert payload ["adversarial" ]["verdict" ] == "verdict text"
393+
394+
395+ def test_all_members_failed_synthesize_mode_exits_one_not_degraded (
396+ monkeypatch , patch_cli_config , patch_call_model
397+ ):
398+ """Regression: zero usable answers keeps exit 1, never the new degraded code.
399+
400+ In default (synthesize) mode, every member erroring also sets
401+ ``synthesis_error`` ("no successful member answers to synthesize"), so
402+ ``result.degraded`` reads ``True`` here too -- the CLI's priority order must
403+ still pick the more severe hard-failure code (1) over the new degraded code.
404+ """
405+ for var in ("XAI_API_KEY" , "GEMINI_API_KEY" ):
406+ monkeypatch .setenv (var , "dummy-key" )
407+
408+ def handler (model , messages , ** kwargs ):
409+ raise RuntimeError ("provider down" )
410+
411+ patch_call_model (handler )
412+ result = runner .invoke (cli .app , ["ask" , "hello" , "--council" , "grok,gemini" , "--json" ])
413+
414+ assert result .exit_code == 1
415+ payload = json .loads (result .stdout )
416+ assert payload ["degraded" ] is True
417+ assert len (payload ["answers" ]) == 2
418+ assert all (a ["error" ] is not None for a in payload ["answers" ])
419+
420+
288421def _elite_cli_result (
289422 * ,
290423 completed : bool = True ,
0 commit comments