diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh index 993b4c3bf47f3..0851c71909fad 100755 --- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh +++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-commands.sh @@ -175,6 +175,279 @@ ${JUNIT_TESTCASES} JEOF } +write_eval_metrics() { + local stream_log="$1" + local eval_result="$2" + local run_id="$3" + local prompt="$4" + local extractor="/opt/ai-helpers/plugins/prow-agent/scripts/extract_metrics.py" + local output="${ARTIFACT_DIR}/claude-session-metrics-autodl.json" + + if [[ ! -f "${extractor}" ]]; then + echo "WARNING: ${extractor} not found; cannot extract eval metrics." + return 1 + fi + + python3 - "${extractor}" "${stream_log}" "${eval_result}" "${output}" \ + "${BUILD_ID:-unknown}" "${run_id}" "${prompt}" <<'PYEOF' +import datetime +import importlib.util +import json +import pathlib +import sys +from collections import Counter + +extractor_path, stream_path, result_path, output_path, build_id, run_id, prompt = sys.argv[1:] + +spec = importlib.util.spec_from_file_location("extract_metrics", extractor_path) +metrics = importlib.util.module_from_spec(spec) +spec.loader.exec_module(metrics) +analyzed_at = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") +rows = [] + + +def empty_row(): + return { + key: "" if field_type == "string" else "0" + for key, field_type in metrics.SCHEMA.items() + } + + +def add_orchestrator_rows(): + stream_file = pathlib.Path(stream_path) + if not stream_file.is_file(): + return + + init = None + result = None + seen_tools = set() + tool_counts = Counter() + skills = [] + files_written = 0 + thinking_blocks = 0 + agent_tools = 0 + with stream_file.open(errors="replace") as stream: + for line in stream: + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if record.get("type") == "system" and record.get("subtype") == "init" and init is None: + init = record + if record.get("type") == "result": + result = record + if record.get("type") != "assistant": + continue + content = record.get("message", {}).get("content", []) + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") == "thinking": + thinking_blocks += 1 + continue + if block.get("type") != "tool_use": + continue + tool_id = str(block.get("id", "")) + if tool_id and tool_id in seen_tools: + continue + if tool_id: + seen_tools.add(tool_id) + tool_name = str(block.get("name", "")) + if not tool_name: + continue + tool_counts[tool_name] += 1 + tool_input = block.get("input", {}) + if not isinstance(tool_input, dict): + tool_input = {} + if tool_name == "Skill": + skill = str(tool_input.get("skill", "")) + if skill: + skills.append(skill) + elif tool_name == "Write": + files_written += 1 + elif tool_name == "Agent": + agent_tools += 1 + + if init is None or result is None: + return + model_usages = result.get("modelUsage", {}) + if not isinstance(model_usages, dict) or not model_usages: + usage = result.get("usage", {}) + if not isinstance(usage, dict): + usage = {} + model_usages = { + init.get("model", "unknown"): { + "inputTokens": usage.get("input_tokens", 0), + "outputTokens": usage.get("output_tokens", 0), + "cacheReadInputTokens": usage.get("cache_read_input_tokens", 0), + "cacheCreationInputTokens": usage.get("cache_creation_input_tokens", 0), + "costUSD": result.get("total_cost_usd", 0), + } + } + total_cost = sum(float(usage.get("costUSD", 0) or 0) for usage in model_usages.values()) + primary_model = str(init.get("model", "")) + plugins = ",".join( + str(plugin.get("name", "")) + for plugin in init.get("plugins", []) + if isinstance(plugin, dict) and plugin.get("name") + ) + for model, usage in model_usages.items(): + cost = float(usage.get("costUSD", 0) or 0) + duration_ratio = cost / total_cost if total_cost else 1 / len(model_usages) + input_tokens = int(usage.get("inputTokens", 0) or 0) + output_tokens = int(usage.get("outputTokens", 0) or 0) + cache_read = int(usage.get("cacheReadInputTokens", 0) or 0) + cache_create = int(usage.get("cacheCreationInputTokens", 0) or 0) + total_input = input_tokens + cache_read + cache_create + cache_hit_rate = cache_read / total_input * 100 if total_input else 0 + is_primary = str(model) == primary_model + session_id = str(result.get("session_id") or init.get("session_id", "")) + if len(model_usages) > 1: + session_id = f"{session_id}:{model}" + row = empty_row() + row.update({ + "session_id": session_id, + "model": str(model), + "claude_code_version": str(init.get("claude_code_version", "")), + "permission_mode": str(init.get("permissionMode", "")), + "entrypoint": "sdk-cli", + "prompt": prompt[:500], + "plugins_loaded": plugins, + "analyzed_at": analyzed_at, + "duration_ms": str(int(int(result.get("duration_ms", 0) or 0) * duration_ratio)), + "duration_api_ms": str(int(int(result.get("duration_api_ms", 0) or 0) * duration_ratio)), + "ttft_ms": str(int(result.get("ttft_ms", 0) or 0) if is_primary else 0), + "num_turns": str(int(result.get("num_turns", 0) or 0) if is_primary else 0), + "total_cost_usd": f"{cost:.6f}", + "input_tokens": str(input_tokens), + "output_tokens": str(output_tokens), + "cache_read_input_tokens": str(cache_read), + "cache_creation_input_tokens": str(cache_create), + "cache_hit_rate_pct": f"{cache_hit_rate:.1f}", + "total_tool_calls": str(sum(tool_counts.values()) if is_primary else 0), + "tool_call_breakdown": json.dumps(dict(tool_counts.most_common())) if is_primary else "{}", + "skills_invoked": ",".join(dict.fromkeys(skills)) if is_primary else "", + "files_written": str(files_written if is_primary else 0), + "num_thinking_blocks": str(thinking_blocks if is_primary else 0), + "num_subagents": str(agent_tools if is_primary else 0), + "is_error": "1" if result.get("is_error", False) else "0", + "terminal_reason": str(result.get("terminal_reason", "")), + "stop_reason": str(result.get("stop_reason", "")), + }) + rows.append(row) + + +def add_harness_rows(): + result_file = pathlib.Path(result_path) if result_path else None + if result_file is None or not result_file.is_file(): + return + with result_file.open() as stream: + result = json.load(stream) + usages = result.get("per_model_usage", {}) + if not isinstance(usages, dict) or not usages: + model = result.get("model", "unknown") + usage = result.get("token_usage", {}) + if not isinstance(usage, dict): + usage = {} + usages = {model: {**usage, "cost_usd": result.get("cost_usd", 0)}} + model_turns = result.get("per_model_turns", {}) + if not isinstance(model_turns, dict): + model_turns = {} + total_turns = sum(int(value or 0) for value in model_turns.values()) + total_cost = sum(float(usage.get("cost_usd", 0) or 0) for usage in usages.values()) + duration_s = float(result.get("duration_s", 0) or result.get("wall_clock_s", 0) or 0) + if not duration_s: + per_case = result.get("per_case", {}) + cases = per_case.values() if isinstance(per_case, dict) else per_case if isinstance(per_case, list) else [] + duration_s = sum( + float(case.get("duration_s", 0) or 0) + for case in cases + if isinstance(case, dict) + ) + duration_ms = max(1, int(duration_s * 1000)) + eval_params = result.get("eval_params", {}) + if not isinstance(eval_params, dict): + eval_params = {} + skill = str(eval_params.get("skill", "")) + skill_args = str(eval_params.get("skill_args", "")) + harness_prompt = f"/{skill} {skill_args}".strip() if skill else prompt + + for model, usage in usages.items(): + turns = int(model_turns.get(model, 0) or 0) + cost = float(usage.get("cost_usd", 0) or 0) + input_tokens = int(usage.get("input", 0) or 0) + output_tokens = int(usage.get("output", 0) or 0) + cache_read = int(usage.get("cache_read", 0) or 0) + cache_create = int(usage.get("cache_creation", 0) or 0) + total_input = input_tokens + cache_read + cache_create + if cost <= 0 and total_input <= 0 and output_tokens <= 0 and turns <= 0: + continue + if total_turns: + duration_ratio = turns / total_turns + elif total_cost: + duration_ratio = cost / total_cost + else: + duration_ratio = 1 / len(usages) + cache_hit_rate = cache_read / total_input * 100 if total_input else 0 + exit_code = int(result.get("exit_code", 0) or 0) + row = empty_row() + row.update({ + "session_id": f"eval-harness:{build_id}:{run_id}:{model}", + "model": str(model), + "claude_code_version": str(result.get("agent_version", "")).split(" ", 1)[0], + "permission_mode": "default", + "entrypoint": "agent-eval-harness", + "prompt": harness_prompt[:500], + "plugins_loaded": "agent-eval-harness", + "analyzed_at": analyzed_at, + "duration_ms": str(max(1, int(duration_ms * duration_ratio))), + "num_turns": str(turns or int(result.get("num_turns", 0) or 0)), + "total_cost_usd": f"{cost:.6f}", + "input_tokens": str(input_tokens), + "output_tokens": str(output_tokens), + "cache_read_input_tokens": str(cache_read), + "cache_creation_input_tokens": str(cache_create), + "cache_hit_rate_pct": f"{cache_hit_rate:.1f}", + "tool_call_breakdown": "{}", + "skills_invoked": skill, + "is_error": "1" if exit_code else "0", + "terminal_reason": "eval_failed" if exit_code else "eval_complete", + "stop_reason": "eval_failed" if exit_code else "end_turn", + }) + rows.append(row) + + +add_orchestrator_rows() +add_harness_rows() +if not rows: + sys.exit(1) + +output = pathlib.Path(output_path) +if output.is_file(): + try: + with output.open() as stream: + document = json.load(stream) + except (json.JSONDecodeError, OSError) as error: + raise RuntimeError(f"cannot read existing AutoDL artifact: {error}") from error + if not isinstance(document, dict) or document.get("schema") != metrics.SCHEMA: + raise RuntimeError("existing AutoDL artifact has an incompatible schema") + existing_rows = document.setdefault("rows", []) + if not isinstance(existing_rows, list): + raise RuntimeError("existing AutoDL artifact rows must be a list") + existing_rows.extend(rows) +else: + document = metrics.build_autodl(rows[0]) + document["rows"] = rows +temporary = output.with_suffix(output.suffix + ".tmp") +with temporary.open("w") as stream: + json.dump(document, stream, indent=4) +temporary.replace(output) +print(f"Wrote {len(rows)} eval metric row(s) to {output}") +PYEOF +} + for config in "${CONFIGS_TO_RUN[@]}"; do ELAPSED=$(( SECONDS - STEP_START )) if [[ ${ELAPSED} -ge ${STEP_TIMEOUT} ]]; then @@ -234,6 +507,7 @@ for config in "${CONFIGS_TO_RUN[@]}"; do EVAL_START=$(date +%s) THIS_EXIT=0 + STREAM_LOG="${ARTIFACT_DIR}/claude-eval-${config_name}.log" timeout 7200 claude \ --model "${CLAUDE_MODEL}" \ --plugin-dir "${EVAL_HARNESS_DIR}" \ @@ -241,7 +515,17 @@ for config in "${CONFIGS_TO_RUN[@]}"; do --output-format stream-json \ --max-turns "${EVAL_MAX_TURNS}" \ -p "/eval-run ${EVAL_RUN_ARGS}" \ - --verbose 2>&1 | tee "${ARTIFACT_DIR}/claude-eval-${config_name}.log" || THIS_EXIT=$? + --verbose 2>&1 | tee "${STREAM_LOG}" || THIS_EXIT=$? + + EVAL_RESULT=$(find "${AGENT_EVAL_RUNS_DIR:-eval/runs}" \ + -path "*/${RUN_ID}/run_result.json" -type f -print -quit 2>/dev/null || true) + if [[ -z "${EVAL_RESULT}" ]]; then + echo "WARNING: eval harness did not produce run_result.json for ${config_name}" + fi + if ! write_eval_metrics "${STREAM_LOG}" "${EVAL_RESULT}" "${RUN_ID}" \ + "/eval-run ${EVAL_RUN_ARGS}"; then + echo "WARNING: failed to write autodl eval metrics for ${config_name}" + fi THIS_DURATION=$(( $(date +%s) - EVAL_START )) TOTAL_DURATION=$(( TOTAL_DURATION + THIS_DURATION )) diff --git a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml index da961ca988c60..7868e87b9d868 100644 --- a/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml +++ b/ci-operator/step-registry/openshift/claude/agent-eval/openshift-claude-agent-eval-ref.yaml @@ -64,5 +64,5 @@ ref: documentation: |- Runs skill evaluations using the agent-eval-harness. Installs the eval harness plugin, runs /eval-run with the specified config and - model, and produces JUnit XML + artifacts. Eval traces and reports - are archived for download. + model, and produces JUnit XML, AutoDL session metrics, and artifacts. + Eval traces and reports are archived for download.