diff --git a/CHANGELOG.md b/CHANGELOG.md index cb130c2c..b6925c58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] ### Added +- Added [Steel](https://github.com/steel-dev/steel-browser) as a managed remote browser runtime, covering both Steel Cloud (`STEEL_API_KEY`) and self-hosted deployments (`STEEL_BASE_URL`). Selecting `--browser-runtime steel` previously raised a not-implemented error. - Added `scripts/export_openeval.py`, an additive script exporting a batch's `rescore-summary.json` as an [EvalPort](https://github.com/adhabnr-ux/evalport) `ResultSet` Thanks to [@adhabnr-ux](https://github.com/adhabnr-ux). - Added a `--browser-runtime kernel` mode to the Harbor adapter that runs each task against one Kernel cloud browser, exposing only a credential-free CDP bridge to the agent, and finalizes the replay and deletes the browser during verification. diff --git a/README.md b/README.md index 756c5082..f81b35ee 100644 --- a/README.md +++ b/README.md @@ -337,7 +337,7 @@ Full registry: [`src/clawbench/runtime/harnesses/harnesses.yaml`](src/clawbench/ | I want to… | Where | | --- | --- | -| Use a managed remote browser instead of a local container | [`docs/browser-runtimes.md`](docs/browser-runtimes.md) — Kernel and Browserbase setup, options, and recordings | +| Use a managed remote browser instead of a local container | [`docs/browser-runtimes.md`](docs/browser-runtimes.md) — Kernel, Browserbase, and Steel setup, options, and recordings | | Run V2 through the Harbor framework (and run it fast) | [`docs/harbor.md`](docs/harbor.md) — conversion, judge wiring, concurrency, troubleshooting | | See every CLI command and flag | [`docs/cli.md`](docs/cli.md) | diff --git a/docs/browser-runtimes.md b/docs/browser-runtimes.md index 29ba4033..3f64c04d 100644 --- a/docs/browser-runtimes.md +++ b/docs/browser-runtimes.md @@ -2,7 +2,7 @@ By default ClawBench launches Chromium inside its own container. You can point it at a managed remote browser instead — useful when the host cannot run containers comfortably, or when you want the provider to handle scaling and session replay. -`--browser-runtime` accepts `local` (default), `kernel`, `browserbase`, `remote-cdp`, and `steel`. **`steel` is reserved and not implemented yet** — selecting it raises an error. +`--browser-runtime` accepts `local` (default), `kernel`, `browserbase`, `steel`, and `remote-cdp`. ## Local container (default) @@ -68,6 +68,56 @@ uv run clawbench-batch --models your-model --all-cases \ **Concurrency.** `--max-concurrent` defaults to **1** with Browserbase (2 for local runs) because parallel sessions consume provider quota. Raise it only as far as your plan's concurrent-session limit allows. +## Steel + +Works against both [Steel Cloud](https://steel.dev) and a self-hosted +[steel-browser](https://github.com/steel-dev/steel-browser); they expose the same +`/v1/sessions` API. + +For Steel Cloud, put the key in `.env.local`: + +```dotenv +STEEL_API_KEY=ste-... +``` + +For a self-hosted deployment, point ClawBench at it instead. No key is required +unless your deployment enforces one: + +```dotenv +STEEL_BASE_URL=http://localhost:3000 +``` + +Then select the runtime on a single or batch run: + +```bash +uv run clawbench-run test-cases/v1/ your-model \ + --browser-runtime steel + +uv run clawbench-batch --models your-model --all-cases \ + --browser-runtime steel +``` + +Steel runs reuse the same CDP action capture, screenshots, HTTP logging, and +request interception as local runs, so scoring is unchanged. Steel serves its +session replay from the session viewer rather than as a downloadable file, so a +Steel run has no local `recording.mp4`; the viewer URL is stored as +`browser_runtime.recording_url` in `run-meta.json`. + +Provider options are passed as JSON. Supported fields are `blockAds`, +`solveCaptcha`, `useProxy`, `proxyUrl`, `region`, `userAgent`, `stealthConfig`, +`sessionContext`, and `extensionIds`: + +```bash +uv run clawbench-batch --models your-model --all-cases \ + --browser-runtime steel \ + --browser-runtime-options '{"blockAds":true,"solveCaptcha":true}' +``` + +`dimensions` and `timeout` are set by ClawBench (1920x1080, and the task time +limit plus 120s of headroom) and are rejected as options. Batch concurrency +defaults to **1**; raise `--max-concurrent` only as far as your Steel plan or +self-hosted capacity allows. + ## Attaching to a browser you already run ```bash diff --git a/docs/cli.md b/docs/cli.md index 9f3bb254..0b607fdf 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -42,7 +42,7 @@ clawbench-run --human # human reference run | `--output-dir ` | `/test-output` | Where run directories are written | | `--no-build` | off | Skip building the container image (assumes it exists) | | `--no-upload` | off | Skip HuggingFace upload even if `HF_TOKEN` is configured | -| `--browser-runtime ` | `local` | `local`, `kernel`, `browserbase`, `remote-cdp` — see [`browser-runtimes.md`](browser-runtimes.md) | +| `--browser-runtime ` | `local` | `local`, `kernel`, `browserbase`, `steel`, `remote-cdp` — see [`browser-runtimes.md`](browser-runtimes.md) | | `--browser-cdp-url ` | — | CDP endpoint for `--browser-runtime remote-cdp` | | `--browser-runtime-options ` | — | Provider-specific options, e.g. `'{"region":"us-west-2"}'` | diff --git a/src/clawbench/runner/batch.py b/src/clawbench/runner/batch.py index 056c4557..c683b9fb 100644 --- a/src/clawbench/runner/batch.py +++ b/src/clawbench/runner/batch.py @@ -44,7 +44,7 @@ "claw-eval": "test-cases/claw-eval", } DEFAULT_CASES_SUITE = "v2" -MANAGED_BROWSER_RUNTIMES = frozenset({"browserbase", "kernel"}) +MANAGED_BROWSER_RUNTIMES = frozenset({"browserbase", "kernel", "steel"}) def load_models_yaml() -> dict: diff --git a/src/clawbench/runner/run_support/browser_runtime/providers.py b/src/clawbench/runner/run_support/browser_runtime/providers.py index 9a852082..3f118c31 100644 --- a/src/clawbench/runner/run_support/browser_runtime/providers.py +++ b/src/clawbench/runner/run_support/browser_runtime/providers.py @@ -31,6 +31,18 @@ "stealth", "tags", } +_STEEL_API_URL = "https://api.steel.dev" +_STEEL_ALLOWED_OPTIONS = { + "blockAds", + "extensionIds", + "proxyUrl", + "region", + "sessionContext", + "solveCaptcha", + "stealthConfig", + "useProxy", + "userAgent", +} DEFAULT_BROWSER_CDP_URL = os.environ.get( "CLAWBENCH_BROWSER_CDP_URL", "http://127.0.0.1:9222", @@ -156,6 +168,12 @@ def __init__(self, message: str, *, status: int | None = None) -> None: self.status = status +class _SteelApiError(RuntimeError): + def __init__(self, message: str, *, status: int | None = None) -> None: + super().__init__(message) + self.status = status + + def _parse_options(raw: str | None) -> dict[str, Any]: if not raw: return {} @@ -173,6 +191,19 @@ def _env_value(env: dict[str, str], key: str) -> str | None: return value if value else None +def _scrub_secret(text: str, secret: str | None) -> str: + """Remove an API key from an error string, including URL-encoded forms.""" + if not secret: + return text + for form in ( + secret, + urllib.parse.quote(secret, safe=""), + urllib.parse.quote_plus(secret, safe=""), + ): + text = text.replace(form, "[REDACTED]") + return text + + def _pick_free_port() -> int: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) @@ -232,22 +263,184 @@ def cleanup(self, session: BrowserSession) -> None: class SteelBrowserRuntimeProvider: + """Steel sessions, either Steel Cloud or a self-hosted steel-browser. + + Both speak the same ``/v1/sessions`` API; the only difference is that + Steel Cloud authenticates with ``steel-api-key`` while a self-hosted + deployment (``STEEL_BASE_URL``) usually takes no key at all. + """ + name = "steel" - default_recording_mode = "disabled" + default_recording_mode = "provider" - def __init__(self, *, options: dict[str, Any]) -> None: + def __init__( + self, + *, + api_key: str | None, + options: dict[str, Any], + api_url: str = _STEEL_API_URL, + ) -> None: + unknown = sorted(set(options) - _STEEL_ALLOWED_OPTIONS) + if unknown: + allowed = ", ".join(sorted(_STEEL_ALLOWED_OPTIONS)) + raise BrowserRuntimeError( + "steel runtime options contain unsupported field(s): " + f"{', '.join(unknown)}; allowed fields: {allowed}" + ) + for key in ("blockAds", "solveCaptcha", "useProxy"): + value = options.get(key) + if value is not None and not isinstance(value, bool): + raise BrowserRuntimeError(f"steel {key} option must be a boolean") + for key in ("proxyUrl", "region", "userAgent"): + value = options.get(key) + if value is not None and not isinstance(value, str): + raise BrowserRuntimeError(f"steel {key} option must be a string") + for key in ("sessionContext", "stealthConfig"): + value = options.get(key) + if value is not None and not isinstance(value, dict): + raise BrowserRuntimeError(f"steel {key} option must be a JSON object") + extension_ids = options.get("extensionIds") + if extension_ids is not None and not isinstance(extension_ids, list): + raise BrowserRuntimeError("steel extensionIds option must be a JSON array") + self.api_key = api_key self.options = options + self.api_url = api_url.rstrip("/") + + @property + def _is_cloud(self) -> bool: + return self.api_url == _STEEL_API_URL + + def _request( + self, + method: str, + path: str, + payload: dict[str, Any] | None = None, + ) -> dict[str, Any]: + data = ( + json.dumps(payload, separators=(",", ":")).encode() + if payload is not None + else None + ) + headers = {"Content-Type": "application/json"} + if self.api_key: + headers["steel-api-key"] = self.api_key + request = urllib.request.Request( + f"{self.api_url}{path}", + data=data, + method=method, + headers=headers, + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + raw = response.read() + except urllib.error.HTTPError as e: + if e.code in {401, 403}: + message = "Steel authentication failed" + elif e.code in {402, 429}: + message = "Steel quota or concurrency limit was exceeded" + else: + message = f"Steel API returned HTTP {e.code}" + raise _SteelApiError(message, status=e.code) from None + except (urllib.error.URLError, TimeoutError, OSError) as e: + reason = _scrub_secret(str(getattr(e, "reason", e)), self.api_key) + raise _SteelApiError(f"Steel API request failed: {reason}") from None + + try: + result = json.loads(raw) + except (json.JSONDecodeError, UnicodeDecodeError): + raise _SteelApiError("Steel API returned malformed JSON") from None + if not isinstance(result, dict): + raise _SteelApiError("Steel API returned non-object JSON") + return result + + def _release(self, session_id: str) -> str: + try: + self._request("POST", f"/v1/sessions/{session_id}/release") + except _SteelApiError as e: + if e.status in {404, 409}: + return "already_closed" + raise + return "released" def start(self, task: dict[str, Any], time_limit_s: int) -> BrowserSession: - raise BrowserRuntimeError( - "steel browser runtime is reserved but not implemented yet" + if not self.api_key and self._is_cloud: + raise BrowserRuntimeError( + "steel browser runtime requires STEEL_API_KEY (or STEEL_BASE_URL " + "pointing at a self-hosted steel-browser)" + ) + + # Steel expresses session timeouts in milliseconds; ClawBench works in + # seconds and adds the same 120s of headroom the other managed runtimes + # use for startup and teardown. + timeout_ms = min(86_400_000, max(60_000, (time_limit_s + 120) * 1000)) + payload = { + **self.options, + "dimensions": {"width": 1920, "height": 1080}, + "timeout": timeout_ms, + } + try: + result = self._request("POST", "/v1/sessions", payload) + except _SteelApiError as e: + raise BrowserRuntimeError(str(e)) from None + + session_id = result.get("id") + cdp_url = result.get("websocketUrl") + if not isinstance(session_id, str) or not session_id: + raise BrowserRuntimeError( + "Steel session response did not include a valid id" + ) + if not isinstance(cdp_url, str) or not cdp_url.startswith(("ws://", "wss://")): + try: + self._release(session_id) + except _SteelApiError: + pass + raise BrowserRuntimeError( + "Steel session response did not include a valid websocketUrl" + ) + + viewer_url = result.get("sessionViewerUrl") + if not isinstance(viewer_url, str) or not viewer_url.startswith( + ("http://", "https://") + ): + viewer_url = None + debug_url = result.get("debugUrl") + if not isinstance(debug_url, str) or not debug_url.startswith( + ("http://", "https://") + ): + debug_url = None + + return BrowserSession( + provider=self.name, + mode="remote", + session_id=session_id, + cdp_url=cdp_url, + viewer_url=viewer_url, + debug_url=debug_url, + # Steel keeps the rrweb session replay behind the same viewer URL + # instead of handing back a downloadable file, so a Steel run has + # no local recording.mp4. + recording_url=viewer_url, + metadata={ + "deployment": "cloud" if self._is_cloud else "self-hosted", + "region": result.get("region"), + "timeout_ms": timeout_ms, + "proxy_source": result.get("proxySource"), + "solve_captcha": result.get("solveCaptcha"), + }, + recording_mode="provider", ) def finalize(self, session: BrowserSession, output_dir: Path) -> None: pass def cleanup(self, session: BrowserSession) -> None: - session.cleanup_status = "not_required" + if not session.session_id: + session.cleanup_status = "not_required" + return + try: + session.cleanup_status = self._release(session.session_id) + except _SteelApiError as e: + raise BrowserRuntimeError(str(e)) from None class BrowserbaseRuntimeProvider: @@ -324,14 +517,7 @@ def _request( message = f"Browserbase API returned HTTP {e.code}" raise _BrowserbaseApiError(message, status=e.code) from None except (urllib.error.URLError, TimeoutError, OSError) as e: - reason = str(getattr(e, "reason", e)) - for secret in ( - self.api_key, - urllib.parse.quote(self.api_key, safe=""), - urllib.parse.quote_plus(self.api_key, safe=""), - ): - if secret: - reason = reason.replace(secret, "[REDACTED]") + reason = _scrub_secret(str(getattr(e, "reason", e)), self.api_key) raise _BrowserbaseApiError( f"Browserbase API request failed: {reason}" ) from None @@ -511,14 +697,7 @@ def _request_response( message = f"Kernel API returned HTTP {e.code}" raise _KernelApiError(message, status=e.code) from None except (urllib.error.URLError, TimeoutError, OSError) as e: - reason = str(getattr(e, "reason", e)) - for secret in ( - self.api_key, - urllib.parse.quote(self.api_key, safe=""), - urllib.parse.quote_plus(self.api_key, safe=""), - ): - if secret: - reason = reason.replace(secret, "[REDACTED]") + reason = _scrub_secret(str(getattr(e, "reason", e)), self.api_key) raise _KernelApiError(f"Kernel API request failed: {reason}") from None def _request( @@ -783,7 +962,20 @@ def make_browser_runtime_provider( }, ) if runtime == "steel": - return SteelBrowserRuntimeProvider(options=options) + api_url = _env_value(env, "STEEL_BASE_URL") or _STEEL_API_URL + api_key = _env_value(env, "STEEL_API_KEY") + # A self-hosted steel-browser normally runs unauthenticated, so the key + # is only mandatory when talking to Steel Cloud. + if not api_key and api_url.rstrip("/") == _STEEL_API_URL: + raise BrowserRuntimeError( + "steel browser runtime requires STEEL_API_KEY (or STEEL_BASE_URL " + "pointing at a self-hosted steel-browser)" + ) + return SteelBrowserRuntimeProvider( + api_key=api_key, + options=options, + api_url=api_url, + ) if runtime == "browserbase": api_key = _env_value(env, "BROWSERBASE_API_KEY") if not api_key: diff --git a/src/clawbench/tui.py b/src/clawbench/tui.py index 93ac165c..2428af5b 100644 --- a/src/clawbench/tui.py +++ b/src/clawbench/tui.py @@ -21,6 +21,7 @@ from rich.table import Table from rich.text import Text +from clawbench.runner.batch import MANAGED_BROWSER_RUNTIMES from clawbench.runner.run_support.harness_registry import HARNESS_REGISTRY from clawbench.utils.paths import ASSET_ROOT, WORKSPACE_ROOT, ensure_workspace_templates @@ -529,6 +530,7 @@ def _pick_browser_runtime(harness: str) -> str | None: "Browserbase cloud browser", value="browserbase", ), + questionary.Choice("Steel cloud browser", value="steel"), ] ) return questionary.select( @@ -684,6 +686,9 @@ def mode_single( else " [dim]Tip: open the Kernel live-view URL printed below\n" " to watch the cloud browser in real time.[/]" if browser_runtime == "kernel" + else " [dim]Tip: open the Steel session viewer URL printed below\n" + " to watch the cloud browser and replay the session.[/]" + if browser_runtime == "steel" else " [dim]Tip: once the container starts, open the noVNC URL\n" " printed below to watch the agent operate the browser\n" " in real-time.[/]" @@ -782,7 +787,7 @@ def mode_batch( case_args = ["--cases"] + [f"{cases_dir_name}/{c}" for c in selected_cases] case_summary = f"{len(selected_cases)} selected" - if browser_runtime in {"browserbase", "kernel"}: + if browser_runtime in MANAGED_BROWSER_RUNTIMES: recommended = 1 console.print( f" {browser_runtime.capitalize()} concurrency depends on the account limit; " diff --git a/tests/test_batch_and_tui_helpers.py b/tests/test_batch_and_tui_helpers.py index c234d237..395e3523 100644 --- a/tests/test_batch_and_tui_helpers.py +++ b/tests/test_batch_and_tui_helpers.py @@ -194,4 +194,4 @@ def fake_select(*args: object, **kwargs: object) -> _Prompt: selected = tui._pick_browser_runtime("openclaw") assert selected == "kernel" - assert captured_values == ["local", "kernel", "browserbase"] + assert captured_values == ["local", "kernel", "browserbase", "steel"] diff --git a/tests/test_browser_runtime.py b/tests/test_browser_runtime.py index 85e6568c..ccab2783 100644 --- a/tests/test_browser_runtime.py +++ b/tests/test_browser_runtime.py @@ -113,13 +113,6 @@ def test_local_browser_runtime_defaults_to_local_mode() -> None: assert isinstance(session.local_viewer_port, int) -def test_steel_provider_is_reserved_not_implemented() -> None: - provider = SteelBrowserRuntimeProvider(options={}) - - with pytest.raises(BrowserRuntimeError, match="not implemented"): - provider.start({}, 60) - - def test_browserbase_requires_api_key() -> None: provider = BrowserbaseRuntimeProvider(api_key=None, options={}) @@ -478,3 +471,279 @@ def test_redact_cdp_url_masks_common_secret_query_params() -> None: "wss://example.test/devtools?apiKey=%5BREDACTED%5D&" "jwt=%5BREDACTED%5D&token=%5BREDACTED%5D&x=ok" ) + + +def test_steel_requires_api_key_for_cloud() -> None: + provider = SteelBrowserRuntimeProvider(api_key=None, options={}) + + with pytest.raises(BrowserRuntimeError, match="STEEL_API_KEY"): + provider.start({}, 60) + with pytest.raises(BrowserRuntimeError, match="STEEL_API_KEY"): + make_browser_runtime_provider(_args(browser_runtime="steel"), {}) + + +def test_steel_self_hosted_does_not_require_api_key() -> None: + provider = make_browser_runtime_provider( + _args(browser_runtime="steel"), + {"STEEL_BASE_URL": "http://localhost:3000"}, + ) + + assert isinstance(provider, SteelBrowserRuntimeProvider) + assert provider.api_key is None + assert provider.api_url == "http://localhost:3000" + + +def test_steel_rejects_reserved_options() -> None: + with pytest.raises(BrowserRuntimeError, match="dimensions"): + SteelBrowserRuntimeProvider( + api_key="steel-secret", + options={"dimensions": {"width": 800, "height": 600}}, + ) + with pytest.raises(BrowserRuntimeError, match="timeout"): + SteelBrowserRuntimeProvider( + api_key="steel-secret", + options={"timeout": 300}, + ) + + +def test_steel_rejects_mistyped_options() -> None: + with pytest.raises(BrowserRuntimeError, match="blockAds"): + SteelBrowserRuntimeProvider(api_key="steel-secret", options={"blockAds": "yes"}) + with pytest.raises(BrowserRuntimeError, match="stealthConfig"): + SteelBrowserRuntimeProvider( + api_key="steel-secret", + options={"stealthConfig": ["humanize"]}, + ) + + +def test_steel_create_session_payload_and_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + requests: list[urllib.request.Request] = [] + + def fake_urlopen( + request: urllib.request.Request, + timeout: int, + ) -> _FakeResponse: + requests.append(request) + assert timeout == 15 + return _FakeResponse( + { + "id": "sess_abc", + "websocketUrl": ( + "wss://connect.steel.dev?sessionId=sess_abc&apiKey=steel-secret" + ), + "sessionViewerUrl": "https://app.steel.dev/sessions/sess_abc", + "debugUrl": ( + "https://app.steel.dev/sessions/sess_abc/debug?apiKey=steel-secret" + ), + "region": "lax", + "proxySource": "steel", + "solveCaptcha": False, + } + ) + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + provider = SteelBrowserRuntimeProvider( + api_key="steel-secret", + options={"blockAds": True, "region": "lax"}, + ) + + session = provider.start({}, 1800) + + assert len(requests) == 1 + request = requests[0] + assert request.get_method() == "POST" + assert request.full_url == "https://api.steel.dev/v1/sessions" + assert request.headers["Steel-api-key"] == "steel-secret" + assert isinstance(request.data, bytes) + assert json.loads(request.data) == { + "blockAds": True, + "region": "lax", + "dimensions": {"width": 1920, "height": 1080}, + "timeout": 1_920_000, + } + assert session.provider == "steel" + assert session.mode == "remote" + assert session.session_id == "sess_abc" + assert session.recording_mode == "provider" + assert session.recording_url == "https://app.steel.dev/sessions/sess_abc" + assert session.viewer_url == session.recording_url + assert session.metadata["deployment"] == "cloud" + metadata = session.to_metadata() + assert "steel-secret" not in json.dumps(metadata) + assert "apiKey=%5BREDACTED%5D" in metadata["cdp_url"] + assert "apiKey=%5BREDACTED%5D" in metadata["debug_url"] + + +def test_steel_timeout_is_bounded_in_milliseconds( + monkeypatch: pytest.MonkeyPatch, +) -> None: + payloads: list[dict[str, object]] = [] + + def fake_urlopen( + request: urllib.request.Request, + timeout: int, + ) -> _FakeResponse: + assert isinstance(request.data, bytes) + payloads.append(json.loads(request.data)) + return _FakeResponse( + { + "id": f"sess_{len(payloads)}", + "websocketUrl": f"wss://connect.steel.dev?sessionId=sess_{len(payloads)}", + } + ) + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + provider = SteelBrowserRuntimeProvider(api_key="steel-secret", options={}) + + provider.start({}, 1) + provider.start({}, 999_999) + + assert [payload["timeout"] for payload in payloads] == [121_000, 86_400_000] + + +def test_steel_start_releases_session_when_websocket_url_is_missing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, str]] = [] + + def fake_urlopen( + request: urllib.request.Request, + timeout: int, + ) -> _FakeResponse: + calls.append((request.get_method(), request.full_url)) + if request.full_url.endswith("/release"): + return _FakeResponse({"success": True}) + return _FakeResponse({"id": "sess_abc"}) + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + provider = SteelBrowserRuntimeProvider(api_key="steel-secret", options={}) + + with pytest.raises(BrowserRuntimeError, match="websocketUrl"): + provider.start({}, 60) + + assert calls == [ + ("POST", "https://api.steel.dev/v1/sessions"), + ("POST", "https://api.steel.dev/v1/sessions/sess_abc/release"), + ] + + +def test_steel_cleanup_releases_session( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, str]] = [] + + def fake_urlopen( + request: urllib.request.Request, + timeout: int, + ) -> _FakeResponse: + calls.append((request.get_method(), request.full_url)) + return _FakeResponse({"success": True}) + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + provider = SteelBrowserRuntimeProvider( + api_key="steel-secret", + options={}, + api_url="http://localhost:3000", + ) + session = BrowserSession( + provider="steel", + mode="remote", + session_id="sess_abc", + cdp_url="ws://localhost:3000/v1/sessions/sess_abc/cdp", + ) + + provider.cleanup(session) + + assert calls == [ + ("POST", "http://localhost:3000/v1/sessions/sess_abc/release"), + ] + assert session.cleanup_status == "released" + + +def test_steel_cleanup_treats_missing_session_as_already_closed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_urlopen( + request: urllib.request.Request, + timeout: int, + ) -> _FakeResponse: + raise urllib.error.HTTPError( + request.full_url, + 404, + "Not Found", + hdrs=Message(), + fp=None, + ) + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + provider = SteelBrowserRuntimeProvider(api_key="steel-secret", options={}) + session = BrowserSession( + provider="steel", + mode="remote", + session_id="sess_abc", + cdp_url="wss://connect.steel.dev?sessionId=sess_abc", + ) + + provider.cleanup(session) + + assert session.cleanup_status == "already_closed" + + +def test_steel_http_errors_do_not_expose_api_key( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_urlopen( + request: urllib.request.Request, + timeout: int, + ) -> _FakeResponse: + raise urllib.error.HTTPError( + request.full_url, + 401, + "steel-secret", + hdrs=Message(), + fp=None, + ) + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + provider = SteelBrowserRuntimeProvider(api_key="steel-secret", options={}) + + with pytest.raises(BrowserRuntimeError) as exc_info: + provider.start({}, 60) + + assert "authentication failed" in str(exc_info.value) + assert "steel-secret" not in str(exc_info.value) + + +def test_steel_network_errors_do_not_expose_api_key( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fake_urlopen( + request: urllib.request.Request, + timeout: int, + ) -> _FakeResponse: + raise urllib.error.URLError("connection refused for key steel-secret") + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + provider = SteelBrowserRuntimeProvider(api_key="steel-secret", options={}) + + with pytest.raises(BrowserRuntimeError) as exc_info: + provider.start({}, 60) + + assert "steel-secret" not in str(exc_info.value) + assert "[REDACTED]" in str(exc_info.value) + + +def test_steel_malformed_response_is_safe( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + urllib.request, + "urlopen", + lambda request, timeout: _FakeResponse(b"not-json"), + ) + provider = SteelBrowserRuntimeProvider(api_key="steel-secret", options={}) + + with pytest.raises(BrowserRuntimeError, match="malformed JSON"): + provider.start({}, 60)