Repository navigation
feat(tier3): expose Harbor's Runta and Mosaic backends #183
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -23,7 +23,7 @@ | |
| import threading | ||
| import time | ||
| import tomllib | ||
| from collections.abc import Iterable, Iterator, Mapping | ||
| from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, as_completed, wait | ||
| from contextlib import ExitStack, contextmanager, suppress | ||
| from dataclasses import dataclass, field | ||
|
|
@@ -889,6 +889,21 @@ def _reserve_run_dir(results_root: Path, timestamp: str) -> Path: | |
| ), | ||
| "hyperbrowser": _DOCKER_HOST_ENV_VARS | frozenset({"HYPERBROWSER_API_KEY", "HYPERBROWSER_BASE_URL"}), | ||
| "vercel": frozenset({"VERCEL_OIDC_TOKEN", "VERCEL_PROJECT_ID", "VERCEL_TEAM_ID", "VERCEL_TOKEN"}), | ||
| "runta": frozenset({"RUNTA_CONFIG", "RUNTA_ENDPOINT", "RUNTA_TOKEN"}), | ||
|
Collaborator
Author
There was a problem hiding this comment. [P2] A relative The Runta SDK opens Fix: anchor
Collaborator
Author
There was a problem hiding this comment. Addressed in d245d41: |
||
| # The Mosaic SDK still reads the MAR_* names its MOSAIC_* names replaced. | ||
| "mosaic": frozenset( | ||
| { | ||
| "MAR_API_TOKEN", | ||
| "MAR_CONFIG", | ||
| "MAR_ENDPOINT", | ||
| "MOSAIC_API_TOKEN", | ||
|
Collaborator
Author
There was a problem hiding this comment. [P1] A skill's This set also decides which names a skill's Scenario:
A skill-controlled Fix: add
Collaborator
Author
There was a problem hiding this comment. Addressed in d245d41: |
||
| "MOSAIC_API_URL", | ||
| "MOSAIC_CONFIG", | ||
| "MOSAIC_REGISTRY_PASSWORD", | ||
| "MOSAIC_REGISTRY_USERNAME", | ||
| "MOSAIC_RETRIES", | ||
| } | ||
| ), | ||
| } | ||
| _BEDROCK_HOST_ENV_VARS = _AWS_HOST_ENV_VARS | { | ||
| "AWS_BEARER_TOKEN_BEDROCK", | ||
|
|
@@ -961,9 +976,12 @@ def _harbor_bin() -> str: | |
| "DOCKER_CONFIG", | ||
| "GOOGLE_APPLICATION_CREDENTIALS", | ||
| "LANGSMITH_CONFIG_FILE", | ||
| "MAR_CONFIG", | ||
| "MODAL_CONFIG_PATH", | ||
| "MOSAIC_CONFIG", | ||
| "NETRC", | ||
| "REQUESTS_CA_BUNDLE", | ||
| "RUNTA_CONFIG", | ||
| "SINGULARITY_AUTHFILE", | ||
| "SINGULARITY_CONFIGDIR", | ||
| "SKYPILOT_GLOBAL_CONFIG", | ||
|
|
@@ -1124,7 +1142,14 @@ def _nvidia_build_key_handoff( | |
| "ec2": frozenset({"iam_instance_profile", "strict_host_key_checking"}), | ||
| "gke": frozenset({"memory_limit_multiplier"}), | ||
| "modal": frozenset({"volumes"}), | ||
| # Persistent volumes, persisted sandboxes, SSH access, and alternate build | ||
| # inputs would let state or access outlive one isolated trial or skip | ||
| # evaluator-staged Dockerfile layers. | ||
| "mosaic": frozenset({"build_args", "build_target", "enable_ssh", "persist", "volume"}), | ||
|
Collaborator
Author
There was a problem hiding this comment. [P2] Mosaic snapshots are reused account-wide and never deleted Harbor reuses any existing snapshot named Fix: either pass
Collaborator
Author
There was a problem hiding this comment. Addressed in d245d41 with documentation rather than |
||
| "openshift": frozenset({"service_account_name"}), | ||
| # Direct mode runs commands in Runta's base runtime and ignores the task | ||
| # image; Harbor's automatic selection honors the task's Dockerfile or Compose. | ||
| "runta": frozenset({"mode"}), | ||
| "singularity": frozenset({"singularity_no_mount"}), | ||
| "use-computer": frozenset({"resources"}), | ||
| "vercel": frozenset({"ports"}), | ||
|
|
@@ -1566,6 +1591,62 @@ def invalid_strings(*names: str) -> list[str]: | |
| return [] | ||
|
|
||
|
|
||
| # Backend SDK configuration files that operators may name in the host environment. | ||
| _BACKEND_CONFIG_FILE_ENV_VARS: Mapping[str, tuple[str, ...]] = MappingProxyType( | ||
| {"mosaic": ("MOSAIC_CONFIG", "MAR_CONFIG"), "runta": ("RUNTA_CONFIG",)} | ||
| ) | ||
|
|
||
|
|
||
| def _backend_config_prerequisite_errors(env_mode: str) -> list[str]: | ||
| """Check Runta and Mosaic credential inputs against what Harbor's child will see.""" | ||
| errors: list[str] = [] | ||
| for name in _BACKEND_CONFIG_FILE_ENV_VARS.get(env_mode, ()): | ||
| raw = os.environ.get(name) | ||
| if raw is None: | ||
| continue | ||
| try: | ||
| is_file = bool(raw.strip()) and Path(raw).expanduser().is_file() | ||
| except (OSError, RuntimeError): | ||
| is_file = False | ||
| if not is_file: | ||
| errors.append(f"Harbor environment '{env_mode}' requires {name} to name an existing regular file.") | ||
| if env_mode == "mosaic" and not errors: | ||
| try: | ||
| from mosaic_sandbox.credentials import resolve_token | ||
| except ImportError: | ||
| return errors # Harbor's preflight reports the missing extra. | ||
| _token, source = resolve_token() | ||
| # The SDK accepts a Mosaic token in E2B_API_KEY for E2B migrations, but | ||
| # that variable belongs to the e2b backend and never reaches Mosaic. | ||
| if source == "e2b_environment": | ||
| errors.append( | ||
| "Harbor environment 'mosaic' does not forward E2B_API_KEY; set MOSAIC_API_TOKEN or run `mos login`." | ||
| ) | ||
| return errors | ||
|
|
||
|
|
||
| def _staged_task_environment_error(env_mode: str, task_dirs: Iterable[Path]) -> str | None: | ||
| """Reject staged task environments the selected backend cannot run.""" | ||
| if env_mode != "mosaic": | ||
| return None | ||
| for task_dir in task_dirs: | ||
| if (task_dir / "environment" / "docker-compose.yaml").exists(): | ||
| return ( | ||
| f"Harbor environment 'mosaic' runs one VM per trial and cannot run Docker Compose task " | ||
| f"'{task_dir.name}'; use a Dockerfile-only environment or another backend." | ||
| ) | ||
| return None | ||
|
|
||
|
|
||
| def _harbor_missing_dependency_summary(exc: ImportError) -> str: | ||
| """Keep Harbor's diagnosis, without its unpinned install commands or cloud-bundle hint.""" | ||
| lines = str(exc).strip().splitlines() | ||
| summary = lines[0] if lines else type(exc).__name__ | ||
| for suffix in (" Install it with:", " Install them with:"): | ||
| summary = summary.removesuffix(suffix) | ||
| return summary.strip().rstrip(".") | ||
|
|
||
|
|
||
| def _cwsandbox_prerequisite_errors(env_mode: str) -> list[str]: | ||
| """Check what Harbor's cwsandbox backend needs; Harbor 0.24 no longer checks it.""" | ||
| if harbor_environment_type(env_mode) != "cwsandbox": | ||
|
|
@@ -1838,6 +1919,8 @@ def _check_prerequisites( | |
|
|
||
| if cwsandbox_errors := _cwsandbox_prerequisite_errors(env_mode): | ||
| return cwsandbox_errors | ||
| if backend_config_errors := _backend_config_prerequisite_errors(env_mode): | ||
| return backend_config_errors | ||
| EnvironmentFactory.run_preflight(EnvironmentType(harbor_environment_type(env_mode))) | ||
| if env_mode == "ack": | ||
| ack_subprocess_env = ( | ||
|
|
@@ -1854,7 +1937,7 @@ def _check_prerequisites( | |
| ) | ||
| except ImportError as exc: | ||
| detail = redact_progress_detail( | ||
| _harbor_missing_dependency_summary(exc), | ||
| secret_values=secret_values_from_environment(os.environ), | ||
| ) | ||
| return [ | ||
|
|
@@ -4167,6 +4250,8 @@ def _persist_pre_execution_failure(errors: list[str]) -> dict[str, Any]: | |
| evaluator_skill_path=evaluator_skill_path, | ||
| arm_suffix=with_arm_suffix, | ||
| ) | ||
| if staged_environment_error := _staged_task_environment_error(env_mode, task_paths): | ||
| raise ValueError(staged_environment_error) | ||
| task_selectors = validate_case_ids(task.name for task in task_paths) | ||
| logical_case_ids = validate_case_ids(_native_entry_id(task) for task in task_paths) | ||
| case_id_by_task_selector = dict(zip(task_selectors, logical_case_ids, strict=True)) | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[P3] The CHANGELOG contradicts itself
The same Unreleased section still says "Exposed 23 Harbor 0.24 backends" and lists
runtaandmosaicas disabled.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Addressed in d245d41: the entry now says 25 backends and no longer lists
runtaormosaicas disabled.