diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 10d37929..c07027cb 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -3,49 +3,178 @@ name: Deploy Docs on: push: branches: [main] + tags: ["v*"] paths: - ".github/workflows/docs.yml" - "docs/**" - "src/api/openapi.yml" workflow_dispatch: + inputs: + ref: + description: >- + Optional: tag/branch/sha to (re)build docs from (e.g. "v0.1.0" to + rebuild an already-released version). Leave empty to build the ref + this run was triggered from. Always trigger this from "main" (via + the branch selector) so the current workflow logic is used; only + the "ref" input controls which content gets built and published. + required: false + type: string concurrency: - group: pages + group: docs-${{ github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: build: - runs-on: ubuntu-22.04 - permissions: - contents: read + runs-on: ubuntu-latest + outputs: + version: ${{ steps.ctx.outputs.version }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref || github.sha }} - - uses: taiki-e/install-action@mdbook + - name: Determine target version + id: ctx + run: | + set -euo pipefail + + if [[ "${{ github.ref_type }}" == "tag" ]]; then + echo "version=${{ github.ref_name }}" >> "$GITHUB_OUTPUT" + exit 0 + fi + + manual_ref="${{ inputs.ref }}" + if [[ "${{ github.event_name }}" == "workflow_dispatch" && -n "$manual_ref" ]]; then + if [[ "$manual_ref" =~ ^v[0-9] ]]; then + echo "version=$manual_ref" >> "$GITHUB_OUTPUT" + else + echo "version=dev" >> "$GITHUB_OUTPUT" + fi + exit 0 + fi - - name: Configure GitHub Pages - uses: actions/configure-pages@v5 + echo "version=dev" >> "$GITHUB_OUTPUT" + + - uses: taiki-e/install-action@mdbook - name: Build docs run: | ln -sf ../../src/api/openapi.yml docs/src/openapi.yml mdbook build docs - - name: Upload GitHub Pages artifact - uses: actions/upload-pages-artifact@v4 + - name: Upload built docs + uses: actions/upload-artifact@v6 with: + name: docs-book path: docs/book + retention-days: 7 - deploy: + publish: needs: build - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest permissions: - pages: write - id-token: write - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} + contents: write + env: + VERSION: ${{ needs.build.outputs.version }} steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 + - uses: actions/checkout@v6 + + - name: Download built docs + uses: actions/download-artifact@v6 + with: + name: docs-book + path: book-output + + - name: Publish to gh-pages branch + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + + book_output="$(cd book-output && pwd)" + redirect_page="$(cd docs/redirects && pwd)/index.html" + + publish_dir="$(mktemp -d)" + cleanup() { + rm -rf "$publish_dir" + } + trap cleanup EXIT + + cd "$publish_dir" + git init -q + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git remote add origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + + max_attempts=5 + for attempt in $(seq 1 "$max_attempts"); do + if git ls-remote --exit-code --heads origin gh-pages >/dev/null 2>&1; then + git fetch --depth=1 origin gh-pages + git checkout -B gh-pages FETCH_HEAD + else + git checkout --orphan gh-pages + git rm -rf . >/dev/null 2>&1 || true + fi + + cp "$redirect_page" ./index.html + + if [[ "$VERSION" == "dev" ]]; then + rm -rf dev + mkdir -p dev + cp -a "$book_output"/. dev/ + else + rm -rf "$VERSION" + mkdir -p "$VERSION" + cp -a "$book_output"/. "$VERSION"/ + + if [[ -f versions.json ]]; then + existing="$(cat versions.json)" + else + existing='{"versions":[]}' + fi + echo "$existing" | jq \ + --arg v "$VERSION" \ + --arg d "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '.versions = ([{version: $v, date: $d}] + (.versions | map(select(.version != $v))))' \ + > versions.json.tmp + mv versions.json.tmp versions.json + + # Only promote this build to /latest/ if it is the highest known + # version. This lets maintainers rebuild an older tag (e.g. via + # workflow_dispatch with a "ref" input) to pick up a docs-only + # fix without accidentally rolling /latest/ back to stale content. + newest_version="$(jq -r '.versions[].version' versions.json | sort -V | tail -n1)" + if [[ "$newest_version" == "$VERSION" ]]; then + rm -rf latest + mkdir -p latest + cp -a "$book_output"/. latest/ + else + echo "Skipping /latest/ update: ${VERSION} is not the newest known version (${newest_version})." + fi + fi + + git add -A + if git diff --cached --quiet; then + echo "No documentation changes to publish for ${VERSION}." + exit 0 + fi + + git commit -q -m "chore: publish ${VERSION} docs for ${GITHUB_SHA}" + + if git push origin gh-pages; then + echo "Published ${VERSION} docs." + exit 0 + fi + + echo "Push rejected (attempt ${attempt}/${max_attempts}), retrying with latest gh-pages..." + git checkout -q --detach + git branch -D gh-pages + sleep $((attempt * 3)) + done + + echo "::error::Failed to push docs to gh-pages branch after ${max_attempts} attempts" + exit 1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fa032955..f9bacd25 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -83,39 +83,59 @@ jobs: - name: Stage artifacts run: | - mkdir -p dist/bundle/ublk - cp target/release/server dist/bundle/server - strip dist/bundle/server - cp target/release/uvm-ublk-daemon dist/bundle/ublk/uvm-ublk-daemon - strip dist/bundle/ublk/uvm-ublk-daemon - cp config/default.toml dist/bundle/default.toml - - STAGE_HOME="$PWD/dist/tmp-aenv" - STAGE_DEPS="$STAGE_HOME/deps" - mkdir -p "$STAGE_HOME" - AENV_HOME_PATH="$STAGE_HOME" dist/bundle/server --setup-only || { - echo "ERROR: --setup-only failed; check network and dependency downloads" - exit 1 - } - - test -d "$STAGE_DEPS" || { - echo "ERROR: deps dir missing at $STAGE_DEPS after --setup-only" - exit 1 - } - test -f "$STAGE_DEPS/overlaybd/etc/overlaybd/overlaybd.json" || { - echo "ERROR: missing staged overlaybd default config after --setup-only" - exit 1 - } - cp -a "$STAGE_DEPS" dist/bundle/deps - mkdir -p dist/bundle/etc/overlaybd - cp "$STAGE_DEPS/overlaybd/etc/overlaybd/overlaybd.json" dist/bundle/etc/overlaybd/overlaybd.json - rm -rf "$STAGE_HOME" - tar -czf dist/aenv-server-linux-x86_64.tar.gz -C dist/bundle . + set -euo pipefail + + for MODE in kvm pvm; do + BUNDLE="dist/bundle-$MODE" + mkdir -p "$BUNDLE/ublk" + cp target/release/server "$BUNDLE/server" + strip "$BUNDLE/server" + cp target/release/uvm-ublk-daemon "$BUNDLE/ublk/uvm-ublk-daemon" + strip "$BUNDLE/ublk/uvm-ublk-daemon" + cp config/default.toml "$BUNDLE/default.toml" + + sed -i \ + "s/^virtualization_mode = .*/virtualization_mode = \"$MODE\"/" \ + "$BUNDLE/default.toml" + grep -qx "virtualization_mode = \"$MODE\"" "$BUNDLE/default.toml" || { + echo "ERROR: failed to configure virtualization_mode=$MODE" + exit 1 + } + + STAGE_HOME="$PWD/dist/tmp-aenv-$MODE" + STAGE_DEPS="$STAGE_HOME/deps" + mkdir -p "$STAGE_HOME" + AENV_VIRTUALIZATION_MODE="$MODE" \ + AENV_HOME_PATH="$STAGE_HOME" \ + "$BUNDLE/server" --setup-only || { + echo "ERROR: $MODE --setup-only failed; check network and dependency downloads" + exit 1 + } + + test -d "$STAGE_DEPS" || { + echo "ERROR: deps dir missing at $STAGE_DEPS after $MODE --setup-only" + exit 1 + } + test -f "$STAGE_DEPS/overlaybd/etc/overlaybd/overlaybd.json" || { + echo "ERROR: missing staged overlaybd default config for $MODE" + exit 1 + } + cp -a "$STAGE_DEPS" "$BUNDLE/deps" + mkdir -p "$BUNDLE/etc/overlaybd" + cp "$STAGE_DEPS/overlaybd/etc/overlaybd/overlaybd.json" "$BUNDLE/etc/overlaybd/overlaybd.json" + rm -rf "$STAGE_HOME" + if [[ "$MODE" == "kvm" ]]; then + ARCHIVE="dist/aenv-server-linux-x86_64.tar.gz" + else + ARCHIVE="dist/aenv-server-linux-x86_64-pvm.tar.gz" + fi + tar -czf "$ARCHIVE" -C "$BUNDLE" . + done - uses: actions/upload-artifact@v7 with: - name: server-bundle - path: dist/aenv-server-linux-x86_64.tar.gz + name: server-bundles + path: dist/aenv-server-linux-x86_64*.tar.gz release: name: Create GitHub Release @@ -154,6 +174,7 @@ jobs: dist/aenv-linux-aarch64 dist/aenv-darwin-x86_64 dist/aenv-darwin-aarch64 + dist/aenv-server-linux-x86_64-pvm.tar.gz dist/aenv-server-linux-x86_64.tar.gz - uses: actions/upload-artifact@v7 @@ -162,9 +183,17 @@ jobs: path: dist/ docker-publish: - name: Build and push Docker images + name: Build and push Docker image (${{ matrix.mode }}) needs: release runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: + - mode: kvm + version_suffix: "" + - mode: pvm + version_suffix: "-pvm" permissions: contents: read packages: write @@ -191,6 +220,8 @@ jobs: file: deploy/docker/Dockerfile.agentenv push: true cache-from: type=gha,scope=agentenv-runtime + build-args: | + AENV_VIRTUALIZATION_MODE=${{ matrix.mode }} tags: | - ghcr.io/${{ env.OWNER }}/aenv-server:${{ github.ref_name }} - ghcr.io/${{ env.OWNER }}/aenv-server:latest + ghcr.io/${{ env.OWNER }}/aenv-server:${{ github.ref_name }}${{ matrix.version_suffix }} + ghcr.io/${{ env.OWNER }}/aenv-server:latest${{ matrix.version_suffix }} diff --git a/CLAUDE.md b/CLAUDE.md index 577125bf..74a854a5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,7 +2,7 @@ ## What is AgentENV -AgentENV is a Rust workspace for running AI agents inside isolated, snapshot-capable Firecracker-based environments. It exposes an E2B-compatible HTTP API so agents can create, pause, resume, and reuse sandboxes. Requires a Linux host with `/dev/kvm` access. +AgentENV is a Rust workspace for running AI agents inside isolated, snapshot-capable Firecracker-based environments. It exposes an E2B-compatible HTTP API so agents can create, pause, resume, and reuse sandboxes. Requires a Linux host with `/dev/kvm` access and a host virtualization setup matching `virtualization_mode` (`kvm` by default; `pvm` requires x86_64 and `kvm_pvm`). ## Build, Lint, Test Commands @@ -28,7 +28,7 @@ make agentenv-server # shorthand for cargo adev codegen server make custom-extension-client # shorthand for cargo adev codegen custom-extension ``` -Dependency downloads, generated OverlayBD runtime configs, and OverlayBD packaging are provisioned automatically during server startup. Machine-wide KVM group access, ublk device permissions, OverlayBD system config, and network sysctls require a one-time root setup via `server --setup-host --runtime-user --runtime-group `; normal startup validates those prerequisites and fails with actionable errors when they are missing. +Dependency downloads, generated OverlayBD runtime configs, and OverlayBD packaging are provisioned automatically during server startup. Machine-wide `/dev/kvm` group access, ublk device permissions, OverlayBD system config, and network sysctls require a one-time root setup via `server --setup-host --runtime-user --runtime-group `; normal startup validates those prerequisites and the selected KVM/PVM mode, and fails with actionable errors when they are missing. AgentENV does not load or install `kvm_pvm`. All registry access goes through `regctl`: userImage manifest fetch, config blob fetch, layer download, tools drive image download (`src/setup/deps.rs::extract_ext4_from_ghcr`, unpacked with `umoci`), and OCI referrers lookup when `[image_resolver].try_referrers_overlaybd_prefixes` is non-empty (referrers lookup failures fall back to the source image). Server setup provisions both automatically: `regctl` is downloaded from the `[regclient]` entry in `config/deps_manifest.toml` to `/usr/local/bin/regctl`, and `umoci` is installed as a `[packages.runtime]` system package. `src/image/oci_image.rs` fetches the manifest via `regctl manifest get` and classifies it — standard OCI tar images trigger a full `regctl image copy` + per-layer conversion into local `.commit` files, while overlaybd-native images skip blob download entirely and emit a remote-ref `image.json` that the overlaybd runtime's `registryfs_v2` backend reads directly from the registry. User-facing image references are normalized by `ImageResolver` from template API `userImage` fields and CLI image arguments. For private registries referenced by `userImage`, run `docker login ` before starting the server; `write_generated_overlaybd_global_config` auto-detects `~/.docker/config.json` (or `$DOCKER_CONFIG/config.json`) and wires the overlaybd runtime's `credentialConfig.mode=file` so the runtime can authenticate too. @@ -57,7 +57,7 @@ sudo -E cargo test -p agentenv --test orchestrator_integration orchestrator:: sudo -E cargo test -p agentenv --test orchestrator_integration orchestrator::test_name ``` -Integration tests require root (network namespaces), `/dev/kvm`, and `AENV_CONFIG_PATH` pointing to a valid config. +Integration tests require root (network namespaces), `/dev/kvm`, host modules matching `AENV_VIRTUALIZATION_MODE`, and `AENV_CONFIG_PATH` pointing to a valid config. ## Architecture @@ -95,7 +95,7 @@ When changing code under `services/`, validate via `make -C services test` (or ` ### Per-Node Subsystems -Each node is an AgentENV server binary (`src/bin/server.rs`) running on a Linux host with `/dev/kvm`. It wires together: +Each node is an AgentENV server binary (`src/bin/server.rs`) running on a Linux host with `/dev/kvm` and one configured KVM/PVM mode. It wires together: **API layer** (`src/api/`): Axum HTTP server with OpenAPI-generated endpoint traits (`src/api/generated/` from `src/api/openapi.yml`) plus a reverse proxy. Implementations live in `src/api/impls/` (sandbox CRUD, snapshot CRUD, template-facing CRUD, auth, generic cursor-based pagination). The proxy (`src/api/proxy.rs`) forwards HTTP/WebSocket to sandboxes using routing headers (`x-agentenv-sandbox-id`, `x-agentenv-target-port`). diff --git a/README.md b/README.md index 1b7d6ba8..4ec1818f 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,8 @@ AgentENV (AENV) is a platform for running agent environments at scale, powering - **Linux kernel 6.8+**; the install script additionally requires **Ubuntu 24.04** (see *Quick Start* below for installation options) - `/dev/kvm` access for Firecracker microVM execution +If your server does not support standard KVM, see the [PVM deployment guide](https://kvcache-ai.github.io/AgentENV/deployment/pvm.html) before installing. + --- ## ⚡ Quick Start (Single Node) @@ -48,6 +50,8 @@ curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/in sudo systemctl start aenv ``` +If this installation fails because standard KVM is unavailable, follow the [PVM deployment guide](https://kvcache-ai.github.io/AgentENV/deployment/pvm.html) instead. + *Option B — Docker* Set up the server: diff --git a/config/default.toml b/config/default.toml index b14fdb9c..faefe75e 100644 --- a/config/default.toml +++ b/config/default.toml @@ -12,11 +12,15 @@ # override. AENV_RUNTIME_PATH separately defaults to "/run/aenv". home_path = "/var/lib/aenv" +# Node-wide mutually exclusive virtualization backend. Override with +# AENV_VIRTUALIZATION_MODE. +virtualization_mode = "kvm" + [firecracker] # Boot arguments passed to the guest kernel. This value takes precedence over # the DEFAULT_BOOT_ARGS constant in src/sandbox/firecracker/config.rs. # Keep the DAMON reclaim parameters in sync between both locations. -boot_args = "console=ttyS0 reboot=k panic=1 pci=off init=/init damon_reclaim.enabled=Y damon_reclaim.min_age=60000000 damon_reclaim.quota_ms=100 damon_reclaim.quota_sz=1073741824 damon_reclaim.quota_reset_interval_ms=1000 damon_reclaim.wmarks_high=900 damon_reclaim.wmarks_mid=700 damon_reclaim.wmarks_low=200 damon_reclaim.skip_anon=Y damon_reclaim.wmarks_interval=5000000" +boot_args = "console=ttyS0 reboot=k panic=1 pci=off mitigations=off init=/init damon_reclaim.enabled=Y damon_reclaim.min_age=60000000 damon_reclaim.quota_ms=100 damon_reclaim.quota_sz=1073741824 damon_reclaim.quota_reset_interval_ms=1000 damon_reclaim.wmarks_high=900 damon_reclaim.wmarks_mid=700 damon_reclaim.wmarks_low=200 damon_reclaim.skip_anon=Y damon_reclaim.wmarks_interval=5000000" # Optional allowlist for cold-start extraBootArgs prefixes. If omitted or empty, # no request-provided extra boot args are appended. # allowed_extra_boot_args_prefixes = ["aenv-custom."] diff --git a/config/deps_manifest.toml b/config/deps_manifest.toml index 40d7889b..efe075e9 100644 --- a/config/deps_manifest.toml +++ b/config/deps_manifest.toml @@ -1,11 +1,19 @@ -[firecracker] +[firecracker.kvm] version = "1.15.1-patch-v1" url = "https://pub-4ee15c400f554ab7a9eac3f5bc8f53de.r2.dev/firecracker-{version}-{arch}.tgz" -[kernel] +[firecracker.pvm] +version = "v1.17.0-next.1" +url = "https://github.com/kvcache-ai/firecracker-next/releases/download/{version}/firecracker-next-{version}-{arch}.tgz" + +[kernel.kvm] version = "vmlinux-6.1.175" url = "https://pub-4ee15c400f554ab7a9eac3f5bc8f53de.r2.dev/{version}" +[kernel.pvm] +version = "6.12.33-pvm" +url = "https://github.com/kvcache-ai/linux/releases/download/pvm-kernel-6.12.33/vmlinux-guest-6.12.33-pvm" + [tools] version = "0.1.0" url = "ghcr.io/zlzgithub-0801/agentenv-tools:{version}" @@ -67,4 +75,3 @@ jq = { default = "jq" } sudo = { default = "sudo" } umoci = { default = "umoci" } zstd = { default = "zstd" } - diff --git a/config/oss_default.toml b/config/oss_default.toml index 61fd24d0..ca34d6ef 100644 --- a/config/oss_default.toml +++ b/config/oss_default.toml @@ -10,11 +10,14 @@ # AENV_SNAPSHOT_LOCAL_CACHE_PATH home_path = "../env" +# Node-wide mutually exclusive virtualization backend. Override with +# AENV_VIRTUALIZATION_MODE. +virtualization_mode = "kvm" # OSS validation keeps its downloaded dependencies directly under ../env. deps_path = "../env" [firecracker] -boot_args = "console=ttyS0 reboot=k panic=1 pci=off init=/init" +boot_args = "console=ttyS0 reboot=k panic=1 pci=off mitigations=off init=/init" # Optional allowlist for cold-start extraBootArgs prefixes. If omitted or empty, # no request-provided extra boot args are appended. # allowed_extra_boot_args_prefixes = ["aenv-custom."] diff --git a/crates/e2e-tests/tests/snapshot_oss_e2e_test.rs b/crates/e2e-tests/tests/snapshot_oss_e2e_test.rs index 803395c8..08156cde 100644 --- a/crates/e2e-tests/tests/snapshot_oss_e2e_test.rs +++ b/crates/e2e-tests/tests/snapshot_oss_e2e_test.rs @@ -115,6 +115,7 @@ async fn snapshot_oss_publish_and_resolve_remote_managed_layers() -> Result<()> startup: None, resources: SandboxResources::default(), runtime_versions: test_runtime_versions(), + virtualization_mode: ConfigManager::global_config().virtualization_mode, image_configs: agentenv::types::ImageConfigs::new(), custom_extension_params: None, }, @@ -209,6 +210,7 @@ async fn snapshot_oss_resolve_alias_cleans_up_stale_binding() -> Result<()> { startup: None, resources: SandboxResources::default(), runtime_versions: test_runtime_versions(), + virtualization_mode: ConfigManager::global_config().virtualization_mode, image_configs: agentenv::types::ImageConfigs::new(), custom_extension_params: None, }, @@ -263,6 +265,7 @@ async fn snapshot_oss_resolve_reports_missing_managed_layer() -> Result<()> { startup: None, resources: SandboxResources::default(), runtime_versions: test_runtime_versions(), + virtualization_mode: ConfigManager::global_config().virtualization_mode, image_configs: agentenv::types::ImageConfigs::new(), custom_extension_params: None, }, @@ -321,6 +324,7 @@ async fn snapshot_oss_delete_by_alias_removes_manifest_and_listing() -> Result<( startup: None, resources: SandboxResources::default(), runtime_versions: test_runtime_versions(), + virtualization_mode: ConfigManager::global_config().virtualization_mode, image_configs: agentenv::types::ImageConfigs::new(), custom_extension_params: None, }, diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index f87621ef..bca0e3c0 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -4,6 +4,7 @@ x-agentenv-base: &agentenv-base working_dir: /workspace environment: &agentenv-environment AENV_CONFIG_PATH: /workspace/config/default.toml + AENV_VIRTUALIZATION_MODE: ${AENV_VIRTUALIZATION_MODE:-kvm} API_ADDR: 0.0.0.0:8000 AENV_UBLK_DAEMON_BINARY_PATH: /usr/local/bin/uvm-ublk-daemon AENV_OBSERVABILITY_SCHEDULER_REPORT_ENABLED: "true" diff --git a/deploy/docker/Dockerfile.agentenv b/deploy/docker/Dockerfile.agentenv index e6128cfa..cb93dd7d 100644 --- a/deploy/docker/Dockerfile.agentenv +++ b/deploy/docker/Dockerfile.agentenv @@ -1,5 +1,7 @@ # syntax=docker/dockerfile:1.7 +ARG AENV_VIRTUALIZATION_MODE=kvm + FROM rust:1-bookworm AS chef WORKDIR /build @@ -90,15 +92,18 @@ RUN mkdir -p /workspace/config # `/usr/local/bin/regctl`. The final stage copies that tree as its own clean # layer, keeping the deps payload independent of the server binary. FROM runtime-base AS deps-stage +ARG AENV_VIRTUALIZATION_MODE COPY --from=builder --chmod=0755 /out-server /server COPY config/default.toml /workspace/config/default.toml ENV AENV_CONFIG_PATH=/workspace/config/default.toml ENV AENV_HOME_PATH=/workspace/env +ENV AENV_VIRTUALIZATION_MODE=${AENV_VIRTUALIZATION_MODE} RUN DEBIAN_FRONTEND=noninteractive /server --setup-only # final image. FROM runtime-base +ARG AENV_VIRTUALIZATION_MODE # overlaybd-apply validates its built-in default service config path even when # AgentENV passes an explicit generated config at runtime. @@ -118,6 +123,7 @@ COPY --from=builder --chmod=0755 /out-server /server COPY config/default.toml /workspace/config/default.toml ENV AENV_CONFIG_PATH=/workspace/config/default.toml ENV AENV_HOME_PATH=/workspace/env +ENV AENV_VIRTUALIZATION_MODE=${AENV_VIRTUALIZATION_MODE} ENV AENV_UBLK_DAEMON_BINARY_PATH=/usr/local/bin/uvm-ublk-daemon # Authoritative final `--setup-only`. If the pre-bake above produced exactly diff --git a/deploy/k8s/base/agentenv-daemonset.yaml b/deploy/k8s/base/agentenv-daemonset.yaml index f509d084..fd608752 100644 --- a/deploy/k8s/base/agentenv-daemonset.yaml +++ b/deploy/k8s/base/agentenv-daemonset.yaml @@ -23,6 +23,8 @@ spec: env: - name: AENV_CONFIG_PATH value: /workspace/config/agentenv.toml + - name: AENV_VIRTUALIZATION_MODE + value: "kvm" - name: API_ADDR value: "0.0.0.0:8000" - name: AENV_UBLK_DAEMON_BINARY_PATH diff --git a/docs/book.toml b/docs/book.toml index 5e3b8f95..a4fa992f 100644 --- a/docs/book.toml +++ b/docs/book.toml @@ -10,3 +10,5 @@ build-dir = "book" [output.html] git-repository-url = "https://github.com/kvcache-ai/AgentENV" edit-url-template = "https://github.com/kvcache-ai/AgentENV/edit/main/docs/{path}" +additional-css = ["theme/version-selector.css"] +additional-js = ["theme/version-selector.js"] diff --git a/docs/redirects/index.html b/docs/redirects/index.html new file mode 100644 index 00000000..21f0e51a --- /dev/null +++ b/docs/redirects/index.html @@ -0,0 +1,15 @@ + + + + + AgentENV Documentation + + + + + +

Redirecting to the latest AgentENV documentation

+ + diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 76c4d02e..d36c70c4 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -13,6 +13,7 @@ - [Docker Compose (Multi-Node Simulation)](./deployment/docker-compose.md) - [Kubernetes (Multi-Node)](./deployment/kubernetes.md) - [Manual Compile (Single Node)](./deployment/manual-compile.md) +- [PVM Deployment (When KVM Is Unavailable)](./deployment/pvm.md) # Configuration diff --git a/docs/src/configuration/env-vars.md b/docs/src/configuration/env-vars.md index 0812a135..92278dde 100644 --- a/docs/src/configuration/env-vars.md +++ b/docs/src/configuration/env-vars.md @@ -27,6 +27,7 @@ These variables are consumed by the repository's Docker Compose and Kubernetes h | `AENV_HOME_PATH` | `/var/lib/aenv` | Override the base directory from which AgentENV derives local state, caches, logs, generated configs, and downloaded dependencies. Component-specific path settings remain available as advanced overrides. | | `AENV_RUNTIME_PATH` | `/run/aenv` | Override the transient runtime directory used for network namespace mount points and the default ublk daemon socket. | | `AENV_DEPS_PATH` | `$AENV_HOME/deps` | Override root directory for auto-downloaded runtime assets (Firecracker, kernel, tools drive). | +| `AENV_VIRTUALIZATION_MODE` | `kvm` | Select the node virtualization mode. Leave unset for normal installations; set to `pvm` only when following the [PVM Deployment](../deployment/pvm.md) guide. | | `AENV_SNAPSHOT_LOCAL_CACHE_PATH` | `$AENV_HOME/snapshot-local-cache` | Override the snapshot manager's node-local artifact/cache root | | `AENV_SNAPSHOT_STORE` | `$AENV_HOME/snapshot-store` | Override the posix_fs snapshot repository root directory | | `AENV_UBLK_DAEMON_BINARY_PATH` | `$AENV_HOME/ublk/uvm-ublk-daemon` | Override path to the `uvm-ublk-daemon` binary | diff --git a/docs/src/configuration/reference.md b/docs/src/configuration/reference.md index aa3dc709..a2d18756 100644 --- a/docs/src/configuration/reference.md +++ b/docs/src/configuration/reference.md @@ -15,6 +15,10 @@ cargo run --bin server -- --config /path/to/config.toml | `home_path` | string | `"/var/lib/aenv"` | Base directory for local AgentENV state. Overridden by `AENV_HOME_PATH` | | `runtime_path` | string | `"/run/aenv"` | Base directory for transient namespace and daemon-socket state. Overridden by `AENV_RUNTIME_PATH` | | `deps_path` | string | `"$AENV_HOME/deps"` | Root directory for auto-downloaded runtime assets. Overridden by `AENV_DEPS_PATH` | +| `virtualization_mode` | `"kvm"` or `"pvm"` | `"kvm"` | Virtualization mode for this node. Keep the default unless following the [PVM Deployment](../deployment/pvm.md) guide. Overridden by `AENV_VIRTUALIZATION_MODE` | + +Snapshots and paused sandboxes can only be restored in the mode in which they +were created. `$AENV_HOME` is a literal placeholder in state-path values, not a shell environment variable. AgentENV replaces it with the resolved `home_path` after @@ -24,8 +28,9 @@ placeholders are resolved against the directory containing the configuration file. Packaged runtime dependency versions and download URLs live in -`config/deps_manifest.toml`. `config.toml` should contain runtime behavior -and explicit local path overrides, not the default dependency catalog. +`config/deps_manifest.toml`. Only the dependencies for the selected mode are +installed. User configuration should contain runtime behavior and explicit +local path overrides, not the default dependency catalog. ## `[firecracker]` diff --git a/docs/src/deployment/docker-compose.md b/docs/src/deployment/docker-compose.md index 30fa7a2d..1dc4093f 100644 --- a/docs/src/deployment/docker-compose.md +++ b/docs/src/deployment/docker-compose.md @@ -18,6 +18,10 @@ Run a full multi-node stack on a single host using Docker Compose. This simulate - Docker and Docker Compose - `build-essential` (`sudo apt install -y build-essential`) +The checked-in Compose setup uses standard KVM. If the host does not support +it, read [PVM Deployment](./pvm.md) before adapting the runtime image and host +configuration. + ## Clone the Repository ```bash diff --git a/docs/src/deployment/docker.md b/docs/src/deployment/docker.md index e01956e9..84d01578 100644 --- a/docs/src/deployment/docker.md +++ b/docs/src/deployment/docker.md @@ -8,12 +8,16 @@ Run a single AgentENV node in a Docker container. This avoids installing the Rus - `/dev/kvm` access for Firecracker microVM execution - Docker +If the server does not support standard KVM, follow +[PVM Deployment](./pvm.md) for the required host setup and PVM image. + ## Build **Option A — Pre-built Image** ```bash docker pull ghcr.io/kvcache-ai/aenv-server:latest + curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/docker-setup.sh | sudo bash ``` diff --git a/docs/src/deployment/kubernetes.md b/docs/src/deployment/kubernetes.md index 9f70fbe1..99ef1245 100644 --- a/docs/src/deployment/kubernetes.md +++ b/docs/src/deployment/kubernetes.md @@ -19,12 +19,16 @@ Deploy AgentENV across a Kubernetes cluster with a gateway, scheduler, and runti ## Prerequisites -- Kubernetes worker nodes with **Linux kernel 6.8+** and `/dev/kvm` access +- Kubernetes worker nodes with **Linux kernel 6.8+** +- `/dev/kvm` access on every runtime worker - Runtime Pods run privileged - Docker - `build-essential` (`sudo apt install -y build-essential`) - `kubectl` with Kustomize support +The provided manifests use standard KVM. To prepare a separate PVM node pool +when standard KVM is unavailable, see [PVM Deployment](./pvm.md). + ## Clone the Repository ```bash git clone https://github.com/kvcache-ai/AgentENV.git diff --git a/docs/src/deployment/manual-compile.md b/docs/src/deployment/manual-compile.md index 974196d6..1c95d59c 100644 --- a/docs/src/deployment/manual-compile.md +++ b/docs/src/deployment/manual-compile.md @@ -11,6 +11,9 @@ If you want to skip building from source, see [Quick Start](../getting-started/q - Rust toolchain (stable) — install via [rustup](https://rustup.rs) - `sudo` access +If the server does not support standard KVM, follow +[PVM Deployment](./pvm.md) instead of this guide. + ## Clone the Repository ```bash diff --git a/docs/src/deployment/pvm.md b/docs/src/deployment/pvm.md new file mode 100644 index 00000000..b32f3b58 --- /dev/null +++ b/docs/src/deployment/pvm.md @@ -0,0 +1,345 @@ +# PVM Deployment + +> **Use this guide when standard KVM is unavailable**, which commonly happens on cloud VMs where nested virtualization is not exposed. If standard KVM already works, use the [Quick Start](../getting-started/quickstart.md) instead. + +> This feature is **EXPERIMENTAL**. The PVM feature has not yet been merged into the mainline Linux kernel, and the forked kernel may not receive the same level of testing and security updates as the mainline kernel. + +PVM, originally proposed in the paper [*PVM: Efficient Shadow Paging for Deploying Secure Containers in Cloud-native Environment*](https://dl.acm.org/doi/10.1145/3600006.3613158), is an alternative virtualization mode that can provide the KVM-compatible interface required by AgentENV without relying on conventional nested virtualization. After the PVM host environment is installed, AgentENV still uses `/dev/kvm` to create Firecracker microVMs. + +Compared with a normal KVM deployment, a PVM deployment adds two host-level steps: + +1. Install and boot a PVM-capable host kernel. +2. Load the PVM virtualization module before starting AgentENV. + +AgentENV then uses its PVM-specific Firecracker and guest-kernel artifacts. + +## Before You Begin + +PVM is not enabled by changing only an AgentENV configuration value. The host must first be prepared with a compatible PVM kernel. + +You need: + +- An x86_64 Linux server. +- Root access. +- Permission to install a host kernel and reboot the server. +- A DEB-based or RPM-based Linux distribution supported by the published PVM host-kernel packages, or the ability to build the kernel from source. +- Linux kernel 6.8 or newer for the remaining AgentENV requirements. + +AgentENV does **not** replace the running host kernel automatically. Prebuilt PVM host-kernel packages are published separately in the [`kvcache-ai/linux` releases](https://github.com/kvcache-ai/linux/releases). You must install the appropriate package, reboot into that kernel, and verify the PVM module before installing AgentENV. + +> Before changing kernels on a production server, confirm that you have console access or another recovery path in case the new kernel does not boot. + +## Host and Guest Kernel Compatibility + +The PVM host kernel and the kernel running inside the AgentENV microVM must use compatible PVM ABIs. An incompatible pair may prevent the guest from booting or cause unexpected runtime failures. + +For the most predictable setup, use host and guest kernels built from the same PVM kernel version. The PVM guest kernel packaged by AgentENV is based on the [`pvm-612` branch of `virt-pvm/linux`](https://github.com/virt-pvm/linux/tree/pvm-612), at Linux version **6.12.33**. The matching prebuilt host packages are published in [`kvcache-ai/linux` release `pvm-kernel-6.12.33`](https://github.com/kvcache-ai/linux/releases/tag/pvm-kernel-6.12.33). + +The host and guest require different kernel configuration options: + +- **Host kernel:** enable `CONFIG_KVM_PVM=m`. +- **Guest kernel:** enable `CONFIG_PVM_GUEST`. + +## How PVM Fits into AgentENV + +An AgentENV node runs in exactly one virtualization mode: + +| Mode | AgentENV setting | Host state | +|------|------------------|------------| +| Standard KVM | `virtualization_mode = "kvm"` | Standard KVM modules; `kvm_pvm` is not loaded | +| PVM | `virtualization_mode = "pvm"` | PVM-capable host kernel with `kvm_pvm` loaded | + +The modes are intentionally isolated: + +- Dependency provisioning installs only the selected Firecracker and guest kernel. +- Snapshots record the mode in which they were captured. +- Persisted paused sandboxes record their mode. +- A node refuses to restore state created in the other mode. + +Do not point KVM and PVM nodes at the same persisted-sandbox directory. If nodes share a snapshot repository, ensure workloads resume only on nodes using the mode in which the snapshot was created. + +## Step 1: Install a PVM-Capable Host Kernel + +Use the package format for your distribution. AgentENV only requires the kernel image and modules package. The separately published headers/development package is not required unless you need to build external kernel modules on the host. + +### Debian and Ubuntu + +Download the kernel image: + +```bash +curl -fLO \ + https://github.com/kvcache-ai/linux/releases/download/pvm-kernel-6.12.33/linux-image-6.12.33_6.12.33-7_amd64.deb +``` + +Install it and refresh the bootloader: + +```bash +sudo dpkg -i linux-image-6.12.33_6.12.33-7_amd64.deb +sudo update-grub +``` + +If another installed kernel has a higher version, select Linux `6.12.33` from the bootloader's advanced options or configure it as the default boot entry before rebooting. + +### RPM-Based Distributions (Fedora, RHEL, CentOS, TencentOS) + +Download the kernel package: + +```bash +curl -fLO \ + https://github.com/kvcache-ai/linux/releases/download/pvm-kernel-6.12.33/kernel-6.12.33_g91e9c9be4472-2.x86_64.rpm +``` + +Install it: + +```bash +sudo rpm -ivh --oldpackage kernel-6.12.33_g91e9c9be4472-2.x86_64.rpm +``` + +On systems using `grubby`, select the installed PVM kernel: + +```bash +sudo grubby --set-default /boot/vmlinuz-6.12.33-g91e9c9be4472 +sudo grubby --default-kernel +``` + +### Build from Source + +If the published packages are not compatible with the distribution, build the host kernel from the [`pvm-612` branch](https://github.com/kvcache-ai/linux/tree/pvm-612). Enable `CONFIG_KVM_PVM=m`, install the kernel and modules, and configure the bootloader according to the distribution's kernel-build documentation. + +### Reboot and Verify + +Reboot the host: + +```bash +sudo reboot +``` + +After reconnecting, confirm that the expected kernel is active: + +```bash +uname -r +``` + +Expected output: + +```text +# DEB package +6.12.33 + +# RPM package +6.12.33-g91e9c9be4472 +``` + +If `uname -r` reports the previous kernel, update the bootloader selection and reboot again before continuing. + +## Step 2: Load and Verify the PVM Module + +Load the module: + +```bash +sudo modprobe kvm_pvm +``` + +Verify that it is loaded: + +```bash +lsmod | grep kvm_pvm +test -d /sys/module/kvm_pvm +``` + +Verify that the KVM-compatible device is now available: + +```bash +ls -l /dev/kvm +``` + +The important result of the host setup is: + +- `/sys/module/kvm_pvm` exists. +- `/dev/kvm` exists. +- The AgentENV runtime account can open `/dev/kvm` for reading and writing. + +After confirming that the module loads successfully, configure the kernel to load it automatically at boot (different distributions may use different directories): + +```bash +echo kvm_pvm | sudo tee /etc/modules-load.d/kvm-pvm.conf +``` + +## Step 3: Install AgentENV in PVM Mode + +### Option A: Install Script + +On Ubuntu 24.04: + +```bash +curl -fsSL https://raw.githubusercontent.com/kvcache-ai/AgentENV/main/scripts/install.sh \ + | sudo AENV_VIRTUALIZATION_MODE=pvm bash + +sudo systemctl start aenv +``` + +The installer: + +- Downloads `aenv-server-linux-x86_64-pvm.tar.gz`. +- Installs the PVM Firecracker and guest-kernel artifacts. +- Writes `AENV_VIRTUALIZATION_MODE="pvm"` to `/etc/default/aenv`. +- Configures the service account's access to `/dev/kvm`. + +It does not install or load the PVM host kernel or `kvm_pvm`. + +### Option B: Docker + +Use the dedicated PVM image: + +```bash +docker pull ghcr.io/kvcache-ai/aenv-server:latest-pvm + +docker run --rm -it \ + --device /dev/kvm \ + --privileged \ + -v /dev:/dev \ + -p 8000:8000 \ + ghcr.io/kvcache-ai/aenv-server:latest-pvm +``` + +The image sets `AENV_VIRTUALIZATION_MODE=pvm` by default and contains only the PVM runtime artifacts. + +### Option C: Build from Source + +Select PVM for both dependency provisioning and server startup: + +```bash +export AENV_VIRTUALIZATION_MODE=pvm + +cargo run --bin server -- --setup-only +make start-server +``` + +You can also set the mode in the TOML configuration: + +```toml +virtualization_mode = "pvm" +``` + +The environment variable takes precedence over the TOML value. + +To build a PVM Docker image: + +```bash +docker build \ + --build-arg AENV_VIRTUALIZATION_MODE=pvm \ + -f deploy/docker/Dockerfile.agentenv \ + -t aenv:pvm . +``` + +## Step 4: Verify AgentENV + +For an install-script deployment, verify the persisted mode: + +```bash +grep AENV_VIRTUALIZATION_MODE /etc/default/aenv +``` + +Expected output: + +```text +AENV_VIRTUALIZATION_MODE="pvm" +``` + +Inspect startup status and logs: + +```bash +sudo systemctl status aenv +sudo journalctl -u aenv -f +``` + +Verify the API: + +```bash +curl http://127.0.0.1:8000/health +``` + +Once the server is healthy, template creation and sandbox operations are the same as in the standard [Quick Start](../getting-started/quickstart.md). + +## Multi-Node Deployment + +Use a consistent virtualization mode within a runtime pool. + +For Docker Compose: + +- Use the PVM runtime image. +- Export `AENV_VIRTUALIZATION_MODE=pvm`. +- Prepare the host before starting the Compose stack. + +For Kubernetes: + +- Label x86_64 worker nodes that boot the PVM kernel. +- Load `kvm_pvm` on each selected node. +- Use the PVM AgentENV image. +- Set `AENV_VIRTUALIZATION_MODE=pvm` in the runtime DaemonSet. +- Add a node selector or affinity rule so PVM Pods cannot run on KVM nodes. + +Avoid mixing KVM and PVM nodes in a pool that schedules from a shared set of snapshots unless the scheduler also enforces virtualization-mode affinity. + +## Troubleshooting + +### `PVM virtualization mode is only supported on x86_64 hosts` + +The current machine architecture is unsupported. Deploy the PVM node on an x86_64 server or use standard KVM. + +### `PVM mode requires the kvm_pvm host module to be loaded` + +The AgentENV mode is set to PVM, but the host module is not active. + +Check the running kernel and try loading the module: + +```bash +uname -r +sudo modprobe kvm_pvm +lsmod | grep kvm_pvm +``` + +If `modprobe` reports that the module cannot be found, the server is not running a compatible PVM host kernel. + +### `/dev/kvm` is missing after loading `kvm_pvm` + +Confirm that `kvm_pvm` loaded successfully and review the kernel log: + +```bash +lsmod | grep kvm_pvm +sudo dmesg | tail -n 100 +``` + +If the module is loaded but `/dev/kvm` is still absent, verify the PVM kernel installation and boot parameters with the kernel provider. + +### `/dev/kvm` is not accessible + +Check ownership and service-account groups: + +```bash +ls -l /dev/kvm +id aenv +``` + +After changing group membership, restart the service or user session. + +### AgentENV downloads or uses standard KVM artifacts + +Confirm the mode is present in the service environment: + +```bash +grep AENV_VIRTUALIZATION_MODE /etc/default/aenv +``` + +Then rerun provisioning: + +```bash +sudo AENV_VIRTUALIZATION_MODE=pvm \ + AENV_CONFIG_PATH=/var/lib/aenv/config/config.toml \ + AENV_HOME_PATH=/var/lib/aenv \ + /usr/local/bin/server --setup-only +``` + +### Snapshot or paused-sandbox mode mismatch + +The persisted state was created in the other virtualization mode. Restore it on a node using its original mode, or rebuild the workload and capture a new snapshot in the target mode. diff --git a/docs/src/getting-started/overview.md b/docs/src/getting-started/overview.md index a7aed99e..c3473bcb 100644 --- a/docs/src/getting-started/overview.md +++ b/docs/src/getting-started/overview.md @@ -38,3 +38,4 @@ AgentENV exposes an HTTP API. There are four ways to use it: - **[Quick Start](./quickstart.md)** — Install the server, run your first sandbox. Takes ~5 minutes on a supported Linux host. - **[Deployment](../deployment/manual-compile.md)** — Build from source, Docker Compose multi-node, or Kubernetes. +- **[PVM Deployment](../deployment/pvm.md)** — Use AgentENV on a server where standard KVM is unavailable. diff --git a/docs/src/getting-started/quickstart.md b/docs/src/getting-started/quickstart.md index 993c0940..58d4c59c 100644 --- a/docs/src/getting-started/quickstart.md +++ b/docs/src/getting-started/quickstart.md @@ -5,9 +5,12 @@ - **Linux kernel 6.8+**; the install script additionally requires **Ubuntu 24.04** - `/dev/kvm` access for Firecracker microVM execution +> If your server does not support standard KVM, use the dedicated +> [PVM Deployment](../deployment/pvm.md) guide instead. + The install script attempts to install missing download and checksum commands, -provisions KVM permissions, loads the `ublk_drv` kernel module, and downloads -all runtime assets on first run. +provisions `/dev/kvm` permissions, loads the `ublk_drv` kernel module, and +downloads the required AgentENV runtime assets. Installation requires root, but the installed service does not run as root. It uses a dedicated `aenv` system account with `CAP_NET_ADMIN` and @@ -102,6 +105,7 @@ aenv start ubuntu # starts a sandbox and attaches an interactive shel ## Next Steps - [Deployment](../deployment/manual-compile.md) — build from source, multi-node options +- [PVM Deployment](../deployment/pvm.md) — deploy when standard KVM is unavailable - [Core Concepts](../concepts/overview.md) — how sandboxes, templates, and snapshots work - [E2B](../integration/e2b.md) — SDK and CLI compatibility - [API Reference](../api/index.md) — full HTTP API diff --git a/docs/src/internals/architecture.md b/docs/src/internals/architecture.md index 8c648d94..db1f4e52 100644 --- a/docs/src/internals/architecture.md +++ b/docs/src/internals/architecture.md @@ -135,7 +135,9 @@ Memory snapshot restore uses ublk-backed overlaybd devices rather than userfault ## Per-Node Subsystems -Each node is an AgentENV server binary (`src/bin/server.rs`) on a Linux host with `/dev/kvm`. +Each node is an AgentENV server binary (`src/bin/server.rs`) on a Linux host +with `/dev/kvm` and one configured virtualization mode. KVM is the default; +PVM currently requires x86_64 and the `kvm_pvm` host module. | Subsystem | Location | Responsibility | |-----------|----------|---------------| @@ -261,6 +263,7 @@ make k8s-apply In Kubernetes deployments, AgentENV runtime nodes run as a privileged DaemonSet so each host gets exactly one runtime Pod with access to `/dev/kvm`, iptables/network-namespace operations, and a hostPath-backed workspace cache. +Runtime Pods on a host must all use the host's selected KVM/PVM mode. The deployment helpers materialize the DaemonSet ConfigMap from `config/default.toml` at render/apply time so AgentENV runtime config remains single-sourced. diff --git a/docs/src/internals/sandbox-testing.md b/docs/src/internals/sandbox-testing.md index dc13777d..83c09d61 100644 --- a/docs/src/internals/sandbox-testing.md +++ b/docs/src/internals/sandbox-testing.md @@ -167,7 +167,8 @@ resumed_sandbox.stop().await?; ### Runtime prerequisites - Linux host -- `/dev/kvm` accessible by current user +- `/dev/kvm` accessible by the current user, with the host modules matching + `virtualization_mode` (KVM by default; PVM requires x86_64 and `kvm_pvm`) - `debugfs` (e2fsprogs) if you use init injection or disk-inspection helpers ## 2) Global Config Manager (`src/cfg.rs`) @@ -516,7 +517,8 @@ resumed_sandbox.stop().await?; performs validation only and never invokes `sudo`. 3. Host setup installs a udev rule for `/dev/ublk-control`, `/dev/ublkc*`, and `/dev/ublkb*`, so the runtime group can access the control and dynamic device nodes. 4. Update `config/default.toml` paths, or point `AENV_CONFIG_PATH` to a custom config file. -5. Ensure `/dev/kvm` is accessible by the runtime user. +5. Ensure `/dev/kvm` is accessible by the runtime user and the configured + virtualization mode matches the host modules. 6. If you run template tests, ensure the host can run `regctl` (server setup installs it automatically from the `[regclient]` manifest entry) and access the registry for template `fromImage` resolution. 7. Run `scripts/tests/e2e/run_e2e.sh` for API-level E2E coverage. The runner exports `E2E_TEMPLATE_USER_IMAGE`. Suite `05_template_lifecycle.sh` also creates a template build with `E2E_SHORT_USER_IMAGE` to verify short-name image resolution. 8. Run `make test-agent-integration` to run the `agentenv` integration test modules in `tests/integration/` as a non-root user with the required capabilities, plus the Docker/MinIO-backed OSS snapshot repository test (`crates/e2e-tests/tests/snapshot_oss_e2e_test.rs`). diff --git a/docs/src/internals/services.md b/docs/src/internals/services.md index ae0f4a54..98dffb02 100644 --- a/docs/src/internals/services.md +++ b/docs/src/internals/services.md @@ -57,7 +57,8 @@ make k8s-apply # apply to cluster Deployment model: - `gateway`: Deployment + ClusterIP Service - `scheduler`: single-replica Deployment + ClusterIP Service -- `agentenv-node`: privileged DaemonSet with `/dev/kvm` and hostPath +- `agentenv-node`: privileged DaemonSet with `/dev/kvm`, one host-compatible + KVM/PVM mode, and hostPath - `agentenv-nodes`: headless Service for scheduler EndpointSlice discovery ## gRPC API diff --git a/docs/src/internals/template-builder-testing.md b/docs/src/internals/template-builder-testing.md index 37bd3cd7..b5b3e806 100644 --- a/docs/src/internals/template-builder-testing.md +++ b/docs/src/internals/template-builder-testing.md @@ -218,6 +218,7 @@ Repository/backend unit tests under `src/snapshot/repository/backends/` cover: - overlaybd base config paths are missing or invalid - `ublk` is disabled while using overlaybd build bases -- Linux host prerequisites for Firecracker / KVM / namespaces are missing +- Linux host prerequisites for Firecracker, the selected KVM/PVM mode, or + network namespaces are missing - repository alias conflicts during publish - runtime resolver cannot materialize local paths from committed artifacts diff --git a/docs/src/troubleshooting/common-issues.md b/docs/src/troubleshooting/common-issues.md index 3f47fa0a..401b3245 100644 --- a/docs/src/troubleshooting/common-issues.md +++ b/docs/src/troubleshooting/common-issues.md @@ -1,16 +1,40 @@ # Common Issues -## `/dev/kvm` not accessible +## `/dev/kvm` is missing or inaccessible -**Symptom**: Server fails to start with a KVM-related error. +**Symptom**: The server cannot find or open `/dev/kvm`. -**Solution**: Ensure your host has hardware virtualization enabled (Intel VT-x or AMD-V) and that `/dev/kvm` is readable by the current user. On most systems: +**Solution**: First check whether standard KVM is available: + +```bash +ls -l /dev/kvm +``` + +If the device exists, ensure it is readable and writable by the runtime user. +On most systems: ```bash sudo usermod -aG kvm $USER # Log out and back in for the group change to take effect ``` +If the cloud server does not expose standard KVM, follow +[PVM Deployment](../deployment/pvm.md). That guide covers the additional host +setup needed before starting AgentENV. + +## The configured virtualization mode does not match the host + +**Symptom**: Startup reports that KVM cannot run while the PVM module is +loaded, or that PVM requires additional host setup. + +**Solution**: Normal installations should use the default KVM mode. If the +host was prepared for PVM, follow [PVM Deployment](../deployment/pvm.md) and +ensure the service environment contains: + +```bash +AENV_VIRTUALIZATION_MODE=pvm +``` + ## Permission denied for network operations **Symptom**: Sandbox creation fails with network namespace or iptables errors. @@ -19,8 +43,8 @@ sudo usermod -aG kvm $USER its effective, permitted, and inheritable sets. The installed systemd unit configures these automatically. For a source checkout, use `make start-server` or `scripts/run-with-capabilities.sh `; do not run the whole -server as root. Also verify that the runtime account belongs to the `kvm` group -and can open `/dev/ublk-control`. +server as root. Also verify that the runtime account belongs to the `kvm` +group, can open `/dev/kvm`, and can open `/dev/ublk-control`. ## Sandbox namespaces are missing from `ip netns list` diff --git a/docs/theme/version-selector.css b/docs/theme/version-selector.css new file mode 100644 index 00000000..29f88ed3 --- /dev/null +++ b/docs/theme/version-selector.css @@ -0,0 +1,46 @@ +/* Styles for the version switcher injected by theme/version-selector.js */ + +.version-selector { + display: inline-flex; + align-items: center; + margin-right: 0.5rem; +} + +.version-preview-banner { + display: flex; + z-index: 1000; + position: sticky; + top: 0; + height: var(--menu-bar-height); + flex-wrap: wrap; + align-items: center; + justify-content: center; + gap: 0.5rem; + padding: 0.6em 1em; + border-bottom: 1px solid #f0c36d; + background: #fff7e6; + color: #6b4c00; + box-sizing: border-box; +} + +.version-preview-banner a { + font-weight: 600; + white-space: nowrap; +} + +/* Keep the banner readable on mdbook's dark themes */ +.navy .version-preview-banner, +.coal .version-preview-banner, +.ayu .version-preview-banner { + background: #3a2f10; + color: #f0dca0; + border-color: #a3801e; +} + +body:has(.version-preview-banner) .sidebar .sidebar-scrollbox { + top: var(--menu-bar-height); +} + +body:has(.version-preview-banner) .page #mdbook-menu-bar.bordered { + top: var(--menu-bar-height) !important; +} diff --git a/docs/theme/version-selector.js b/docs/theme/version-selector.js new file mode 100644 index 00000000..869340f0 --- /dev/null +++ b/docs/theme/version-selector.js @@ -0,0 +1,122 @@ +// Lightweight, framework-free version switcher for the AgentENV mdbook site. +// +// The site is deployed as a GitHub Pages *project* page, so every published +// version lives under a fixed base path: +// https://.github.io/AgentENV/{version}/... +// where {version} is one of: "dev" (unreleased preview built from `main`), +// "latest" (alias of the most recently released tag), or a release tag such +// as "v0.1.0". +// +// A JSON manifest at "/AgentENV/versions.json" (maintained by +// .github/workflows/docs.yml) lists all released tag versions. "dev" and +// "latest" always exist and are not required to be present in that manifest. +(function () { + "use strict"; + + var REPO_BASE = "/AgentENV/"; + + function currentVersionAndRest() { + var path = window.location.pathname; + if (path.indexOf(REPO_BASE) !== 0) { + return { version: "", rest: "" }; + } + var remainder = path.slice(REPO_BASE.length); + var parts = remainder.split("/"); + var version = parts.shift() || ""; + return { version: version, rest: parts.join("/") }; + } + + function insertPreviewBanner() { + var banner = document.createElement("div"); + banner.className = "version-preview-banner"; + + var text = document.createElement("span"); + text.textContent = + "You are viewing a preview built from the latest main branch. Some features described here may not be released yet."; + banner.appendChild(text); + + var link = document.createElement("a"); + link.href = REPO_BASE + "latest/"; + link.textContent = "View latest released docs \u2192"; + banner.appendChild(link); + + var content = document.getElementById("content") || document.body; + content.insertBefore(banner, content.firstChild); + } + + function navigateToVersion(version, rest) { + var target = REPO_BASE + version + "/" + rest; + fetch(target, { method: "HEAD" }) + .then(function (res) { + window.location.href = res.ok ? target : REPO_BASE + version + "/"; + }) + .catch(function () { + window.location.href = REPO_BASE + version + "/"; + }); + } + + function insertVersionSelector(versions, current) { + var known = versions.slice(); + if (current.version && known.indexOf(current.version) === -1) { + known.unshift(current.version); + } + + var wrapper = document.createElement("div"); + wrapper.className = "version-selector"; + + var select = document.createElement("select"); + select.setAttribute("aria-label", "Documentation version"); + + known.forEach(function (v) { + var opt = document.createElement("option"); + opt.value = v; + opt.textContent = v; + if (v === current.version) { + opt.selected = true; + } + select.appendChild(opt); + }); + + select.addEventListener("change", function () { + navigateToVersion(select.value, current.rest); + }); + + wrapper.appendChild(select); + + var menuBar = + document.querySelector(".menu-bar .right-buttons") || + document.querySelector(".right-buttons") || + document.querySelector(".menu-bar"); + if (menuBar) { + menuBar.insertBefore(wrapper, menuBar.firstChild); + } else { + document.body.appendChild(wrapper); + } + } + + document.addEventListener("DOMContentLoaded", function () { + var current = currentVersionAndRest(); + + if (current.version === "dev") { + insertPreviewBanner(); + } + + fetch(REPO_BASE + "versions.json", { cache: "no-cache" }) + .then(function (res) { + if (!res.ok) { + throw new Error("versions.json not available"); + } + return res.json(); + }) + .then(function (manifest) { + var releasedVersions = (manifest.versions || []).map(function (entry) { + return entry.version; + }); + var versions = ["latest", "dev"].concat(releasedVersions); + insertVersionSelector(versions, current); + }) + .catch(function () { + insertVersionSelector(["latest", "dev"], current); + }); + }); +})(); diff --git a/scripts/install.sh b/scripts/install.sh index a2dbce88..33de4e25 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -11,7 +11,7 @@ # service -> /etc/systemd/system/aenv.service # env -> /etc/default/aenv # Runs: sudo server --setup-only (provisions runtime dependencies) -# sudo server --setup-host (provisions KVM, ublk, and networking) +# sudo server --setup-host (provisions virtualization access, ublk, and networking) # # Usage: # curl -fsSL https://github.com/kvcache-ai/AgentENV/releases/latest/download/install.sh | sudo bash @@ -40,6 +40,15 @@ SERVICE_GROUP="${AENV_SERVICE_GROUP:-aenv}" SERVICE_FILE="/etc/systemd/system/${SERVICE_NAME}.service" ENV_FILE="/etc/default/${SERVICE_NAME}" RUNTIME_DIR="/run/aenv" +VIRTUALIZATION_MODE="${AENV_VIRTUALIZATION_MODE:-kvm}" + +case "$VIRTUALIZATION_MODE" in + kvm|pvm) ;; + *) + echo "error: unsupported AENV_VIRTUALIZATION_MODE=${VIRTUALIZATION_MODE@Q}; expected kvm or pvm" >&2 + exit 1 + ;; +esac ARCH="$(uname -m)" case "$ARCH" in @@ -57,7 +66,11 @@ if [[ "$OS" != "linux" ]]; then fi RELEASE_API="https://api.github.com/repos/${REPO}/releases/latest" -TARBALL="aenv-server-${OS}-${ARCH_TAG}.tar.gz" +if [[ "$VIRTUALIZATION_MODE" == "pvm" ]]; then + TARBALL="aenv-server-${OS}-${ARCH_TAG}-pvm.tar.gz" +else + TARBALL="aenv-server-${OS}-${ARCH_TAG}.tar.gz" +fi curl_get() { curl -fsSL --retry 5 --retry-delay 10 --retry-max-time 60 "$@" @@ -213,10 +226,12 @@ if [[ "$SKIP_SETUP" == "1" ]]; then else echo "Running dependency setup ..." sudo AENV_CONFIG_PATH="${CONFIG_PATH}" AENV_HOME_PATH="${DATA_DIR}" \ + AENV_VIRTUALIZATION_MODE="${VIRTUALIZATION_MODE}" \ "${INSTALL_DIR}/server" --setup-only - echo "Provisioning KVM, ublk, and host networking for ${SERVICE_USER} ..." + echo "Provisioning virtualization device access, ublk, and host networking for ${SERVICE_USER} ..." sudo AENV_CONFIG_PATH="${CONFIG_PATH}" AENV_HOME_PATH="${DATA_DIR}" \ + AENV_VIRTUALIZATION_MODE="${VIRTUALIZATION_MODE}" \ "${INSTALL_DIR}/server" --setup-host \ --runtime-user "$SERVICE_USER" --runtime-group "$SERVICE_GROUP" fi @@ -240,6 +255,7 @@ API_ADDR="127.0.0.1:8000" AENV_CONFIG_PATH="${CONFIG_PATH}" AENV_HOME_PATH="${DATA_DIR}" AENV_RUNTIME_PATH="${RUNTIME_DIR}" +AENV_VIRTUALIZATION_MODE="${VIRTUALIZATION_MODE}" EOF ENV_FILE_STATUS="written" else @@ -262,6 +278,7 @@ EOF found_config=0 found_home=0 found_runtime=0 + found_virtualization=0 while IFS= read -r line || [[ -n "$line" ]]; do case "$line" in AENV_CONFIG_PATH=*) @@ -276,6 +293,10 @@ EOF printf 'AENV_RUNTIME_PATH="%s"\n' "$RUNTIME_DIR" >> "$tmp_env" found_runtime=1 ;; + AENV_VIRTUALIZATION_MODE=*) + printf 'AENV_VIRTUALIZATION_MODE="%s"\n' "$VIRTUALIZATION_MODE" >> "$tmp_env" + found_virtualization=1 + ;; *) printf '%s\n' "$line" >> "$tmp_env" ;; @@ -290,6 +311,9 @@ EOF if [[ "$found_runtime" == "0" ]]; then printf 'AENV_RUNTIME_PATH="%s"\n' "$RUNTIME_DIR" >> "$tmp_env" fi + if [[ "$found_virtualization" == "0" ]]; then + printf 'AENV_VIRTUALIZATION_MODE="%s"\n' "$VIRTUALIZATION_MODE" >> "$tmp_env" + fi sudo install -m 0644 "$tmp_env" "$ENV_FILE" rm -f "$current_env" "$tmp_env" ENV_FILE_STATUS="updated" @@ -340,6 +364,7 @@ echo " CLI : ${INSTALL_DIR}/aenv" echo " Server : ${INSTALL_DIR}/server" echo " Data : ${DATA_DIR}" echo " Config : ${CONFIG_PATH}" +echo " Mode : ${VIRTUALIZATION_MODE}" if [[ -d /run/systemd/system ]]; then if [[ "$ENV_FILE_STATUS" == "written" ]]; then echo " Env : ${ENV_FILE}" @@ -365,6 +390,7 @@ else echo " --inh-caps=+net_admin,+sys_admin --ambient-caps=+net_admin,+sys_admin \\" echo " --bounding-set=-all,+net_admin,+sys_admin --nnp \\" echo " env AENV_CONFIG_PATH=${CONFIG_PATH} AENV_HOME_PATH=${DATA_DIR} AENV_RUNTIME_PATH=${RUNTIME_DIR} \\" + echo " AENV_VIRTUALIZATION_MODE=${VIRTUALIZATION_MODE} \\" echo " API_ADDR=127.0.0.1:8000 ${INSTALL_DIR}/server" fi echo "" diff --git a/scripts/tests/e2e/run_e2e.sh b/scripts/tests/e2e/run_e2e.sh index eb584498..68dd66e3 100755 --- a/scripts/tests/e2e/run_e2e.sh +++ b/scripts/tests/e2e/run_e2e.sh @@ -2,7 +2,8 @@ # End-to-end test runner for AgentENV. # # Builds the server, starts it, creates a base template, then runs each suite. -# Requires: curl, jq, and a Linux host with /dev/kvm. +# Requires: curl, jq, and a Linux host whose /dev/kvm and modules match +# AENV_VIRTUALIZATION_MODE (kvm by default). # # Environment variables: # AENV_PORT - Port for the test server (default: 18080) diff --git a/src/api/impls/sandbox.rs b/src/api/impls/sandbox.rs index b8b04c81..5e8812a9 100644 --- a/src/api/impls/sandbox.rs +++ b/src/api/impls/sandbox.rs @@ -1133,6 +1133,7 @@ impl Sandboxes<()> for ApiImpl { startup: capture.metadata.startup.clone(), resources: capture.metadata.resources, runtime_versions: capture.metadata.runtime_versions.clone(), + virtualization_mode: capture.metadata.virtualization_mode, image_configs: capture.metadata.image_configs.clone(), custom_extension_params: capture.metadata.custom_extension_params.clone(), }, diff --git a/src/api/proxy.rs b/src/api/proxy.rs index 207872b5..9f8abdbd 100644 --- a/src/api/proxy.rs +++ b/src/api/proxy.rs @@ -1544,7 +1544,7 @@ mod tests { let orchestrator = Orchestrator::new( crate::orchestrator::InMemoryMetadataStore::new(), crate::sandbox::FirecrackerSandboxFactory::new(), - FileBackedSandboxPersister::new(root.path().to_path_buf()), + FileBackedSandboxPersister::new_for_test(root.path().to_path_buf()), ) .await .unwrap(); diff --git a/src/cfg.rs b/src/cfg.rs index baf32196..13d409d7 100644 --- a/src/cfg.rs +++ b/src/cfg.rs @@ -13,12 +13,14 @@ pub use network::{NetworkConfig, NetworkEgressConfig, NetworkInternalConfig}; use overlaybd::config::UpperMode; use serde::Deserialize; +use crate::virtualization::VirtualizationMode; + const ENV_CONFIG_PATH: &str = "AENV_CONFIG_PATH"; #[derive(Debug, Deserialize)] struct SetupDependencyManifest { - firecracker: ManifestDownload, - kernel: ManifestDownload, + firecracker: ManifestVirtualizationDownloads, + kernel: ManifestVirtualizationDownloads, tools: ManifestTools, overlaybd: ManifestDownload, #[serde(rename = "regclient")] @@ -30,6 +32,21 @@ struct ManifestDownload { version: String, } +#[derive(Debug, Deserialize)] +struct ManifestVirtualizationDownloads { + kvm: ManifestDownload, + pvm: ManifestDownload, +} + +impl ManifestVirtualizationDownloads { + fn for_mode(&self, mode: VirtualizationMode) -> &ManifestDownload { + match mode { + VirtualizationMode::Kvm => &self.kvm, + VirtualizationMode::Pvm => &self.pvm, + } + } +} + #[derive(Debug, Deserialize)] struct ManifestTools { version: String, @@ -74,6 +91,8 @@ pub struct AppConfig { default = "$AENV_HOME/deps" )] pub deps_path: PathBuf, + #[config(default = "kvm", env = "AENV_VIRTUALIZATION_MODE")] + pub virtualization_mode: VirtualizationMode, #[config(nested)] pub firecracker: FirecrackerConfig, #[config(nested)] @@ -537,14 +556,25 @@ impl_config_default!( ); impl AppConfig { + fn manifest_firecracker(&self) -> &ManifestDownload { + SetupDependencyManifest::get() + .firecracker + .for_mode(self.virtualization_mode) + } + + fn manifest_kernel(&self) -> &ManifestDownload { + SetupDependencyManifest::get() + .kernel + .for_mode(self.virtualization_mode) + } + pub fn resolved_firecracker_binary_path(&self) -> PathBuf { self.firecracker.binary_path.clone().unwrap_or_else(|| { - let manifest = SetupDependencyManifest::get(); let version = self .firecracker .version .as_deref() - .unwrap_or(&manifest.firecracker.version); + .unwrap_or(&self.manifest_firecracker().version); self.deps_path .join("firecracker") .join(version) @@ -554,12 +584,11 @@ impl AppConfig { pub fn resolved_kernel_image_path(&self) -> PathBuf { self.kernel.image_path.clone().unwrap_or_else(|| { - let manifest = SetupDependencyManifest::get(); let version = self .kernel .version .as_deref() - .unwrap_or(&manifest.kernel.version); + .unwrap_or(&self.manifest_kernel().version); self.deps_path .join("kernel") .join(version) @@ -606,12 +635,11 @@ impl AppConfig { /// Resolve the cpu-template-helper binary path derived from deps_path + version. /// Returns `None` if the binary does not exist on disk. pub fn resolved_cpu_template_helper(&self) -> Option { - let manifest = SetupDependencyManifest::get(); let version = self .firecracker .version .as_deref() - .unwrap_or(&manifest.firecracker.version); + .unwrap_or(&self.manifest_firecracker().version); let path = self .deps_path .join("firecracker") @@ -1335,6 +1363,42 @@ mod tests { Ok(()) } + #[test] + fn managed_dependency_paths_select_the_active_mode_versions() { + let manifest = SetupDependencyManifest::get(); + let mut config = AppConfig { + deps_path: PathBuf::from("/deps"), + virtualization_mode: VirtualizationMode::Kvm, + ..Default::default() + }; + assert_eq!( + config.resolved_firecracker_binary_path(), + PathBuf::from("/deps/firecracker") + .join(&manifest.firecracker.kvm.version) + .join("firecracker") + ); + assert_eq!( + config.resolved_kernel_image_path(), + PathBuf::from("/deps/kernel") + .join(&manifest.kernel.kvm.version) + .join("vmlinux.bin") + ); + + config.virtualization_mode = VirtualizationMode::Pvm; + assert_eq!( + config.resolved_firecracker_binary_path(), + PathBuf::from("/deps/firecracker") + .join(&manifest.firecracker.pvm.version) + .join("firecracker") + ); + assert_eq!( + config.resolved_kernel_image_path(), + PathBuf::from("/deps/kernel") + .join(&manifest.kernel.pvm.version) + .join("vmlinux.bin") + ); + } + #[test] fn home_relative_toml_values_derive_from_home_path() -> Result<()> { let temp = tempdir()?; diff --git a/src/lib.rs b/src/lib.rs index 537b09a6..5968c26e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,3 +16,4 @@ pub mod setup; pub mod snapshot; pub mod template; pub mod types; +pub mod virtualization; diff --git a/src/orchestrator/mod.rs b/src/orchestrator/mod.rs index e99a0254..90c2923b 100644 --- a/src/orchestrator/mod.rs +++ b/src/orchestrator/mod.rs @@ -7,6 +7,7 @@ mod store; mod types; use crate::types::SandboxId; +use crate::virtualization::VirtualizationMode; pub use metrics::OrchestratorMetrics; pub use persistence::{ @@ -45,15 +46,28 @@ pub enum SandboxOperation { pub enum OrchestratorError { #[error("failed to load sandbox config")] ConfigLoadFailed(#[from] anyhow::Error), + + #[error( + "{resource} uses virtualization mode '{resource_mode}', but this node runs in mode '{node_mode}'" + )] + VirtualizationModeMismatch { + resource: String, + resource_mode: VirtualizationMode, + node_mode: VirtualizationMode, + }, + #[error("orchestrator is shutting down")] ShuttingDown, + #[error("sandbox {0} not found")] SandboxNotFound(SandboxId), + #[error("sandbox {sandbox_id} is in invalid state {state:?}")] InvalidSandboxState { sandbox_id: SandboxId, state: SandboxState, }, + #[error("sandbox {sandbox_id} operation {operation:?} failed: {source}")] SandboxOperationFailed { sandbox_id: SandboxId, @@ -61,20 +75,25 @@ pub enum OrchestratorError { #[source] source: anyhow::Error, }, + #[error("sandbox {sandbox_id} operation {operation:?} conflicted with another operation")] SandboxOperationConflict { sandbox_id: SandboxId, operation: SandboxOperation, }, + #[error("store operation failed: {0}")] StoreOperationFailed(#[source] store::StoreError), + #[error("sandbox persistence failed: {0}")] SandboxPersistenceFailed(#[from] SandboxPersistenceError), + #[error("invalid timeout for {sandbox_id}: {timeout}")] InvalidTimeout { sandbox_id: SandboxId, timeout: String, }, + #[error("internal error: {0}")] InternalError(String), } diff --git a/src/orchestrator/persistence/file_backed.rs b/src/orchestrator/persistence/file_backed.rs index 78e74de0..53082f66 100644 --- a/src/orchestrator/persistence/file_backed.rs +++ b/src/orchestrator/persistence/file_backed.rs @@ -14,6 +14,7 @@ use crate::local_store::{LocalKvStore, LocalStoreDurability}; use crate::orchestrator::{store::SandboxMetadata, SandboxState}; use crate::sandbox::{PausedSandboxState, SandboxBackendFactory}; use crate::types::SandboxId; +use crate::virtualization::VirtualizationMode; const RECORD_VERSION: u32 = 1; const RECORD_DB_DIR: &str = "records.db"; @@ -53,6 +54,12 @@ impl PersistedPausedRecord { Ok(self.metadata) } + + fn into_metadata_without_runtime_state(mut self) -> SandboxMetadata { + self.metadata.state = SandboxState::Paused; + self.metadata.paused_state = None; + self.metadata + } } fn decode_record(bytes: &[u8]) -> PersistenceResult { @@ -78,23 +85,31 @@ fn ensure_supported_version(version: u32) -> PersistenceResult<()> { pub struct FileBackedSandboxPersister { root: PathBuf, + virtualization_mode: VirtualizationMode, durability: LocalStoreDurability, db: OnceCell, } impl FileBackedSandboxPersister { - pub fn new(root: PathBuf) -> Self { - Self::with_durability(root, LocalStoreDurability::Sync) - } - - pub fn with_durability(root: PathBuf, durability: LocalStoreDurability) -> Self { + pub fn new(root: PathBuf, virtualization_mode: VirtualizationMode) -> Self { Self { root, - durability, + virtualization_mode, + durability: LocalStoreDurability::Sync, db: OnceCell::new(), } } + #[cfg(test)] + pub(crate) fn new_for_test(root: PathBuf) -> Self { + Self::new(root, VirtualizationMode::Kvm) + } + + pub fn with_durability(mut self, durability: LocalStoreDurability) -> Self { + self.durability = durability; + self + } + fn records_db_path(&self) -> PathBuf { self.root.join(RECORD_DB_DIR) } @@ -266,6 +281,18 @@ impl SandboxPersister for FileBackedSandboxPersister { continue; } + if record.metadata.virtualization_mode != self.virtualization_mode { + warn!( + sandbox_id = %sandbox_id, + record_mode = %record.metadata.virtualization_mode, + node_mode = %self.virtualization_mode, + "loading paused sandbox metadata without resumable runtime state because its virtualization mode is incompatible" + ); + retained_artifacts.insert(sandbox_id); + sandboxes.push(record.into_metadata_without_runtime_state()); + continue; + } + match record.into_metadata(factory) { Ok(metadata) => { retained_artifacts.insert(sandbox_id); @@ -441,10 +468,8 @@ mod tests { } fn test_persister(root: &Path) -> FileBackedSandboxPersister { - FileBackedSandboxPersister::with_durability( - root.to_path_buf(), - LocalStoreDurability::Memory, - ) + FileBackedSandboxPersister::new_for_test(root.to_path_buf()) + .with_durability(LocalStoreDurability::Memory) } async fn persist_test_record( @@ -454,6 +479,7 @@ mod tests { let paused_state = paused_state(snapshot_root); let metadata = SandboxMetadata { id: SandboxId::new(), + virtualization_mode: persister.virtualization_mode, paused_state: Some(Arc::clone(&paused_state)), ..Default::default() }; @@ -504,6 +530,68 @@ mod tests { Ok(()) } + #[tokio::test] + async fn paused_record_from_other_mode_is_visible_but_not_resumable() -> anyhow::Result<()> { + let temp = TempDir::new()?; + let kvm_persister = test_persister(temp.path()); + let snapshot_root = temp.path().join("artifacts"); + let (sandbox_id, _paused_state) = + persist_test_record(&kvm_persister, &snapshot_root).await?; + drop(kvm_persister); + let pvm_persister = + FileBackedSandboxPersister::new(temp.path().to_path_buf(), VirtualizationMode::Pvm) + .with_durability(LocalStoreDurability::Memory); + + let loaded = pvm_persister.load_all(&MockBackendFactory::new()).await?; + + assert_eq!(loaded.len(), 1); + assert_eq!(loaded[0].id, sandbox_id); + assert_eq!(loaded[0].state, SandboxState::Paused); + assert_eq!(loaded[0].virtualization_mode, VirtualizationMode::Kvm); + assert!(loaded[0].paused_state.is_none()); + assert!(has_record(&pvm_persister, &sandbox_id).await?); + assert!(snapshot_root.exists()); + Ok(()) + } + + #[tokio::test] + async fn mixed_mode_records_are_both_visible_and_retained() -> anyhow::Result<()> { + let temp = TempDir::new()?; + let kvm_persister = test_persister(temp.path()); + let kvm_root = temp.path().join("kvm-artifacts"); + let (kvm_id, _kvm_state) = persist_test_record(&kvm_persister, &kvm_root).await?; + drop(kvm_persister); + + let pvm_persister = + FileBackedSandboxPersister::new(temp.path().to_path_buf(), VirtualizationMode::Pvm) + .with_durability(LocalStoreDurability::Memory); + let pvm_root = temp.path().join("pvm-artifacts"); + let (pvm_id, _pvm_state) = persist_test_record(&pvm_persister, &pvm_root).await?; + + let mut loaded = pvm_persister.load_all(&MockBackendFactory::new()).await?; + loaded.sort_by_key(|metadata| metadata.id); + + let kvm_metadata = loaded + .iter() + .find(|metadata| metadata.id == kvm_id) + .expect("KVM metadata should remain visible"); + assert_eq!(kvm_metadata.virtualization_mode, VirtualizationMode::Kvm); + assert!(kvm_metadata.paused_state.is_none()); + + let pvm_metadata = loaded + .iter() + .find(|metadata| metadata.id == pvm_id) + .expect("PVM metadata should load"); + assert_eq!(pvm_metadata.virtualization_mode, VirtualizationMode::Pvm); + assert!(pvm_metadata.paused_state.is_some()); + + assert!(has_record(&pvm_persister, &kvm_id).await?); + assert!(has_record(&pvm_persister, &pvm_id).await?); + assert!(kvm_root.exists()); + assert!(pvm_root.exists()); + Ok(()) + } + #[tokio::test] async fn allocate_artifact_root_creates_unique_snapshot_roots() -> anyhow::Result<()> { let temp = TempDir::new()?; diff --git a/src/orchestrator/service.rs b/src/orchestrator/service.rs index eba86e11..3e4d88d4 100644 --- a/src/orchestrator/service.rs +++ b/src/orchestrator/service.rs @@ -127,6 +127,7 @@ where let store = InMemoryMetadataStore::new(); let persister = FileBackedSandboxPersister::new( config.orchestrator.persisted_sandbox_store_path.clone(), + config.virtualization_mode, ); Self::new(store, factory, persister).await } @@ -359,6 +360,15 @@ where SandboxLaunchSource::Snapshot(snapshot) => { let record = snapshot.record(); let committed = snapshot.committed(); + let configured_mode = ConfigManager::global_config().virtualization_mode; + if committed.virtualization_mode != configured_mode { + self.counters.record_create_fail(1); + return Err(OrchestratorError::VirtualizationModeMismatch { + resource: format!("snapshot {}", record.id), + resource_mode: committed.virtualization_mode, + node_mode: configured_mode, + }); + } let launch_image_configs = committed.image_configs.clone(); // Effective custom config: a launch-provided value overrides the // one persisted in the source snapshot; otherwise inherit it. @@ -377,6 +387,7 @@ where id: sandbox_id, snapshot_id: record.id.to_string(), snapshot_alias: record.alias.as_ref().map(ToString::to_string), + virtualization_mode: committed.virtualization_mode, runtime_versions: committed.runtime_versions.clone(), resources: *snapshot.resources(), context: committed.context.clone(), @@ -428,6 +439,7 @@ where id: sandbox_id, snapshot_id: image_ref, snapshot_alias: None, + virtualization_mode: ConfigManager::global_config().virtualization_mode, runtime_versions: configured_runtime_versions(), resources, context, @@ -1236,6 +1248,15 @@ where } } + let node_mode = ConfigManager::global_config().virtualization_mode; + if metadata.virtualization_mode != node_mode { + return Err(OrchestratorError::VirtualizationModeMismatch { + resource: format!("paused sandbox {sandbox_id}"), + resource_mode: metadata.virtualization_mode, + node_mode, + }); + } + match self .store .update_state_if_state(&sandbox_id, SandboxState::Resuming, &[SandboxState::Paused]) diff --git a/src/orchestrator/store/metadata.rs b/src/orchestrator/store/metadata.rs index 5a11a0bb..891335da 100644 --- a/src/orchestrator/store/metadata.rs +++ b/src/orchestrator/store/metadata.rs @@ -11,6 +11,7 @@ use crate::sandbox::CustomExtensionParams; use crate::sandbox::{PausedSandboxState, SandboxNetworkPolicy}; use crate::snapshot::{CommandContext, SnapshotRuntimeVersions, StartupCommand}; use crate::types::{ImageConfigs, SandboxId, SandboxResources}; +use crate::virtualization::VirtualizationMode; #[derive(Clone, Copy, Debug, Serialize, Deserialize)] pub enum SandboxTimeoutAction { @@ -37,6 +38,9 @@ pub struct SandboxMetadata { pub timeout_action: SandboxTimeoutAction, pub expires_at: Option, pub auto_resume: bool, + /// Virtualization mode used by this sandbox for its entire lifecycle. + #[serde(default)] + pub virtualization_mode: VirtualizationMode, pub runtime_versions: SnapshotRuntimeVersions, pub resources: SandboxResources, pub context: CommandContext, @@ -68,6 +72,7 @@ impl Default for SandboxMetadata { timeout_action: SandboxTimeoutAction::Pause, expires_at: None, auto_resume: false, + virtualization_mode: VirtualizationMode::default(), runtime_versions: SnapshotRuntimeVersions::new( "unknown".to_string(), "unknown".to_string(), diff --git a/src/orchestrator/tests.rs b/src/orchestrator/tests.rs index ab4f05fb..bd72cdfd 100644 --- a/src/orchestrator/tests.rs +++ b/src/orchestrator/tests.rs @@ -3440,6 +3440,64 @@ async fn resume_running_with_none_timeout_clears_timeout() -> Result<()> { Ok(()) } +#[tokio::test] +async fn resume_rejects_paused_sandbox_from_other_virtualization_mode_without_mutation() { + use crate::virtualization::VirtualizationMode; + + setup(); + let sandbox_id = SandboxId::new(); + let node_mode = ConfigManager::global_config().virtualization_mode; + let sandbox_mode = match node_mode { + VirtualizationMode::Kvm => VirtualizationMode::Pvm, + VirtualizationMode::Pvm => VirtualizationMode::Kvm, + }; + let persister = RecordingPersister::with_loaded(vec![SandboxMetadata { + id: sandbox_id, + state: SandboxState::Paused, + virtualization_mode: sandbox_mode, + paused_state: None, + ..Default::default() + }]); + let orchestrator = Orchestrator::new_inner( + InMemoryMetadataStore::new(), + MockBackendFactory::new(), + persister.clone(), + test_runtime_image_refs(), + ) + .await + .expect("orchestrator should retain incompatible paused metadata"); + + let listed = orchestrator.list_sandboxes().await.expect("list metadata"); + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].id, sandbox_id); + assert_eq!(listed[0].virtualization_mode, sandbox_mode); + + let error = orchestrator + .resume_sandbox(sandbox_id, NewTimeout::UseExisting) + .await + .expect_err("cross-mode paused sandbox must not resume"); + + assert!(matches!( + error, + OrchestratorError::VirtualizationModeMismatch { + resource, + resource_mode, + node_mode: actual_node_mode, + } if resource == format!("paused sandbox {sandbox_id}") + && resource_mode == sandbox_mode + && actual_node_mode == node_mode + )); + let metadata = orchestrator + .get_sandbox(&sandbox_id) + .await + .expect("get metadata") + .expect("metadata remains visible"); + assert_eq!(metadata.state, SandboxState::Paused); + assert_eq!(metadata.virtualization_mode, sandbox_mode); + assert!(metadata.paused_state.is_none()); + assert_eq!(persister.calls(), vec![RecordingCall::LoadAll]); +} + #[tokio::test] async fn auto_evict_expired_sandbox() -> anyhow::Result<()> { setup(); diff --git a/src/sandbox/firecracker/config.rs b/src/sandbox/firecracker/config.rs index a18ac70e..dc7e07a4 100644 --- a/src/sandbox/firecracker/config.rs +++ b/src/sandbox/firecracker/config.rs @@ -460,6 +460,15 @@ impl FirecrackerSnapshotConfig { let manifest = snapshot.manifest(); let app_config = ConfigManager::global_config(); + let snapshot_mode = snapshot.committed().virtualization_mode; + if snapshot_mode != app_config.virtualization_mode { + bail!( + "snapshot '{}' uses virtualization mode '{}', but this node runs in mode '{}'", + snapshot.record().id, + snapshot_mode, + app_config.virtualization_mode + ); + } if !app_config.ublk.enabled { bail!("repository-backed snapshot launch requires ublk to be enabled"); } @@ -908,4 +917,26 @@ mod tests { .to_string() .contains("snapshot does not record a tools drive version")); } + + #[test] + fn runnable_snapshot_rejects_cross_virtualization_mode() { + use crate::virtualization::VirtualizationMode; + + let node_mode = ConfigManager::global_config().virtualization_mode; + let snapshot_mode = match node_mode { + VirtualizationMode::Kvm => VirtualizationMode::Pvm, + VirtualizationMode::Pvm => VirtualizationMode::Kvm, + }; + let mut committed = CommittedSnapshot::mock(); + committed.virtualization_mode = snapshot_mode; + let snapshot = + RunnableSnapshot::from_test_manifest(SnapshotRecord::mock_ready(committed), Vec::new()); + + let err = FirecrackerSnapshotConfig::from_runnable_snapshot(&snapshot) + .expect_err("node must reject a snapshot from the other virtualization mode"); + + assert!(err + .to_string() + .contains(&format!("uses virtualization mode '{snapshot_mode}'"))); + } } diff --git a/src/setup/deps.rs b/src/setup/deps.rs index 19dc2204..3a757b16 100644 --- a/src/setup/deps.rs +++ b/src/setup/deps.rs @@ -14,11 +14,12 @@ use crate::cfg::{ AppConfig, OssBackendConfig, OverlaybdDependencyConfig, SnapshotRepositoryBackendKind, }; use crate::digest::FileDigest; +use crate::virtualization::VirtualizationMode; #[derive(Debug, Deserialize)] struct SetupDependencyManifest { - firecracker: ManifestDownload, - kernel: ManifestDownload, + firecracker: ManifestVirtualizationDownloads, + kernel: ManifestVirtualizationDownloads, tools: ManifestTools, overlaybd: OverlaybdDependencyConfig, #[serde(rename = "regclient")] @@ -31,6 +32,21 @@ struct ManifestDownload { url: String, } +#[derive(Debug, Deserialize)] +struct ManifestVirtualizationDownloads { + kvm: ManifestDownload, + pvm: ManifestDownload, +} + +impl ManifestVirtualizationDownloads { + fn for_mode(&self, mode: VirtualizationMode) -> &ManifestDownload { + match mode { + VirtualizationMode::Kvm => &self.kvm, + VirtualizationMode::Pvm => &self.pvm, + } + } +} + #[derive(Debug, Deserialize)] struct ManifestTools { url: String, @@ -67,6 +83,9 @@ pub async fn ensure(config: &AppConfig, deps_path: &Path) -> Result<()> { // Architecture is resolved once and reused across dependency URL/path derivation. let arch = detect_arch()?; + if config.virtualization_mode == VirtualizationMode::Pvm && arch != "x86_64" { + bail!("PVM virtualization mode is only supported on x86_64 hosts"); + } std::fs::create_dir_all(deps_path)?; @@ -105,16 +124,17 @@ async fn ensure_firecracker( return Ok(()); } + let mode_manifest = manifest.firecracker.for_mode(config.virtualization_mode); let fc_version = config .firecracker .version .as_deref() - .unwrap_or(&manifest.firecracker.version); + .unwrap_or(&mode_manifest.version); let fc_url_template = config .firecracker .url .as_deref() - .unwrap_or(&manifest.firecracker.url); + .unwrap_or(&mode_manifest.url); let fc_dir = fc_path .parent() .context("resolved firecracker binary path has no parent")?; @@ -140,12 +160,13 @@ async fn ensure_kernel(config: &AppConfig, manifest: &SetupDependencyManifest) - return Ok(()); } + let mode_manifest = manifest.kernel.for_mode(config.virtualization_mode); let kernel_version = config .kernel .version .as_deref() - .unwrap_or(&manifest.kernel.version); - let kernel_url_template = config.kernel.url.as_deref().unwrap_or(&manifest.kernel.url); + .unwrap_or(&mode_manifest.version); + let kernel_url_template = config.kernel.url.as_deref().unwrap_or(&mode_manifest.url); let kernel_url = resolve_url(kernel_url_template, &[("version", kernel_version)]); download_file(&kernel_url, &kernel_path).await } diff --git a/src/setup/kvm.rs b/src/setup/kvm.rs index dc4484a7..bd2487db 100644 --- a/src/setup/kvm.rs +++ b/src/setup/kvm.rs @@ -4,6 +4,8 @@ use std::process::Command; use anyhow::{bail, Context, Result}; use tracing::info; +use crate::virtualization::VirtualizationMode; + fn kvm_accessible() -> bool { std::fs::OpenOptions::new() .read(true) @@ -39,19 +41,60 @@ pub fn add_user_to_group(user: &str, group: &str) -> Result { Ok(true) } -pub fn check() -> Result<()> { +pub fn check(mode: VirtualizationMode) -> Result<()> { + validate_mode( + mode, + std::env::consts::ARCH, + Path::new("/sys/module/kvm_pvm").exists(), + )?; + if !Path::new("/dev/kvm").exists() { bail!( "KVM device not found (/dev/kvm). \ - Please enable virtualization (VT-x/AMD-v) and load kvm modules." + Ensure the host virtualization module for {mode} is loaded and exposes /dev/kvm \ + (see the deployment documentation for more information)" ); } if kvm_accessible() { - info!("/dev/kvm is accessible"); + info!(virtualization_mode = %mode, "/dev/kvm is accessible"); return Ok(()); } bail!( "/dev/kvm is not accessible for read/write; add the runtime user to the kvm group and restart its session" ) } + +fn validate_mode(mode: VirtualizationMode, arch: &str, pvm_loaded: bool) -> Result<()> { + if mode == VirtualizationMode::Pvm && arch != "x86_64" { + bail!("PVM virtualization mode is only supported on x86_64 hosts"); + } + + match mode { + VirtualizationMode::Kvm if pvm_loaded => bail!( + "KVM mode cannot start while the kvm_pvm module is loaded; configure virtualization_mode = \"pvm\" or unload kvm_pvm" + ), + VirtualizationMode::Pvm if !pvm_loaded => { + bail!("PVM mode requires the kvm_pvm host module to be loaded") + } + _ => Ok(()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn kvm_mode_requires_pvm_module_to_be_absent() { + validate_mode(VirtualizationMode::Kvm, "x86_64", false).unwrap(); + assert!(validate_mode(VirtualizationMode::Kvm, "x86_64", true).is_err()); + } + + #[test] + fn pvm_mode_requires_x86_64_and_pvm_module() { + validate_mode(VirtualizationMode::Pvm, "x86_64", true).unwrap(); + assert!(validate_mode(VirtualizationMode::Pvm, "aarch64", true).is_err()); + assert!(validate_mode(VirtualizationMode::Pvm, "x86_64", false).is_err()); + } +} diff --git a/src/setup/mod.rs b/src/setup/mod.rs index ec0a864a..98fc5e16 100644 --- a/src/setup/mod.rs +++ b/src/setup/mod.rs @@ -97,7 +97,7 @@ fn is_valid_runtime_account_name(name: &str) -> bool { /// Run all environment setup steps. Fails fast if any prerequisite is unmet. /// /// Steps: -/// 1. Verify KVM is available and user has access +/// 1. Verify the configured KVM/PVM host mode and `/dev/kvm` access /// 2. Ensure ublk kernel module is loaded and permissions are set /// 3. Download dependencies (firecracker, kernel, tools drive, overlaybd) if missing pub async fn ensure_environment( @@ -118,9 +118,9 @@ pub async fn ensure_environment( // 1. Validate runtime OS packages without attempting elevation. packages::check_runtime()?; - // 2. KVM check - info!("checking KVM availability"); - kvm::check()?; + // 2. Selected virtualization mode and /dev/kvm check. + info!(virtualization_mode = %config.virtualization_mode, "checking virtualization availability"); + kvm::check(config.virtualization_mode)?; // 3. ublk setup (only if ublk is enabled in config) if config.ublk.enabled { diff --git a/src/snapshot/repository/backends/oss/repository.rs b/src/snapshot/repository/backends/oss/repository.rs index 06f747ef..0af7465f 100644 --- a/src/snapshot/repository/backends/oss/repository.rs +++ b/src/snapshot/repository/backends/oss/repository.rs @@ -279,6 +279,7 @@ impl SnapshotRepository for OssSnapshotRepository { context: metadata.context.clone(), startup: metadata.startup.clone(), runtime_versions: metadata.runtime_versions.clone(), + virtualization_mode: metadata.virtualization_mode, image_configs: metadata.image_configs.clone(), custom_extension_params: metadata.custom_extension_params.clone(), rootfs_layers, diff --git a/src/snapshot/repository/backends/posixfs/backend.rs b/src/snapshot/repository/backends/posixfs/backend.rs index 4dc7f7ba..769c7440 100644 --- a/src/snapshot/repository/backends/posixfs/backend.rs +++ b/src/snapshot/repository/backends/posixfs/backend.rs @@ -132,6 +132,7 @@ impl PosixFsSnapshotRepository { context: metadata.context.clone(), startup: metadata.startup.clone(), runtime_versions: metadata.runtime_versions.clone(), + virtualization_mode: metadata.virtualization_mode, image_configs: metadata.image_configs.clone(), custom_extension_params: metadata.custom_extension_params.clone(), rootfs_layers: built.rootfs_layers, @@ -639,6 +640,7 @@ mod tests { context: metadata.context.clone(), startup: metadata.startup.clone(), runtime_versions: metadata.runtime_versions.clone(), + virtualization_mode: metadata.virtualization_mode, image_configs: metadata.image_configs.clone(), rootfs_layers: vec![OverlaybdLayerRef::Managed(ManagedLayer { digest: "sharedfs:missing".to_string(), @@ -695,6 +697,7 @@ mod tests { context: metadata.context.clone(), startup: metadata.startup.clone(), runtime_versions: metadata.runtime_versions.clone(), + virtualization_mode: metadata.virtualization_mode, image_configs: metadata.image_configs.clone(), rootfs_layers: vec![OverlaybdLayerRef::Managed(ManagedLayer { digest: "sharedfs:test".to_string(), diff --git a/src/snapshot/types/snapshot.rs b/src/snapshot/types/snapshot.rs index 59072025..7cb67c69 100644 --- a/src/snapshot/types/snapshot.rs +++ b/src/snapshot/types/snapshot.rs @@ -9,6 +9,7 @@ use std::sync::OnceLock; use serde::{Deserialize, Serialize}; use crate::sandbox::CustomExtensionParams; +use crate::virtualization::VirtualizationMode; use shell_util::shell_quote; use super::drive::{CommittedAttachedDrive, ResolvedAttachedDrive}; @@ -17,7 +18,7 @@ use super::version::SnapshotRuntimeVersions; use crate::sandbox::FirecrackerSnapshotManifest; use crate::types::{ImageConfigs, SandboxResources}; -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug)] pub struct SnapshotPublishMetadata { pub id: SnapshotId, pub alias: Option, @@ -26,11 +27,10 @@ pub struct SnapshotPublishMetadata { pub startup: Option, pub resources: SandboxResources, pub runtime_versions: SnapshotRuntimeVersions, - #[serde(default, skip_serializing_if = "ImageConfigs::is_empty")] + pub virtualization_mode: VirtualizationMode, pub image_configs: ImageConfigs, /// Opaque user-provided JSON passed through to the custom extension hooks. /// Template launches inherit it unless overridden at create time. - #[serde(default, skip_serializing_if = "Option::is_none")] pub custom_extension_params: Option, } @@ -50,6 +50,7 @@ impl SnapshotPublishMetadata { envd_version: "envd".to_string(), tools_drive_version: "0.1.0".to_string(), }, + virtualization_mode: crate::cfg::ConfigManager::global_config().virtualization_mode, image_configs: ImageConfigs::new(), custom_extension_params: None, } @@ -285,6 +286,9 @@ pub struct CommittedSnapshot { pub context: CommandContext, pub startup: Option, pub runtime_versions: SnapshotRuntimeVersions, + /// Node virtualization ABI used to capture this snapshot. + #[serde(default)] + pub virtualization_mode: VirtualizationMode, #[serde(default, skip_serializing_if = "ImageConfigs::is_empty")] pub image_configs: ImageConfigs, pub rootfs_layers: Vec, @@ -310,6 +314,7 @@ impl CommittedSnapshot { envd_version: "envd".to_string(), tools_drive_version: "0.1.0".to_string(), }, + virtualization_mode: crate::cfg::ConfigManager::global_config().virtualization_mode, image_configs: ImageConfigs::new(), rootfs_layers: Vec::new(), attached_drives: Vec::new(), diff --git a/src/template/builder.rs b/src/template/builder.rs index c934e154..dea8b874 100644 --- a/src/template/builder.rs +++ b/src/template/builder.rs @@ -135,6 +135,7 @@ impl TemplateBuilder { startup: build_execution.startup, resources, runtime_versions: build_execution.runtime_versions, + virtualization_mode: context.virtualization_mode, image_configs: build_execution.image_configs, // Template builds intentionally do not propagate the base // snapshot's extension custom config: it is a per-sandbox @@ -211,6 +212,7 @@ impl TemplateBuilder { steps: spec.steps().to_vec(), base, cpu_config_json: self.current_cpu_config(), + virtualization_mode: ConfigManager::global_config().virtualization_mode, }) } @@ -231,6 +233,15 @@ impl TemplateBuilder { )); } + let node_mode = ConfigManager::global_config().virtualization_mode; + let base_mode = base_snapshot.committed().virtualization_mode; + if base_mode != node_mode { + return Err(TemplateBuildError::invalid_input(format!( + "base snapshot '{}' uses virtualization mode '{base_mode}', but this node runs in mode '{node_mode}'", + base_snapshot.record().id + ))); + } + let resources = *base_snapshot.resources(); if let Some(new_resources) = spec.resources_ref().copied() { if new_resources.cpu_count != resources.cpu_count @@ -265,6 +276,7 @@ impl TemplateBuilder { base_snapshot: Box::new(base_snapshot.clone()), }, cpu_config_json: self.current_cpu_config(), + virtualization_mode: base_mode, }) } @@ -425,6 +437,38 @@ mod tests { assert!(matches!(err, TemplateBuildError::InvalidInput { .. })); } + #[test] + fn snapshot_base_context_rejects_other_virtualization_mode() { + let manager = TemplateBuilder::new(); + let node_mode = ConfigManager::global_config().virtualization_mode; + let base_mode = match node_mode { + crate::virtualization::VirtualizationMode::Kvm => { + crate::virtualization::VirtualizationMode::Pvm + } + crate::virtualization::VirtualizationMode::Pvm => { + crate::virtualization::VirtualizationMode::Kvm + } + }; + let mut committed = CommittedSnapshot::mock(); + committed.virtualization_mode = base_mode; + let runnable = + RunnableSnapshot::from_test_manifest(SnapshotRecord::mock_ready(committed), Vec::new()); + + let error = manager + .prepare_snapshot_base_context( + &TemplateBuildSpec::new(), + SnapshotId::generate(), + &runnable, + ) + .expect_err("snapshot-based build must reject the other mode"); + + assert!( + matches!(error, TemplateBuildError::InvalidInput { ref reason } + if reason.contains(&format!("uses virtualization mode '{base_mode}'")) + && reason.contains(&format!("node runs in mode '{node_mode}'"))) + ); + } + #[test] fn snapshot_base_context_keeps_base_snapshot_attached_drives() { let manager = TemplateBuilder::new(); diff --git a/src/template/runner.rs b/src/template/runner.rs index 1879f11e..5812bc3f 100644 --- a/src/template/runner.rs +++ b/src/template/runner.rs @@ -20,6 +20,7 @@ use crate::snapshot::{ StartupCommand, }; use crate::types::{ImageConfigs, SandboxId, SandboxResources}; +use crate::virtualization::VirtualizationMode; /// Default command to use for ready check when start command is provided but ready command is not. /// Use the same default ready command as E2B @@ -66,6 +67,7 @@ pub(crate) struct TemplateBuildContext { pub steps: Vec, pub base: TemplateBuildBase, pub cpu_config_json: Option, + pub virtualization_mode: VirtualizationMode, } impl TemplateBuildContext { diff --git a/src/virtualization.rs b/src/virtualization.rs new file mode 100644 index 00000000..ca9aad3a --- /dev/null +++ b/src/virtualization.rs @@ -0,0 +1,52 @@ +use serde::{Deserialize, Serialize}; + +/// Node-wide virtualization backend and snapshot compatibility domain. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum VirtualizationMode { + #[default] + Kvm, + Pvm, +} + +impl std::fmt::Display for VirtualizationMode { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(match self { + Self::Kvm => "kvm", + Self::Pvm => "pvm", + }) + } +} + +impl std::str::FromStr for VirtualizationMode { + type Err = String; + + fn from_str(raw: &str) -> std::result::Result { + match raw.trim().to_ascii_lowercase().as_str() { + "kvm" => Ok(Self::Kvm), + "pvm" => Ok(Self::Pvm), + other => Err(format!( + "unsupported virtualization mode {other:?}; expected \"kvm\" or \"pvm\"" + )), + } + } +} + +#[cfg(test)] +mod tests { + use super::VirtualizationMode; + + #[test] + fn defaults_to_kvm_and_parses_supported_values() { + assert_eq!(VirtualizationMode::default(), VirtualizationMode::Kvm); + assert_eq!( + "KVM".parse::().unwrap(), + VirtualizationMode::Kvm + ); + assert_eq!( + "pvm".parse::().unwrap(), + VirtualizationMode::Pvm + ); + assert!("nested".parse::().is_err()); + } +} diff --git a/tests/integration/orchestrator.rs b/tests/integration/orchestrator.rs index ca69239b..9df80a8f 100644 --- a/tests/integration/orchestrator.rs +++ b/tests/integration/orchestrator.rs @@ -1,5 +1,6 @@ use crate::common; +use agentenv::cfg::ConfigManager; use agentenv::orchestrator::{ CreateSandboxRequest, FileBackedSandboxPersister, InMemoryMetadataStore, NewTimeout, Orchestrator, ProxyLookupResult, SandboxLaunchSource, SandboxState, SandboxTimeoutAction, @@ -11,12 +12,17 @@ use agentenv::snapshot::{ }; use anyhow::Result; +use std::path::PathBuf; use tempfile::tempdir; use tokio::time::{timeout, Duration}; use uuid::Uuid; const TEST_TIMEOUT: Duration = Duration::from_secs(120); +fn host_file_persister(root: PathBuf) -> FileBackedSandboxPersister { + FileBackedSandboxPersister::new(root, ConfigManager::global_config().virtualization_mode) +} + #[tokio::test] async fn orchestrator_lifecycle() -> Result<()> { common::setup().await; @@ -38,7 +44,7 @@ async fn orchestrator_lifecycle() -> Result<()> { let store = InMemoryMetadataStore::new(); let factory = FirecrackerSandboxFactory::new(); let paused_store = root.path().join("paused-sandboxes"); - let persister = FileBackedSandboxPersister::new(paused_store.clone()); + let persister = host_file_persister(paused_store.clone()); let orchestrator = Orchestrator::new(store, factory, persister).await?; let case_id = Uuid::now_v7().to_string(); @@ -95,7 +101,7 @@ async fn orchestrator_lifecycle() -> Result<()> { let restarted = Orchestrator::new( InMemoryMetadataStore::new(), FirecrackerSandboxFactory::new(), - FileBackedSandboxPersister::new(paused_store), + host_file_persister(paused_store), ) .await?; let restored = restarted @@ -205,6 +211,7 @@ async fn orchestrator_capture_snapshot_can_be_published_and_relaunched() -> Resu startup: capture.metadata.startup.clone(), resources: capture.metadata.resources, runtime_versions: capture.metadata.runtime_versions.clone(), + virtualization_mode: capture.metadata.virtualization_mode, image_configs: capture.metadata.image_configs.clone(), custom_extension_params: None, }, diff --git a/tests/integration/snapshot.rs b/tests/integration/snapshot.rs index 96bb6004..28df0b9d 100644 --- a/tests/integration/snapshot.rs +++ b/tests/integration/snapshot.rs @@ -90,6 +90,8 @@ async fn publish_captured_snapshot_for_test( disk_size_mib: 0, }, runtime_versions: sample_runtime_versions(), + virtualization_mode: agentenv::cfg::ConfigManager::global_config() + .virtualization_mode, image_configs: agentenv::types::ImageConfigs::new(), custom_extension_params: None, }, diff --git a/tests/integration/snapshot_attached_drive.rs b/tests/integration/snapshot_attached_drive.rs index 53055fdc..bb66ed78 100644 --- a/tests/integration/snapshot_attached_drive.rs +++ b/tests/integration/snapshot_attached_drive.rs @@ -180,6 +180,7 @@ async fn publish_sandbox_snapshot_with_attached_drive( envd_version: "test-envd".to_string(), tools_drive_version: "0.1.0".to_string(), }, + virtualization_mode: agentenv::cfg::ConfigManager::global_config().virtualization_mode, image_configs: agentenv::types::ImageConfigs::new(), custom_extension_params: None, };