diff --git a/Makefile b/Makefile index 116b30c0d..10bf7d4cf 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,11 @@ AENV_INSTALL_SUDO ?= sudo TEST_SCRIPTS_DIR := ./scripts/tests # Runner for tests that require AENV's network and namespace capabilities. -CAPABILITY_RUNNER := CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="$(CURDIR)/scripts/run-with-capabilities.sh" +# The target triple is derived from uname -m so the runner works on both +# x86_64 and aarch64 hosts without manual configuration. +HOST_TRIPLE := $(shell uname -m)-unknown-linux-gnu +HOST_TRIPLE_UPPER := $(shell echo $(HOST_TRIPLE) | tr '[:lower:]-' '[:upper:]_') +CAPABILITY_RUNNER := CARGO_TARGET_$(HOST_TRIPLE_UPPER)_RUNNER="$(CURDIR)/scripts/run-with-capabilities.sh" AENV_TEST_STATE_ID ?= $(if $(GITHUB_RUN_ID),$(GITHUB_RUN_ID)-$(GITHUB_RUN_ATTEMPT),local-$$(id -u)) AENV_TEST_STATE_DIR ?= /tmp/aenv-test-$(AENV_TEST_STATE_ID) AENV_TEST_DEPS_PATH ?= $(if $(AENV_DEPS_PATH),$(AENV_DEPS_PATH),/var/lib/aenv/deps) diff --git a/README.md b/README.md index d6612f4ab..51604bdbd 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,8 @@ AgentENV (AENV) is a platform for running agent environments at scale, powering ## 📋 Prerequisites -- **Linux kernel 6.8+**; the install script additionally requires **Ubuntu 24.04** (see *Quick Start* below for installation options) +- **Linux kernel 6.8+** on x86_64 or aarch64; the install script requires **Ubuntu 24.04 (x86_64)** +- ARM (aarch64) servers are supported via source build — see [ARM Quick Start](docs/src/getting-started/quickstart-arm.md) - `/dev/kvm` access for Firecracker microVM execution If your server does not support standard KVM, see the [PVM deployment guide](https://kvcache-ai.github.io/AgentENV/dev/deployment/pvm.html) before installing. @@ -73,6 +74,11 @@ docker run -d --privileged -v /dev:/dev -p 8000:8000 ghcr.io/kvcache-ai/aenv-ser The server is accessible at `http://127.0.0.1:8000` by default. +*Option C — ARM (aarch64) source build* + +ARM servers (AWS Graviton, Ampere Altra, Kunpeng) require building from source. +See the [ARM Quick Start](docs/src/getting-started/quickstart-arm.md) for full instructions. + **2. Install the aenv CLI** *(skip if you used Option A in step 1)* Install separately if you used the Docker method above, or if you are running @@ -104,6 +110,7 @@ aenv start ubuntu # starts a sandbox and attaches an interactive shel For Docker Compose / Kubernetes cluster deployment and build-from-source instructions, see 📖 [Deployment](https://kvcache-ai.github.io/AgentENV/latest/deployment/manual-compile.html). +For ARM (aarch64) source-build installation, see 📖 [ARM Quick Start](docs/src/getting-started/quickstart-arm.md). --- diff --git a/config/deps_manifest.toml b/config/deps_manifest.toml index efe075e92..a2814e268 100644 --- a/config/deps_manifest.toml +++ b/config/deps_manifest.toml @@ -2,6 +2,12 @@ version = "1.15.1-patch-v1" url = "https://pub-4ee15c400f554ab7a9eac3f5bc8f53de.r2.dev/firecracker-{version}-{arch}.tgz" +# ARM (aarch64) uses the kvcache-ai/firecracker release because the patched +# R2 build is only published for x86_64. +[firecracker.kvm.arch.aarch64] +version = "1.15.1-patch-v1" +url = "https://github.com/kvcache-ai/firecracker/releases/download/aenv-deps/firecracker-{version}-{arch}.tgz" + [firecracker.pvm] version = "v1.17.0-next.1" url = "https://github.com/kvcache-ai/firecracker-next/releases/download/{version}/firecracker-next-{version}-{arch}.tgz" @@ -10,6 +16,11 @@ url = "https://github.com/kvcache-ai/firecracker-next/releases/download/{version version = "vmlinux-6.1.175" url = "https://pub-4ee15c400f554ab7a9eac3f5bc8f53de.r2.dev/{version}" +# ARM kernel is published in the same kvcache-ai/firecracker release. +[kernel.kvm.arch.aarch64] +version = "vmlinux-6.1.175" +url = "https://github.com/kvcache-ai/firecracker/releases/download/aenv-deps/{version}-{arch}" + [kernel.pvm] version = "6.12.33-pvm" url = "https://github.com/kvcache-ai/linux/releases/download/pvm-kernel-6.12.33/vmlinux-guest-6.12.33-pvm" diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 7d02dab98..82423075b 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -3,7 +3,8 @@ # Getting Started - [Overview](./getting-started/overview.md) -- [Quick Start](./getting-started/quickstart.md) +- [Quick Start (x86_64)](./getting-started/quickstart.md) +- [Quick Start (ARM)](./getting-started/quickstart-arm.md) - [On-Demand Loading](./getting-started/on-demand-loading.md) - [aenv CLI Reference](./getting-started/aenv-cli.md) diff --git a/docs/src/getting-started/quickstart-arm.md b/docs/src/getting-started/quickstart-arm.md new file mode 100644 index 000000000..af6808d41 --- /dev/null +++ b/docs/src/getting-started/quickstart-arm.md @@ -0,0 +1,174 @@ +# ARM (aarch64) Quick Start + +Build and run AgentENV from source on ARM servers (AWS Graviton, Ampere Altra, +Kunpeng, etc.). Pre-built binaries are not yet published for aarch64; follow the +source-build steps below. + +## Prerequisites + +- Linux aarch64 host with KVM enabled (`/dev/kvm`) +- Rust toolchain (stable) +- Docker with Buildx (for building the tools drive) +- ~25 GB disk, ~8 GB RAM + +### Ubuntu / Debian + +```bash +sudo apt-get install -y \ + gcc g++ make protobuf-compiler libprotobuf-dev \ + clang libclang-dev dpkg jq e2fsprogs \ + iptables iproute2 umoci zstd curl libaio1t64 +``` + +### openEuler / CentOS / RHEL + +```bash +sudo dnf install -y \ + gcc gcc-c++ make protobuf-compiler protobuf-devel \ + clang clang-devel dpkg jq e2fsprogs \ + iptables iproute umoci zstd curl libaio +``` + +> **ublk**: enabled by default (overlaybd snapshots + on-demand loading). +> Works on Ubuntu/Debian ARM out of the box. openEuler kernels ship with +> `CONFIG_BLK_DEV_UBLK` disabled — if you need full functionality on +> openEuler, rebuild the host kernel with ublk enabled. + +## 1. Install Rust + +```bash +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +source "$HOME/.cargo/env" +``` + +If behind a firewall, download the script manually and use a mirror: + +```bash +curl -o rustup-init.sh https://sh.rustup.rs +chmod +x rustup-init.sh +RUSTUP_UPDATE_ROOT=https://mirrors.aliyun.com/rustup/rustup \ +RUSTUP_DIST_SERVER=https://mirrors.aliyun.com/rustup \ +./rustup-init.sh -y +source "$HOME/.cargo/env" +``` + +## 2. Build the ARM tools drive + +The pre-built tools drive image is currently amd64-only. Build the ARM variant: + +```bash +cd tools-image +make build ARCH=arm64 +# output: out/tools-0.1.0-arm64.ext4 +cd .. +``` + +## 3. Build AgentENV + +```bash +ulimit -n 65536 +make release +``` + +The first build takes 20–40 minutes (`librocksdb-sys` compiles a large C++ +codebase). On memory-constrained hosts use `CARGO_BUILD_JOBS=2 make release`. + +## 4. Configure + +Create an ARM configuration: + +```bash +cp config/default.toml config/arm.toml +``` + +Edit `config/arm.toml`: + +```toml +# Path to the ARM tools drive built in step 2. +[tools] +drive_path = "/path/to/AgentENV/tools-image/out/tools-0.1.0-arm64.ext4" +version = "0.1.0" + +# ARM uses MMIO, not PCI. +[firecracker] +boot_args = "console=ttyS0 reboot=k panic=1 init=/init" +``` + +> **ublk (overlaybd storage)**: ublk is enabled by default and works on ARM hosts +> where the kernel has `CONFIG_BLK_DEV_UBLK` (Ubuntu ARM, Debian ARM, etc.). +> If your host kernel lacks ublk (e.g. openEuler default), disable it: +> ```toml +> [ublk] +> enabled = false +> ``` + +## 5. First-run setup + +```bash +# One-time host provisioning (KVM group, network sysctls). +sudo ./target/release/server \ + --config config/arm.toml \ + --setup-host \ + --runtime-user "$USER" \ + --runtime-group "$(id -gn)" + +# Download runtime dependencies (Firecracker aarch64, kernel, overlaybd, regctl). +./target/release/server --config config/arm.toml --setup-only +``` + +If the server cannot reach GitHub, download these manually and place them at the +expected paths: + +| Asset | Destination | +|-------|-------------| +| [firecracker-1.15.1-patch-v1-aarch64.tgz](https://github.com/kvcache-ai/firecracker/releases/download/aenv-deps/firecracker-1.15.1-patch-v1-aarch64.tgz) | Extract to `/var/lib/aenv/deps/firecracker/1.15.1-patch-v1/firecracker` | +| [vmlinux-6.1.175-aarch64](https://github.com/kvcache-ai/firecracker/releases/download/aenv-deps/vmlinux-6.1.175-aarch64) | `/var/lib/aenv/deps/kernel/vmlinux-6.1.175/aarch64/vmlinux.bin` | +| [regctl-linux-arm64](https://github.com/regclient/regclient/releases/download/v0.11.5/regctl-linux-arm64) | `/var/lib/aenv/deps/regctl/v0.11.5/regctl` (chmod +x) | +| [overlaybd aarch64 .deb](https://github.com/containerd/overlaybd/releases/tag/v1.0.18) | Extract with `dpkg-deb -x`, copy `opt/overlaybd/` to `/var/lib/aenv/deps/overlaybd/` | + +## 6. Start the server + +```bash +sudo -E env API_ADDR=0.0.0.0:8000 \ + ./target/release/server --config config/arm.toml +``` + +Verify: + +```bash +curl http://127.0.0.1:8000/health +# HTTP 204 No Content +``` + +## Troubleshooting + +### `/dev/kvm` not accessible + +```bash +sudo modprobe kvm +sudo usermod -aG kvm "$USER" +# re-login +ls -l /dev/kvm +``` + +### `unsupported Linux distribution` + +openEuler / CentOS / RHEL hosts may need the distro-recognition patch +(see `src/setup/packages.rs`). Ensure you are on the `feat/arm-support` branch. + +### Too many open files + +```bash +ulimit -n 65536 +``` + +### protoc errors + +```bash +sudo dnf install -y protobuf-compiler protobuf-devel # RPM +sudo apt-get install -y protobuf-compiler # DEB +``` + +### Firecracker pool prime timed out + +Harmless warning on first start. The pool warms lazily on first sandbox creation. diff --git a/src/cfg.rs b/src/cfg.rs index 31c4bf0a1..9c6bbd88f 100644 --- a/src/cfg.rs +++ b/src/cfg.rs @@ -30,6 +30,28 @@ struct SetupDependencyManifest { #[derive(Debug, Deserialize)] struct ManifestDownload { version: String, + /// Per-architecture overrides (see `config/deps_manifest.toml`). + /// When resolving a path for the current host architecture, the + /// override's `version` replaces the top-level default. + #[serde(default)] + arch: std::collections::HashMap, +} + +#[derive(Debug, Clone, Deserialize)] +struct ArchManifestDownload { + version: String, +} + +impl ManifestDownload { + /// Return the effective version for the current host architecture. + /// Checks for an arch-specific override in the manifest first; falls + /// back to the top-level `version`. + fn version_for_host_arch(&self) -> &str { + self.arch + .get(std::env::consts::ARCH) + .map(|a| a.version.as_str()) + .unwrap_or(&self.version) + } } #[derive(Debug, Deserialize)] @@ -594,7 +616,7 @@ impl AppConfig { .firecracker .version .as_deref() - .unwrap_or(&self.manifest_firecracker().version); + .unwrap_or_else(|| self.manifest_firecracker().version_for_host_arch()); self.deps_path .join("firecracker") .join(version) @@ -608,10 +630,12 @@ impl AppConfig { .kernel .version .as_deref() - .unwrap_or(&self.manifest_kernel().version); + .unwrap_or_else(|| self.manifest_kernel().version_for_host_arch()); + let arch = std::env::consts::ARCH; self.deps_path .join("kernel") .join(version) + .join(arch) .join("vmlinux.bin") }) } @@ -659,7 +683,7 @@ impl AppConfig { .firecracker .version .as_deref() - .unwrap_or(&self.manifest_firecracker().version); + .unwrap_or_else(|| self.manifest_firecracker().version_for_host_arch()); let path = self .deps_path .join("firecracker") diff --git a/src/setup/deps.rs b/src/setup/deps.rs index efeb135a1..457077e68 100644 --- a/src/setup/deps.rs +++ b/src/setup/deps.rs @@ -30,6 +30,17 @@ struct SetupDependencyManifest { struct ManifestDownload { version: String, url: String, + /// Per-architecture overrides. When the host architecture matches a key, + /// the override's `version` and `url` replace the top-level defaults + /// (unless the user has set explicit config values). + #[serde(default)] + arch: std::collections::HashMap, +} + +#[derive(Debug, Clone, Deserialize)] +struct ArchManifestDownload { + version: String, + url: String, } #[derive(Debug, Deserialize)] @@ -90,7 +101,7 @@ pub async fn ensure(config: &AppConfig, deps_path: &Path) -> Result<()> { std::fs::create_dir_all(deps_path)?; ensure_firecracker(config, manifest, &arch).await?; - ensure_kernel(config, manifest).await?; + ensure_kernel(config, manifest, &arch).await?; // regctl is a runtime dependency for all registry access, not just tools // drive extraction, so it remains provisioned for explicit tools drives. @@ -125,16 +136,17 @@ async fn ensure_firecracker( } let mode_manifest = manifest.firecracker.for_mode(config.virtualization_mode); - let fc_version = config - .firecracker - .version - .as_deref() - .unwrap_or(&mode_manifest.version); - let fc_url_template = config - .firecracker - .url - .as_deref() - .unwrap_or(&mode_manifest.url); + let arch_override = mode_manifest.arch.get(arch); + let fc_version = config.firecracker.version.as_deref().unwrap_or( + arch_override + .map(|a| a.version.as_str()) + .unwrap_or(&mode_manifest.version), + ); + let fc_url_template = config.firecracker.url.as_deref().unwrap_or( + arch_override + .map(|a| a.url.as_str()) + .unwrap_or(&mode_manifest.url), + ); let fc_dir = fc_path .parent() .context("resolved firecracker binary path has no parent")?; @@ -149,7 +161,11 @@ async fn ensure_firecracker( Ok(()) } -async fn ensure_kernel(config: &AppConfig, manifest: &SetupDependencyManifest) -> Result<()> { +async fn ensure_kernel( + config: &AppConfig, + manifest: &SetupDependencyManifest, + arch: &str, +) -> Result<()> { if let Some(kernel_path) = config.kernel.image_path.as_deref() { return validate_explicit_file("kernel.image_path", kernel_path, false); } @@ -161,13 +177,21 @@ async fn ensure_kernel(config: &AppConfig, manifest: &SetupDependencyManifest) - } let mode_manifest = manifest.kernel.for_mode(config.virtualization_mode); - let kernel_version = config - .kernel - .version - .as_deref() - .unwrap_or(&mode_manifest.version); - let kernel_url_template = config.kernel.url.as_deref().unwrap_or(&mode_manifest.url); - let kernel_url = resolve_url(kernel_url_template, &[("version", kernel_version)]); + let arch_override = mode_manifest.arch.get(arch); + let kernel_version = config.kernel.version.as_deref().unwrap_or( + arch_override + .map(|a| a.version.as_str()) + .unwrap_or(&mode_manifest.version), + ); + let kernel_url_template = config.kernel.url.as_deref().unwrap_or( + arch_override + .map(|a| a.url.as_str()) + .unwrap_or(&mode_manifest.url), + ); + let kernel_url = resolve_url( + kernel_url_template, + &[("version", kernel_version), ("arch", arch)], + ); download_file(&kernel_url, &kernel_path).await } @@ -865,7 +889,7 @@ mod tests { ensure_firecracker(&config, bundled_manifest(), "x86_64") .await .expect("accept explicit firecracker"); - ensure_kernel(&config, bundled_manifest()) + ensure_kernel(&config, bundled_manifest(), "x86_64") .await .expect("accept explicit kernel"); ensure_tools(&config, &deps_path, bundled_manifest()).expect("import explicit tools"); diff --git a/src/setup/overlaybd.rs b/src/setup/overlaybd.rs index 70ea2abc0..da4fa641e 100644 --- a/src/setup/overlaybd.rs +++ b/src/setup/overlaybd.rs @@ -136,7 +136,8 @@ fn configured_overlaybd_release( // so it runs fine on other glibc-based distros. Fall back to the // oldest published Ubuntu build (lowest glibc requirement) for known // RPM-family distros so the CLI tools install without a native asset. - "tencentos" | "centos" | "centos-stream" | "rhel" | "redhat" | "redhatenterpriseserver" => { + "tencentos" | "centos" | "centos-stream" | "rhel" | "redhat" + | "redhatenterpriseserver" | "openeuler" | "openEuler" => { format!("ubuntu1.{}.{}", FALLBACK_UBUNTU_VERSION, target.arch) } other => bail!( @@ -576,7 +577,7 @@ mod tests { ), }; - for os_id in ["centos", "centos-stream", "tencentos", "rhel"] { + for os_id in ["centos", "centos-stream", "tencentos", "rhel", "openeuler", "openEuler"] { let release = configured_overlaybd_release( &config, &OverlaybdReleaseTarget { diff --git a/src/setup/packages.rs b/src/setup/packages.rs index 82aeec03f..a62a0a68c 100644 --- a/src/setup/packages.rs +++ b/src/setup/packages.rs @@ -258,7 +258,7 @@ fn distro_from_id(id: &str) -> Option { match id { "ubuntu" => Some(Distro::Ubuntu), "debian" => Some(Distro::Debian), - "centos" | "centos-stream" | "tencentos" => Some(Distro::Centos), + "centos" | "centos-stream" | "tencentos" | "openeuler" | "openEuler" => Some(Distro::Centos), "rhel" | "redhat" | "redhatenterpriseserver" => Some(Distro::Rhel), "arch" | "manjaro" => Some(Distro::Arch), _ => None, diff --git a/tools-image/Dockerfile b/tools-image/Dockerfile index 2413f2b7d..ae7925666 100644 --- a/tools-image/Dockerfile +++ b/tools-image/Dockerfile @@ -44,7 +44,9 @@ ARG ENVD_REF=2026.17 ARG ENVD_UPSTREAM_REPO=https://github.com/e2b-dev/infra.git ARG TARGETARCH -RUN apt-get update \ +RUN sed -i 's/deb.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \ + sed -i 's/deb.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list; \ + apt-get update \ && apt-get install -y --no-install-recommends ca-certificates git \ && rm -rf /var/lib/apt/lists/* @@ -68,6 +70,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ esac; \ commit_sha="$(git -C /src/infra rev-parse --short HEAD)"; \ CGO_ENABLED=0 GOOS=linux GOARCH="$TARGETARCH" \ + GOPROXY=https://goproxy.cn,direct \ go build \ -ldflags "-X=main.commitSHA=${commit_sha} -s -w" \ -o /out/envd \ @@ -81,6 +84,14 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ echo "Skipping envd execution smoke test for cross-built ${TARGETARCH} binary on ${build_arch} builder"; \ fi +# --------------------------------------------------------------------------- +# Stage 2a: pull the full busybox binary (with runsv + svlogd) for the target +# architecture. We cannot RUN inside this image on hosts without binfmt +# emulation for the target arch, but COPY --from works because it only reads +# image layers. +# --------------------------------------------------------------------------- +FROM --platform=${TARGETPLATFORM} ${BUSYBOX_IMAGE} AS busybox-bin + # --------------------------------------------------------------------------- # Stage 2: assemble the guest-visible tools rootfs. # @@ -90,11 +101,16 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ # /agentenv/pivot-init post-pivot bootstrap # /agentenv/envd compiled control-plane daemon # /agentenv/bin/busybox stable helper binary for init scripts +# +# debian (instead of busybox/alpine) is used as the rootfs base so this stage +# can RUN on ARM hosts that cannot exec the busybox image's /bin/sh. # --------------------------------------------------------------------------- -FROM --platform=${TARGETPLATFORM} ${BUSYBOX_IMAGE} AS rootfs +FROM --platform=$BUILDPLATFORM debian:${DEBIAN_VERSION}-slim AS rootfs ARG TOOLS_VERSION +COPY --chmod=0755 --from=busybox-bin /bin/busybox /usr/local/bin/busybox-full + COPY --chmod=0755 init /tools-rootfs/init COPY --chmod=0755 pivot-init /tools-rootfs/agentenv/pivot-init COPY --chmod=0755 --from=envd-builder /out/envd /tools-rootfs/agentenv/envd @@ -107,7 +123,7 @@ RUN set -eux; \ /tools-rootfs/proc \ /tools-rootfs/sys/fs/cgroup \ /tools-rootfs/agentenv/bin; \ - cp /bin/busybox /tools-rootfs/agentenv/bin/busybox; \ + cp /usr/local/bin/busybox-full /tools-rootfs/agentenv/bin/busybox; \ printf '%s\n' "$TOOLS_VERSION" > /tools-rootfs/agentenv/tools-drive-version; \ ln -sf /proc/self/mounts /tools-rootfs/etc/mtab; \ chmod 0755 /tools-rootfs/agentenv/bin/busybox; \ @@ -125,7 +141,9 @@ RUN set -eux; \ # --------------------------------------------------------------------------- FROM --platform=$BUILDPLATFORM debian:${DEBIAN_VERSION}-slim AS ext4-builder -RUN apt-get update \ +RUN sed -i 's/deb.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list.d/debian.sources 2>/dev/null || \ + sed -i 's/deb.debian.org/mirrors.aliyun.com/g' /etc/apt/sources.list; \ + apt-get update \ && apt-get install -y --no-install-recommends e2fsprogs findutils \ && rm -rf /var/lib/apt/lists/*