diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..7f9e68cd --- /dev/null +++ b/.dockerignore @@ -0,0 +1,55 @@ +# Build context exclusion list for `docker build .`. Keeping the +# context lean matters: every byte sent to the daemon is hashed for +# the build's source-changed signal, and the production multi-stage +# Dockerfile already invalidates the dependency layer cache on any +# Cargo.lock / Cargo.toml change. + +# Cargo build artefacts — re-built inside the build stage anyway. +/target/ +target/ +**/target/ + +# Runtime state directory the engine writes the redb file into. Never +# part of the image. +/data/ +data/ + +# Backtest tooling output: large JSON fixtures + Python venv state. +# Re-collected on demand via `tools/backtest-collect/backtest_collect.py`. +tools/backtest-collect/fixtures-*.json +tools/baseline-latency/data/ +tools/**/__pycache__/ +tools/**/*.pyc + +# NOTE: `modules/fixtures/*-bomb` are listed in the workspace +# `Cargo.toml`, so excluding them breaks `cargo build` ("failed to +# load manifest for workspace member"). They're tiny crates and the +# Dockerfile doesn't COPY them to the runtime stage, so the +# image size impact is zero. Keep them in the build context. + +# Local-only configs. The production `engine.toml` is supplied at +# runtime via a bind-mount (`/etc/shepherd/engine.toml`). +engine.toml +engine.e2e.toml +engine.load.toml +engine.m2.toml +engine.m3.toml + +# Git + GitHub metadata. +/.git/ +/.github/ +.gitignore + +# Editor / OS noise. +.vscode/ +.idea/ +.DS_Store +*.swp + +# Operator-side docs reports the image doesn't need. Source markdown +# stays so it's discoverable inside the container if an operator +# `docker exec`s in for a quick `cat docs/production.md`. +docs/operations/load-reports/ +docs/operations/e2e-reports/ +docs/operations/backtest-reports/ +docs/operations/baselines/ diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..809e711e --- /dev/null +++ b/.env.example @@ -0,0 +1,25 @@ +# Operator template — copy to `.env` and fill in your paid RPC URLs. +# `.env` is gitignored; never commit a populated copy. +# +# Workflow: +# cp .env.example .env +# $EDITOR .env +# docker compose up -d +# +# The engine reads these via `${VAR}` placeholders in +# `engine.docker.toml` (substitution happens at config-load time, +# before TOML parse, so a missing variable fails fast). +# +# Use `wss://` schemes — `eth_subscribe` is WebSocket-only and the +# engine emits a boot-time ERROR on http(s):// URLs (see +# docs/production.md §6 and engine_config::validate_transports). + +MAINNET_RPC_URL=wss://eth-mainnet.g.alchemy.com/v2/REPLACE_ME +GNOSIS_RPC_URL=wss://gnosis-mainnet.g.alchemy.com/v2/REPLACE_ME +SEPOLIA_RPC_URL=wss://eth-sepolia.g.alchemy.com/v2/REPLACE_ME +ARBITRUM_RPC_URL=wss://arb-mainnet.g.alchemy.com/v2/REPLACE_ME +BASE_RPC_URL=wss://base-mainnet.g.alchemy.com/v2/REPLACE_ME + +# Optional: override the published image with a locally-built or +# pinned-by-SHA tag. Leave unset to pull `:latest` from ghcr.io. +# SHEPHERD_IMAGE=ghcr.io/bleu/nullis-shepherd:sha-abc1234 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..d03d87ff --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,88 @@ +# Docker image build + publish to ghcr.io. +# +# Triggers: +# - push to `main` → publish `latest` + `sha-` +# - tag push `v*` → publish `v` + `latest` +# - workflow_dispatch (manual) → publish `manual-` +# - pull_request to `main` → build only, no push (CI smoke) +# +# Image: ghcr.io//nullis-shepherd +# Auth: GITHUB_TOKEN (scoped to packages:write below). +# +# Pinned action SHAs match the style of `.github/workflows/ci.yml`. + +name: docker + +on: + push: + branches: [main] + tags: ["v*"] + pull_request: + branches: [main] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + packages: write + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push: + name: build + push (${{ github.event_name }}) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Docker buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Log in to ghcr.io + if: github.event_name != 'pull_request' + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Compute image metadata + id: meta + uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + # `latest` on push to main and on tag. + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=tag + # `sha-` on every event so a soak run can pin an + # exact build. + type=sha,prefix=sha-,format=short + # manual- for workflow_dispatch. + type=raw,value=manual-${{ github.run_id }},enable=${{ github.event_name == 'workflow_dispatch' }} + # `pr-` on pull-request builds so the smoke artefact + # is identifiable. PR builds are NOT pushed (see `push:`). + type=ref,event=pr,prefix=pr- + + - name: Build + push + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: . + file: ./Dockerfile + # Push on every non-PR event; PR builds are local-only smoke. + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + # Layer cache via the registry: the previous successful + # build's intermediate layers are reused so a Cargo.toml-only + # change re-compiles only the changed crate. + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max,ignore-error=true + # `amd64` is enough for the soak VM. Add `arm64` once an + # operator surfaces a real need; multi-arch ~2x the build. + platforms: linux/amd64 diff --git a/.gitignore b/.gitignore index a8837c6b..0a4a7c71 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,9 @@ Thumbs.db # Environment .env .env.* +# Exception: the committed template (operator copies it to `.env`, +# which is then caught by the rule above). +!.env.example # Agent skills / AI tooling — installed locally, never committed. .agents/ @@ -36,6 +39,11 @@ data/ scripts/.state scripts/.env +# Operator-supplied engine config (carries paid RPC URLs / API keys). +# The committed siblings `engine.example.toml`, `engine.docker.toml`, +# and `engine.{m2,m3,e2e,load}.toml` are placeholder templates. +/engine.toml + # Generated reports under e2e-reports/ (operator commits the filled-in ones # manually via `git add -f`). docs/operations/e2e-reports/engine-*.log diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..f4dafed0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,107 @@ +# syntax=docker/dockerfile:1.6 +# +# Multi-stage build for `nexum-engine` (Shepherd) — the engine binary +# plus the five production WASM modules baked into a single image. +# +# Stage 1 (`build`): full Rust toolchain + wasm32-wasip2 target, builds +# the engine in release mode + each module to a Component Model wasm +# artefact. +# +# Stage 2 (`runtime`): minimal Debian slim. Just `ca-certificates` +# (for HTTPS to cow.fi / paid RPCs), `tini` as PID 1 (forwards SIGINT +# for graceful shutdown per docs/production.md §2), and a non-root +# `shepherd` user owning `/var/lib/shepherd`. +# +# The runtime entrypoint expects `/etc/shepherd/engine.toml` to be +# mounted (read-only) — see `docker-compose.yml` and +# `docs/deployment/docker.md`. + +# ----------------------------------------------------------------- build + +# Pin the Rust toolchain to a version recent enough for the +# transitive wasmtime 45.x crates (which require rustc >= 1.93). +# Bump in lockstep with workspace Cargo.lock minimum-supported +# rustc — `cargo msrv` if uncertain. +FROM rust:1.96-slim-bookworm AS build + +# Build deps for ring/openssl/cmake-using crates pulled in via alloy +# and cowprotocol. `clang` is for any inline-C bindings (e.g. +# pycryptodome-equivalent in the wasm side); cheap enough to bundle. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + pkg-config libssl-dev cmake clang ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +RUN rustup target add wasm32-wasip2 + +WORKDIR /src + +# Copy the whole workspace. `.dockerignore` should keep the build +# context lean (no `target/`, no `data/`, no large baseline / backtest +# fixtures). +COPY . . + +# Engine binary in release. +RUN cargo build -p nexum-engine --release + +# Five production modules. The wasm artefacts land under +# `target/wasm32-wasip2/release/.wasm`. +RUN cargo build -p twap-monitor --target wasm32-wasip2 --release \ + && cargo build -p ethflow-watcher --target wasm32-wasip2 --release \ + && cargo build -p price-alert --target wasm32-wasip2 --release \ + && cargo build -p balance-tracker --target wasm32-wasip2 --release \ + && cargo build -p stop-loss --target wasm32-wasip2 --release + +# ----------------------------------------------------------------- runtime + +FROM debian:bookworm-slim AS runtime + +# `tini` reaps zombies + forwards SIGINT/SIGTERM to the engine so the +# COW-1072 graceful-shutdown path actually runs (drain in-flight +# dispatch, persist `last_dispatched_block:{chain_id}` to local-store). +# `ca-certificates` is mandatory for HTTPS calls to cow.fi + paid RPC +# endpoints; the engine has no embedded TLS roots. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates tini \ + && rm -rf /var/lib/apt/lists/* \ + && useradd -r -s /usr/sbin/nologin -d /var/lib/shepherd shepherd \ + && install -d -o shepherd -g shepherd -m 0755 /var/lib/shepherd \ + && install -d -o root -g root -m 0755 /opt/shepherd \ + && install -d -o root -g root -m 0755 /opt/shepherd/modules \ + && install -d -o root -g root -m 0755 /opt/shepherd/manifests \ + && install -d -o root -g root -m 0755 /etc/shepherd + +# Engine binary. +COPY --from=build /src/target/release/nexum-engine /usr/local/bin/nexum-engine + +# Module .wasm artefacts. The Component Model wasm files are loaded +# by the engine at boot via the `[[modules]]` entries in engine.toml. +COPY --from=build /src/target/wasm32-wasip2/release/*.wasm /opt/shepherd/modules/ + +# Module manifests (the `module.toml` next to each cdylib crate). The +# engine resolves capability declarations + chain subscriptions from +# these at supervisor boot. +COPY --from=build /src/modules/twap-monitor/module.toml /opt/shepherd/manifests/twap-monitor.toml +COPY --from=build /src/modules/ethflow-watcher/module.toml /opt/shepherd/manifests/ethflow-watcher.toml +COPY --from=build /src/modules/examples/price-alert/module.toml /opt/shepherd/manifests/price-alert.toml +COPY --from=build /src/modules/examples/balance-tracker/module.toml /opt/shepherd/manifests/balance-tracker.toml +COPY --from=build /src/modules/examples/stop-loss/module.toml /opt/shepherd/manifests/stop-loss.toml + +# Drop privileges. The engine never needs root at runtime: it only +# reads /etc/shepherd/engine.toml, writes to /var/lib/shepherd, and +# binds 127.0.0.1:9100 inside the container. +USER shepherd +WORKDIR /var/lib/shepherd + +# Metrics endpoint. The engine binds 127.0.0.1:9100 inside the +# container by default; docker-compose maps it to the host's +# loopback so Prometheus scrapes it via the docker network without +# exposing /metrics to the public internet. +EXPOSE 9100 + +# `--engine-config /etc/shepherd/engine.toml` matches the production +# guide's expected mount point. Operators override via +# `docker run ... -v /path/to/engine.toml:/etc/shepherd/engine.toml:ro`. +ENTRYPOINT ["/usr/bin/tini", "--", "nexum-engine"] +CMD ["--engine-config", "/etc/shepherd/engine.toml"] diff --git a/crates/nexum-engine/src/engine_config.rs b/crates/nexum-engine/src/engine_config.rs index 0a43f847..9860143f 100644 --- a/crates/nexum-engine/src/engine_config.rs +++ b/crates/nexum-engine/src/engine_config.rs @@ -120,6 +120,20 @@ pub struct ChainConfig { /// `tools/orderbook-mock` for the COW-1079 load test). #[serde(default)] pub orderbook_url: Option, + /// Escape hatch: silence the boot-time warning when an `http(s)://` + /// `rpc_url` is configured. Default `true` — every production + /// module today subscribes to blocks or logs, so an HTTP URL is + /// almost certainly an operator mistake (drpc / Alchemy / Infura + /// expose BOTH `https://...` and `wss://...` per endpoint; the WS + /// form is what `eth_subscribe` needs). Flip this to `false` only + /// for a chain consumed exclusively by poll-style modules + /// (request/response `chain::request`, no block / log subscriptions). + #[serde(default = "default_require_ws")] + pub require_ws: bool, +} + +fn default_require_ws() -> bool { + true } fn default_state_dir() -> PathBuf { @@ -148,12 +162,348 @@ pub fn load_or_default(path: Option<&Path>) -> anyhow::Result { } let raw = std::fs::read_to_string(&path)?; - let cfg: EngineConfig = toml::from_str(&raw)?; + // Operators reference RPC URLs (which carry API keys) via + // `${VAR_NAME}` placeholders so the committed `engine.toml` / + // `engine.docker.toml` stays secret-free. The substitution runs + // before TOML parse so a missing var fails fast with the exact + // variable name, not a downstream "invalid URI" several layers + // deep. + let substituted = substitute_env_vars(&raw).map_err(|e| { + anyhow::anyhow!("engine config env-var substitution failed: {e}") + })?; + let cfg: EngineConfig = toml::from_str(&substituted)?; info!( path = %path.display(), chains = cfg.chains.len(), state_dir = %cfg.engine.state_dir.display(), "engine config loaded", ); + // `validate_transports()` is intentionally NOT called here: + // `load_or_default` runs before `tracing_subscriber::init()` in + // `main.rs`, so any ERROR logs emitted here would be silently + // dropped. The validator is invoked explicitly from `main.rs` + // after the subscriber is up. Ok(cfg) } + +/// Replace every `${VAR_NAME}` token in `raw` with the value of the +/// corresponding environment variable. Returns an error naming any +/// missing variable so the operator sees the exact fix. +/// +/// Recognised variable names: `[A-Z_][A-Z0-9_]*` (matches shell env +/// var conventions). Anything else inside `${...}` is rejected so a +/// typo doesn't silently pass through. +/// +/// Note: substitution runs over the whole TOML text, including +/// comments. This is fine in practice — comments are stripped during +/// the subsequent `toml::from_str` parse, and the only realistic +/// `${VAR}` payload is in string values anyway. +fn substitute_env_vars(raw: &str) -> Result { + let mut out = String::with_capacity(raw.len()); + let bytes = raw.as_bytes(); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'$' + && i + 1 < bytes.len() + && bytes[i + 1] == b'{' + { + // Find the closing `}`. + let start = i + 2; + let Some(end_offset) = raw[start..].find('}') else { + return Err(EnvVarError::Unclosed { offset: i }); + }; + let end = start + end_offset; + let name = &raw[start..end]; + if !is_valid_env_name(name) { + return Err(EnvVarError::InvalidName { + name: name.to_owned(), + }); + } + match std::env::var(name) { + Ok(val) => out.push_str(&val), + Err(_) => return Err(EnvVarError::Missing { name: name.to_owned() }), + } + i = end + 1; + } else { + // Push one UTF-8 char (find the next char boundary). + let ch = raw[i..].chars().next().expect("byte index is on char boundary"); + out.push(ch); + i += ch.len_utf8(); + } + } + Ok(out) +} + +fn is_valid_env_name(s: &str) -> bool { + let mut chars = s.chars(); + let Some(first) = chars.next() else { + return false; + }; + if !(first.is_ascii_uppercase() || first == '_') { + return false; + } + chars.all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || c == '_') +} + +#[derive(Debug, thiserror::Error)] +pub enum EnvVarError { + #[error( + "environment variable `{name}` referenced via ${{{name}}} in engine.toml but not set. \ + Export it before launching the engine (e.g. via a `.env` file consumed by `docker compose`)." + )] + Missing { name: String }, + #[error( + "invalid env var name `{name}` inside ${{...}} in engine.toml — names must match \ + [A-Z_][A-Z0-9_]*. Typo, or did you mean `${{{name_upper}}}`?", + name_upper = name.to_uppercase() + )] + InvalidName { name: String }, + #[error("unclosed `${{` at byte offset {offset} in engine.toml — every `${{` needs a matching `}}`.")] + Unclosed { offset: usize }, +} + +impl EngineConfig { + /// Surface configuration footguns at boot time, before the event + /// loop opens any transport. Today's only check: an HTTP(S) + /// `rpc_url` will refuse `eth_subscribe` (the protocol requires a + /// WebSocket transport), and the engine's COW-1071 reconnect + /// backoff will loop forever waiting for a subscription that can + /// never open. We emit a single loud ERROR-level structured log + /// per offending chain pointing the operator at the exact swap. + /// + /// `[chains.] require_ws = false` opts a chain out of the + /// check (poll-only deployments where no module subscribes). + pub fn validate_transports(&self) { + for (chain_id, chain) in &self.chains { + if !chain.require_ws { + continue; + } + let url = chain.rpc_url.trim().to_lowercase(); + if url.starts_with("ws://") || url.starts_with("wss://") { + continue; + } + // Redact BOTH the original URL and the suggested swap — + // log files often end up in shared aggregators (Loki, + // Datadog), and the swap is straightforward enough that + // the operator doesn't need the full URL printed back. + let suggested = redact_url(&suggest_ws_swap(&chain.rpc_url)); + tracing::error!( + chain_id = chain_id, + rpc_url = %redact_url(&chain.rpc_url), + suggested = %suggested, + "rpc_url uses HTTP transport but the engine subscribes to \ + blocks/logs via eth_subscribe (WS-only). Modules expecting \ + these events will never receive them; the event-loop will \ + log retry-with-backoff lines forever. Switch the URL to \ + `wss://` (every paid provider exposes both forms) or set \ + `[chains.{chain_id}] require_ws = false` if this chain is \ + consumed by poll-only modules.", + ); + } + } +} + +/// Best-effort swap of an `http(s)://` URL to the operator-likely WS +/// variant so the boot-time error message can suggest a concrete fix. +/// Falls back to the original URL if the scheme doesn't match. +fn suggest_ws_swap(url: &str) -> String { + if let Some(rest) = url.strip_prefix("https://") { + return format!("wss://{rest}"); + } + if let Some(rest) = url.strip_prefix("http://") { + return format!("ws://{rest}"); + } + url.to_owned() +} + +/// Drop an embedded API key from a URL so the validation log line is +/// safe to share. Heuristic: replace any path segment longer than 20 +/// characters with `` (matches Alchemy / drpc / Infura key +/// shapes). +/// +/// Public so other engine call sites that log the configured RPC URL +/// (provider pool boot, host-side debug traces) can apply the same +/// redaction; log aggregators (Loki, Datadog, Splunk) routinely +/// retain weeks of logs and the key should never sit in cold storage. +pub fn redact_url(url: &str) -> String { + url.split('/') + .map(|seg| { + if seg.len() > 20 && !seg.contains('.') && !seg.contains(':') { + "".to_owned() + } else { + seg.to_owned() + } + }) + .collect::>() + .join("/") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn cfg_with_url(url: &str, require_ws: bool) -> EngineConfig { + let mut chains = BTreeMap::new(); + chains.insert( + 11155111, + ChainConfig { + rpc_url: url.into(), + orderbook_url: None, + require_ws, + }, + ); + EngineConfig { + chains, + ..Default::default() + } + } + + #[test] + fn validate_accepts_wss_url() { + let cfg = cfg_with_url("wss://lb.drpc.org/sepolia/", true); + cfg.validate_transports(); + // No assertion needed — passes if no panic and (in a real + // logger setup) no ERROR line was emitted. + } + + #[test] + fn validate_accepts_ws_url() { + let cfg = cfg_with_url("ws://localhost:8545", true); + cfg.validate_transports(); + } + + #[test] + fn validate_is_silent_when_require_ws_is_false() { + // Operator explicitly opted out — HTTP is intentional (poll + // only). The validator must not nag. + let cfg = cfg_with_url("https://eth-mainnet.example.com/v2/abc", false); + cfg.validate_transports(); + } + + #[test] + fn validate_runs_without_panicking_on_http_url() { + // The validator's contract is *log + continue*, not *abort*. + // Catching a panic here would mask the only-WARN behaviour we + // ship today. + let cfg = cfg_with_url("https://eth-mainnet.example.com/v2/abc", true); + cfg.validate_transports(); + } + + #[test] + fn suggest_swaps_https_to_wss() { + assert_eq!( + suggest_ws_swap("https://lb.drpc.org/sepolia/abc"), + "wss://lb.drpc.org/sepolia/abc", + ); + } + + #[test] + fn suggest_swaps_http_to_ws() { + assert_eq!( + suggest_ws_swap("http://localhost:8545"), + "ws://localhost:8545", + ); + } + + #[test] + fn suggest_passes_through_already_ws_url() { + assert_eq!( + suggest_ws_swap("wss://x.example/k"), + "wss://x.example/k", + ); + } + + #[test] + fn redact_replaces_long_path_segments() { + let redacted = redact_url( + "https://lb.drpc.live/sepolia/AnOfyGnZ_0nWpS-OOwQzqAnFj_Naa0sR8ZxkVjewFaCJ", + ); + assert!(redacted.contains("")); + assert!(!redacted.contains("AnOfyGnZ")); + } + + #[test] + fn redact_keeps_short_segments_intact() { + // Hostnames + "v1" path bits must not be redacted. + let redacted = redact_url("https://eth-mainnet.g.alchemy.com/v2/abc"); + assert!(redacted.contains("eth-mainnet.g.alchemy.com")); + assert!(redacted.contains("v2")); + } + + // ----------------- env var substitution ----------------------- + // + // These tests stash + restore process env vars under unique names + // so parallel `cargo test` runs don't trip on each other. + + fn with_env(name: &str, value: &str, body: F) { + let prev = std::env::var(name).ok(); + // SAFETY: tests are single-threaded within one test fn; setting + // an env var here is fine since the unique-name convention + // avoids cross-test races. + unsafe { std::env::set_var(name, value) }; + body(); + match prev { + Some(v) => unsafe { std::env::set_var(name, v) }, + None => unsafe { std::env::remove_var(name) }, + } + } + + #[test] + fn substitute_replaces_known_variable() { + with_env("COW1078_TEST_RPC", "wss://example.test/abc", || { + let raw = r#"rpc_url = "${COW1078_TEST_RPC}""#; + let out = substitute_env_vars(raw).unwrap(); + assert_eq!(out, r#"rpc_url = "wss://example.test/abc""#); + }); + } + + #[test] + fn substitute_errors_on_missing_variable() { + // Variable name must not collide with anything in the operator + // environment. Use a guaranteed-unique prefix. + let err = substitute_env_vars(r#"x = "${COW1078_DEFINITELY_UNSET_VAR_XYZ}""#) + .unwrap_err(); + let msg = err.to_string(); + assert!(msg.contains("COW1078_DEFINITELY_UNSET_VAR_XYZ")); + assert!(msg.contains("not set")); + } + + #[test] + fn substitute_errors_on_invalid_name() { + let err = substitute_env_vars(r#"x = "${lowercase_name}""#).unwrap_err(); + assert!(matches!(err, EnvVarError::InvalidName { .. })); + } + + #[test] + fn substitute_errors_on_unclosed_brace() { + let err = substitute_env_vars(r#"x = "${UNCLOSED"#).unwrap_err(); + assert!(matches!(err, EnvVarError::Unclosed { .. })); + } + + #[test] + fn substitute_passes_text_with_no_placeholders_through() { + let raw = "no placeholders here\nrpc_url = \"wss://x\""; + assert_eq!(substitute_env_vars(raw).unwrap(), raw); + } + + #[test] + fn substitute_handles_multiple_placeholders_in_one_line() { + with_env("COW1078_A", "alpha", || { + with_env("COW1078_B", "beta", || { + let raw = "k = \"${COW1078_A}-${COW1078_B}\""; + let out = substitute_env_vars(raw).unwrap(); + assert_eq!(out, "k = \"alpha-beta\""); + }); + }); + } + + #[test] + fn substitute_preserves_utf8_around_placeholder() { + // The hand-rolled byte loop must respect multi-byte UTF-8. + with_env("COW1078_U", "X", || { + let raw = "# 河 ${COW1078_U} ⚙️\n"; + let out = substitute_env_vars(raw).unwrap(); + assert_eq!(out, "# 河 X ⚙️\n"); + }); + } +} diff --git a/crates/nexum-engine/src/host/provider_pool.rs b/crates/nexum-engine/src/host/provider_pool.rs index 017b35bd..7933ebd9 100644 --- a/crates/nexum-engine/src/host/provider_pool.rs +++ b/crates/nexum-engine/src/host/provider_pool.rs @@ -40,7 +40,16 @@ impl ProviderPool { let mut providers: BTreeMap = BTreeMap::new(); for (chain_id, chain_cfg) in &cfg.chains { let url = chain_cfg.rpc_url.as_str(); - info!(chain_id, url, "opening chain RPC provider"); + // The boot log carries the URL with embedded API keys + // redacted — log aggregators (Loki, Datadog, splunk) often + // ingest these lines and the key shouldn't end up in + // long-term storage. The engine still uses the full URL + // when actually connecting to the provider below. + info!( + chain_id, + url = %crate::engine_config::redact_url(url), + "opening chain RPC provider", + ); let provider = if url.starts_with("ws://") || url.starts_with("wss://") { ProviderBuilder::new() .connect_ws(WsConnect::new(url)) diff --git a/crates/nexum-engine/src/main.rs b/crates/nexum-engine/src/main.rs index 104c79b8..ca3c8255 100644 --- a/crates/nexum-engine/src/main.rs +++ b/crates/nexum-engine/src/main.rs @@ -56,6 +56,14 @@ async fn main() -> anyhow::Result<()> { info!("nexum-engine starting"); + // Surface config footguns now that the tracing subscriber is + // up. Today's only check: an HTTP `rpc_url` would loop forever + // in the event-loop's WS reconnect backoff because + // `eth_subscribe` is WS-only. One ERROR log per offending chain + // with the exact `wss://` swap suggested. See + // `engine_config::validate_transports`. + engine_cfg.validate_transports(); + // COW-1034: install the Prometheus exporter. When // `[engine.metrics].enabled = true` the HTTP listener also binds // and serves `/metrics`. Otherwise the recorder is still diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..b8e5c585 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,123 @@ +# Operator-facing Docker Compose for Shepherd. +# +# Two profiles: +# +# - default — just the engine. `docker compose up -d`. +# - observability — engine + Prometheus pre-wired to scrape the +# engine's /metrics endpoint. Opt in via +# `docker compose --profile observability up -d`. +# +# The image either builds from the repo's Dockerfile (`docker compose +# build`) or pulls the published ghcr.io artefact when the +# `SHEPHERD_IMAGE` env var is set (CI publishes +# `ghcr.io/bleu/nullis-shepherd:` and `:latest` on main). +# +# Required mounts: +# - ./engine.toml -> /etc/shepherd/engine.toml (operator-supplied) +# +# See docs/deployment/docker.md for the operator runbook. + +services: + shepherd: + image: ${SHEPHERD_IMAGE:-ghcr.io/bleu/nullis-shepherd:latest} + # Comment out `build` if you `docker compose pull` instead of + # building from source. Leaving it lets `docker compose up + # --build` re-build from the local checkout when the image + # isn't published yet. + build: + context: . + dockerfile: Dockerfile + container_name: shepherd + restart: unless-stopped + # The engine handles SIGINT for graceful shutdown (COW-1072); + # docker stop sends SIGTERM by default, so override. + stop_signal: SIGINT + # Match docs/production.md §2 TimeoutStopSec=30s. + stop_grace_period: 30s + volumes: + # Engine config. Default points at the committed + # `engine.docker.toml` template (uses `${VAR}` placeholders + # the engine substitutes from env at boot). Override with a + # bespoke `./engine.toml` by setting + # `SHEPHERD_ENGINE_CONFIG=./engine.toml` in `.env`. + - ${SHEPHERD_ENGINE_CONFIG:-./engine.docker.toml}:/etc/shepherd/engine.toml:ro + # Local-store redb file lives on a named volume so it survives + # container recreation (image upgrades). + - shepherd-state:/var/lib/shepherd + ports: + # Metrics endpoint pinned to the HOST's loopback so Prometheus + # scrapes via the docker network without exposing /metrics + # publicly. Override to `9100:9100` only if you front the + # endpoint with authn/authz (NGINX + basic auth, etc.). + - "127.0.0.1:9100:9100" + environment: + RUST_BACKTRACE: "1" + # Forward the paid-RPC URLs the engine substitutes into + # `engine.docker.toml` via `${VAR}` placeholders. Compose + # picks these up from the repo-root `.env` (gitignored; + # operator copies from `.env.example`). Missing variables + # fail fast at engine boot with the exact name. + MAINNET_RPC_URL: + GNOSIS_RPC_URL: + SEPOLIA_RPC_URL: + ARBITRUM_RPC_URL: + BASE_RPC_URL: + # Defence-in-depth resource caps. The engine already caps each + # module's wasmtime fuel + memory at 1B inst/event + 64 MiB; this + # is the outer envelope on the host process. + deploy: + resources: + limits: + memory: 2g + cpus: "2.0" + # Keep the engine on the same network as Prometheus so the scrape + # config can reach `shepherd:9100` (DNS via compose service name). + networks: + - shepherd-net + # Health: a successful `curl` against /metrics implies the engine + # is up and the supervisor's metrics exporter has bound. NB the + # engine returns 200 even when individual modules are quarantined; + # alert on `shepherd_module_poisoned` for that, not on health. + healthcheck: + # `/dev/tcp//` is a bash builtin, not POSIX sh — + # the default `CMD-SHELL` runs through `/bin/sh` (dash on + # debian:bookworm-slim), so we invoke `bash` explicitly. bash + # ships in the slim base by default; no extra apt install + # needed. A successful TCP open on the metrics port proves the + # supervisor finished its boot path and the metrics exporter + # bound. Failure marks the container unhealthy. + test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/127.0.0.1/9100"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 20s + + # ----------- optional observability stack -------------------------- + # Enable with: `docker compose --profile observability up -d`. + + prometheus: + image: prom/prometheus:v2.55.0 + container_name: shepherd-prometheus + restart: unless-stopped + profiles: ["observability"] + volumes: + - ./docs/deployment/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.retention.time=30d" + ports: + - "127.0.0.1:9090:9090" + networks: + - shepherd-net + depends_on: + shepherd: + condition: service_healthy + +volumes: + shepherd-state: + prometheus-data: + +networks: + shepherd-net: + driver: bridge diff --git a/docs/deployment/docker.md b/docs/deployment/docker.md new file mode 100644 index 00000000..abfadce8 --- /dev/null +++ b/docs/deployment/docker.md @@ -0,0 +1,195 @@ +# Docker deployment runbook + +Operator-facing quickstart for running Shepherd in production via the +published container image. For the full hardening surface (systemd +unit, backup recipes, RPC selection, alerting rules) read +`docs/production.md`. + +The image is published on every push to `main` and on every +`v*` tag: + +``` +ghcr.io/bleu/nullis-shepherd:latest # main branch HEAD +ghcr.io/bleu/nullis-shepherd:sha- # exact-build pin +ghcr.io/bleu/nullis-shepherd:v0.2.0 # tag +``` + +`linux/amd64` only for now (the soak VM is x86_64; add `arm64` once +an operator surfaces a real need). + +--- + +## 1. First boot on a fresh VM + +```bash +# On the VM: +git clone https://github.com/bleu/nullis-shepherd /opt/shepherd +cd /opt/shepherd + +# Operator-supplied RPC URLs. `.env` is gitignored; the template +# committed at `.env.example` lists every variable the engine +# substitutes into `engine.docker.toml` via `${VAR}` placeholders. +cp .env.example .env +${EDITOR:-vi} .env # paste your paid wss:// URLs + +# Pull the published image (no local build needed). +docker compose pull + +# Start the engine. Compose reads `.env` automatically and passes +# the listed variables into the container, where the engine +# substitutes them at config-load time. +docker compose up -d + +# Logs (JSON line-per-event, see `docs/production.md §5`). +docker compose logs -f shepherd +``` + +If you want the observability stack on the same host: + +```bash +docker compose --profile observability up -d +# Prometheus UI: http://127.0.0.1:9090 +``` + +The metrics endpoint binds the **host's loopback** by default +(`127.0.0.1:9100`); the Prometheus container scrapes via the +compose-internal DNS name `shepherd:9100`. Never expose `:9100` to +the public internet without authn — see `docs/production.md §7`. + +--- + +## 2. Configuring `engine.toml` + +The image bind-mounts the committed `engine.docker.toml` at +`/etc/shepherd/engine.toml` read-only. It uses `${VAR}` placeholders +for every paid-RPC URL, which the engine substitutes at load time +from environment (Docker compose forwards them in from `.env`). +A missing variable fails the boot fast with the exact name. + +To run with a custom config (different module mix, extra chains) +instead of `engine.docker.toml`, point compose at it via +`SHEPHERD_ENGINE_CONFIG=./engine.local.toml` in `.env` — the bind +mount picks up whichever path is set. + +Minimum production shape if you write your own: + +```toml +[engine] +state_dir = "/var/lib/shepherd" # mapped to the `shepherd-state` named volume +log_level = "info" + +[engine.metrics] +enabled = true +bind_addr = "0.0.0.0:9100" # inside the container; compose maps to 127.0.0.1 + +# One per chain you subscribe to. `${VAR}` placeholders are +# substituted at load time from environment — keep the actual URL +# in `.env`, not in any committed file. Must be `wss://`; the +# engine emits a boot-time ERROR otherwise (see docs/production.md §6). +[chains.11155111] +rpc_url = "${SEPOLIA_RPC_URL}" + +[chains.42161] +rpc_url = "${ARBITRUM_RPC_URL}" + +# One [[modules]] per .wasm baked into /opt/shepherd/modules/. +# `manifest` defaults to /module.toml if omitted. +[[modules]] +path = "/opt/shepherd/modules/twap_monitor.wasm" +manifest = "/opt/shepherd/manifests/twap-monitor.toml" + +[[modules]] +path = "/opt/shepherd/modules/ethflow_watcher.wasm" +manifest = "/opt/shepherd/manifests/ethflow-watcher.toml" +# Add price-alert / balance-tracker / stop-loss the same way. +``` + +If you want compose to use this file instead of the bundled +`engine.docker.toml`, set `SHEPHERD_ENGINE_CONFIG=./engine.local.toml` +in `.env` and put your file there (the `*.local.toml` pattern is +already gitignored). + +Public RPCs throttle `eth_subscribe` + `eth_getLogs` under sustained +load (independently confirmed by the baseline-latency tool — see +`docs/operations/baselines/`). The soak (COW-1031) explicitly +requires paid endpoints. + +--- + +## 3. Upgrade / rollback + +```bash +# Roll forward to the latest main-branch build. +docker compose pull +docker compose up -d # picks up the new image; graceful + # shutdown drains in-flight dispatch + # (COW-1072) before the new container + # takes over. + +# Roll back to a specific build. +export SHEPHERD_IMAGE=ghcr.io/bleu/nullis-shepherd:sha-abc1234 +docker compose up -d + +# Cold roll: stop, prune image, pull fresh. +docker compose down +docker image rm ghcr.io/bleu/nullis-shepherd:latest +docker compose pull && docker compose up -d +``` + +The `shepherd-state` named volume survives container recreation — +the redb file with all `submitted:` / `dropped:` / `backoff:` markers +persists across upgrades by design (idempotency lives there). + +--- + +## 4. Building the image locally + +The CI publishes on every push, so the local build path is only for +testing un-merged changes: + +```bash +docker compose build # uses repo-root Dockerfile +docker compose up -d # runs the locally-built image +``` + +To pin the locally-built tag and avoid accidentally pulling `:latest`: + +```bash +export SHEPHERD_IMAGE=shepherd:local +docker build -t "$SHEPHERD_IMAGE" . +docker compose up -d +``` + +--- + +## 5. Verifying the deploy + +```bash +# Engine is up, modules are loaded, no module is quarantined. +curl -s http://127.0.0.1:9100/metrics \ + | grep -E '^shepherd_(module_poisoned|module_restarts_total|stream_reconnects_total)' + +# Tail the structured logs. +docker compose logs -f shepherd | grep -E '"level":(("ERROR")|("WARN"))' + +# In a separate shell: confirm the engine wrote a last-dispatched- +# block marker after the first 30s of uptime (proof the supervisor +# is dispatching events, not just idle-looping). +docker compose exec shepherd ls -la /var/lib/shepherd/ +``` + +Green: `shepherd_module_poisoned == 0`, no ERROR/WARN lines beyond +boot, and a non-empty redb file under `/var/lib/shepherd/`. + +--- + +## 6. Cross-references + +- `docs/production.md` — full process-level deploy (systemd path), + backup recipes, RPC selection, alerting rules, runbook. +- `docs/06-production-hardening.md` — resource-limit design (fuel, + memory, storage), restart policy, RPC resilience, observability + design. +- `docs/operations/m3-testnet-runbook.md` — staging validation + playbook; reuse the same steps before the production soak. +- `engine.example.toml` — annotated reference for the engine config. diff --git a/docs/deployment/prometheus.yml b/docs/deployment/prometheus.yml new file mode 100644 index 00000000..74734160 --- /dev/null +++ b/docs/deployment/prometheus.yml @@ -0,0 +1,19 @@ +# Prometheus scrape config consumed by the `observability` profile in +# `docker-compose.yml`. Scrapes the engine's /metrics endpoint via the +# compose DNS name `shepherd:9100`. Adjust intervals if you front a +# Grafana stack and want denser samples. +# +# Metric surface is documented in `docs/production.md §7`. + +global: + scrape_interval: 15s + evaluation_interval: 30s + +scrape_configs: + - job_name: shepherd + static_configs: + - targets: ["shepherd:9100"] + labels: + # Pin a deployment label so a multi-VM setup can tag where + # the scrape came from. Override per deployment. + deployment: "shepherd-vm" diff --git a/docs/production.md b/docs/production.md index 642858e3..f06f3ae1 100644 --- a/docs/production.md +++ b/docs/production.md @@ -406,6 +406,17 @@ configured at boot. Public nodes throttle `eth_subscribe` and `eth_call` aggressively; production deployments **must** use a paid endpoint. +> **Use `wss://`, not `https://`.** `eth_subscribe` (the engine's +> block + log event source) is WebSocket-only in the JSON-RPC spec; +> HTTP transports return `"subscriptions are not available on this +> provider"` and the supervisor's COW-1071 reconnect backoff will +> loop forever waiting for a subscription that can never open. +> Every paid provider exposes both schemes per endpoint — pick the +> WS form. The engine surfaces a boot-time ERROR log line for any +> `http(s)://` `rpc_url`, with the exact `wss://` swap suggested. +> Set `[chains.] require_ws = false` to opt out (for poll-only +> deployments that never subscribe). + | Provider | Plan recommendation | Notes | |---|---|---| | Alchemy | Growth tier (≥ 660M CU/mo) | First-class WS pubsub; SLA-backed. | diff --git a/engine.docker.toml b/engine.docker.toml new file mode 100644 index 00000000..82784a1e --- /dev/null +++ b/engine.docker.toml @@ -0,0 +1,79 @@ +# Docker-ready engine config. Pre-wired for the file layout the +# repo's `Dockerfile` bakes: +# +# /opt/shepherd/modules/.wasm — compiled components +# /opt/shepherd/manifests/.toml — per-module manifests +# /var/lib/shepherd/ — redb state (named volume) +# +# Secrets come from env vars (see `.env.example`). The engine +# substitutes `${VAR_NAME}` tokens at load time; a missing variable +# fails fast with the exact name. `docker compose` reads the +# repo-root `.env` automatically and forwards the listed variables +# into the container. +# +# Workflow: +# +# cp .env.example .env +# $EDITOR .env # paste real wss:// RPC URLs +# docker compose up -d +# +# Mount target inside the container is `/etc/shepherd/engine.toml` +# (see docker-compose.yml). + +[engine] +state_dir = "/var/lib/shepherd" +log_level = "info" + +[engine.metrics] +enabled = true +# Bind to all interfaces inside the container so the compose port +# mapping (127.0.0.1:9100 host -> 9100 container) reaches it. NEVER +# publish `0.0.0.0:9100` on the host without authn — see +# docs/production.md §7. +bind_addr = "0.0.0.0:9100" + +# ---- chains ---- +# +# Drop any [chains.] entry whose `*_RPC_URL` env var isn't set. +# Engines that subscribe (the default) require `wss://`; opt-out per +# chain with `require_ws = false` for poll-only deployments. + +[chains.1] # Ethereum Mainnet +rpc_url = "${MAINNET_RPC_URL}" + +[chains.100] # Gnosis Chain +rpc_url = "${GNOSIS_RPC_URL}" + +[chains.11155111] # Sepolia (recommended for soak) +rpc_url = "${SEPOLIA_RPC_URL}" + +[chains.42161] # Arbitrum One +rpc_url = "${ARBITRUM_RPC_URL}" + +[chains.8453] # Base +rpc_url = "${BASE_RPC_URL}" + +# ---- modules ---- +# +# The image bakes all five production modules at the paths below. +# Comment out any you don't intend to run on this deployment. + +[[modules]] +path = "/opt/shepherd/modules/twap_monitor.wasm" +manifest = "/opt/shepherd/manifests/twap-monitor.toml" + +[[modules]] +path = "/opt/shepherd/modules/ethflow_watcher.wasm" +manifest = "/opt/shepherd/manifests/ethflow-watcher.toml" + +[[modules]] +path = "/opt/shepherd/modules/price_alert.wasm" +manifest = "/opt/shepherd/manifests/price-alert.toml" + +[[modules]] +path = "/opt/shepherd/modules/balance_tracker.wasm" +manifest = "/opt/shepherd/manifests/balance-tracker.toml" + +[[modules]] +path = "/opt/shepherd/modules/stop_loss.wasm" +manifest = "/opt/shepherd/manifests/stop-loss.toml" diff --git a/engine.example.toml b/engine.example.toml index d6513b0d..58747857 100644 --- a/engine.example.toml +++ b/engine.example.toml @@ -3,32 +3,52 @@ # Distinct from `nexum.toml` (per-module manifest): this file # describes the *engine*'s I/O wiring. Copy to `engine.toml` next to # the binary, or pass the path as the third positional argument. +# +# ## Secrets workflow +# +# Paid RPC URLs (Alchemy / drpc / Infura / QuickNode keys) live in +# environment variables, not in this file. The engine substitutes +# `${VAR_NAME}` tokens at load time: +# +# export MAINNET_RPC_URL=wss://eth-mainnet.g.alchemy.com/v2/ +# export GNOSIS_RPC_URL=wss://gnosis-mainnet.g.alchemy.com/v2/ +# export SEPOLIA_RPC_URL=wss://eth-sepolia.g.alchemy.com/v2/ +# export ARBITRUM_RPC_URL=wss://arb-mainnet.g.alchemy.com/v2/ +# export BASE_RPC_URL=wss://base-mainnet.g.alchemy.com/v2/ +# +# The Docker compose path picks these up from a gitignored `.env` +# file at the repo root — see `.env.example` and +# `docs/deployment/docker.md`. +# +# ## RPC scheme choice +# +# Every URL must be `wss://` (or `ws://`). The engine subscribes to +# blocks and logs via `eth_subscribe`, which is a **WebSocket-only** +# JSON-RPC method (HTTP transports return "subscriptions are not +# available on this provider"). The engine emits a boot-time ERROR +# if it sees an HTTP URL; set `[chains.] require_ws = false` to +# opt out for poll-only chains that never subscribe. [engine] -# Directory the local-store redb file (and future engine artefacts) -# will be created under. Created automatically at boot. state_dir = "./data" - -# `tracing_subscriber::EnvFilter`-compatible directive. `RUST_LOG` -# overrides at process start. log_level = "info" # One [chains.] table per chain the engine should be able to talk -# to. Chain ids are EVM decimal. `ws://` and `wss://` URLs engage -# alloy's pubsub transport (needed for `eth_subscribe`); `http://` and -# `https://` use the HTTP transport. +# to. Chain ids are EVM decimal. Drop any entry whose env var you +# haven't exported — `${VAR}` substitution fails fast with the exact +# missing variable named. -[chains.1] -rpc_url = "https://ethereum-rpc.publicnode.com" +[chains.1] # Ethereum Mainnet +rpc_url = "${MAINNET_RPC_URL}" -[chains.100] -rpc_url = "https://rpc.gnosischain.com" +[chains.100] # Gnosis Chain +rpc_url = "${GNOSIS_RPC_URL}" -[chains.11155111] -rpc_url = "wss://ethereum-sepolia-rpc.publicnode.com" +[chains.11155111] # Sepolia +rpc_url = "${SEPOLIA_RPC_URL}" -[chains.42161] -rpc_url = "https://arb1.arbitrum.io/rpc" +[chains.42161] # Arbitrum One +rpc_url = "${ARBITRUM_RPC_URL}" -[chains.8453] -rpc_url = "https://mainnet.base.org" +[chains.8453] # Base +rpc_url = "${BASE_RPC_URL}"