refactor(testkit): make the sandbox and output helpers uniform

Move expect_listed, expect_absent, wait_absent and the CLI's sandbox get JSON (now get_json) from Context onto Sandbox, and drop delete_and_expect_absent. Give ExecOutput failure() and CliOutput expect_stdout(), so both outputs offer the same checks. Put with_deadline's description first, like poll_until. Make CliSandboxBuilder::run consume the builder like create, and rename its policy setter to policy_yaml. Update the testkit README's list of what a test can do.

Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>

Author: matthewgrossman

tests/suites/conformance/src/cli.rs

@@ -25,7 +25,7 @@ async fn cli_create_exec_list_delete(cx: &mut Context) -> testkit::Result {
 
     let sandbox = cx.cli_sandbox("cs").create().await?;
     let name = sandbox.name().to_string();
-    let got = cx.sandbox_details(&name).await?;
+    let got = sandbox.get_json().await?;
     testkit::ensure!(
         got["name"] == name.as_str(),
         "sandbox get returned {:?}; expected {name:?}",
@@ -41,22 +41,15 @@ async fn cli_create_exec_list_delete(cx: &mut Context) -> testkit::Result {
     testkit::ensure!(listed.is_some(), "sandbox {name} is not in sandbox list");
 
     let marker = format!("openshell-e2e-{}", cx.run_id());
-    let exec = cx
-        .cli([
-            "sandbox", "exec", "--name", &name, "--no-tty", "--", "echo", &marker,
-        ])
-        .await?
-        .success()?;
-    testkit::ensure!(
-        exec.stdout == format!("{marker}\n"),
-        "sandbox exec stdout was {:?}; expected {:?}",
-        exec.stdout,
-        format!("{marker}\n")
-    );
+    cx.cli([
+        "sandbox", "exec", "--name", &name, "--no-tty", "--", "echo", &marker,
+    ])
+    .await?
+    .expect_stdout(&format!("{marker}\n"))?;
 
     cx.cli(["sandbox", "delete", &name]).await?.success()?;
     wait_unlisted(cx, &name).await?;
-    cx.expect_absent(&sandbox).await
+    sandbox.expect_absent().await
 }
 
 /// A sandbox created with the CLI can be stopped, rejects `sandbox exec` while
@@ -88,7 +81,7 @@ async fn cli_stop_start_delete(cx: &mut Context) -> testkit::Result {
 
     cx.cli(["sandbox", "delete", &name]).await?.success()?;
     wait_unlisted(cx, &name).await?;
-    cx.expect_absent(&sandbox).await
+    sandbox.expect_absent().await
 }
 
 /// Wait until `sandbox list` no longer shows the sandbox.

tests/suites/conformance/src/filesystem_policy.rs

@@ -133,8 +133,8 @@ async fn replaced_policy_path_rejects_next_launch(cx: &mut Context) -> testkit::
         // The exec's own error is the rejection report, so keep it apart from
         // the deadline's.
         let next = with_deadline(
-            REJECTED_LAUNCH_TIMEOUT,
             &format!("({case}) the rejected launch"),
+            REJECTED_LAUNCH_TIMEOUT,
             async { Ok(sandbox.exec(["true"]).await) },
         )
         .await?;

tests/suites/conformance/src/lifecycle.rs

@@ -28,7 +28,8 @@ async fn long_running_exec_delete(cx: &mut Context) -> testkit::Result {
         .await?
         .expect_stdout(&format!("{marker}\n"))?;
 
-    cx.delete_and_expect_absent(&sandbox).await
+    sandbox.delete().await?;
+    sandbox.expect_absent().await
 }
 
 /// Longer than the supervisor's 30s startup recovery deadline.
@@ -52,8 +53,8 @@ async fn exec_outlives_startup_recovery_deadline(cx: &mut Context) -> testkit::R
         PAST_RECOVERY_DEADLINE.as_secs()
     );
     with_deadline(
-        PAST_RECOVERY_EXEC_DEADLINE,
         "the exec past the recovery deadline",
+        PAST_RECOVERY_EXEC_DEADLINE,
         sandbox.exec(["sh", "-c", script.as_str()]),
     )
     .await?
@@ -114,9 +115,12 @@ async fn delete_is_prompt_when_workload_is_slow_to_handle_term(
     // Boundary containment may kill the workload before its handler finishes;
     // this asserts bounded deletion, not a graceful workload exit.
     with_deadline(
-        DELETE_DEADLINE,
         &format!("deleting sandbox {}", sandbox.name()),
-        cx.delete_and_expect_absent(&sandbox),
+        DELETE_DEADLINE,
+        async {
+            sandbox.delete().await?;
+            sandbox.expect_absent().await
+        },
     )
     .await
 }
@@ -160,7 +164,8 @@ async fn run_to_completion_exit_delete(cx: &mut Context) -> testkit::Result {
         again.restart_count
     );
 
-    cx.delete_and_expect_absent(&sandbox).await
+    sandbox.delete().await?;
+    sandbox.expect_absent().await
 }
 
 /// A main command that exits during provisioning is reported as the
@@ -236,7 +241,7 @@ async fn default_command_sandbox_is_listed_ready(cx: &mut Context) -> testkit::R
         ready.name,
         sandbox.name()
     );
-    cx.expect_listed(&sandbox, SandboxPhase::Ready).await?;
+    sandbox.expect_listed(SandboxPhase::Ready).await?;
 
     let marker = format!("openshell-e2e-{}", cx.run_id());
     sandbox
@@ -246,7 +251,8 @@ async fn default_command_sandbox_is_listed_ready(cx: &mut Context) -> testkit::R
     // The gateway's default image is OpenShell's Ubuntu base image.
     crate::image::expect_ubuntu(&sandbox).await?;
 
-    cx.delete_and_expect_absent(&sandbox).await
+    sandbox.delete().await?;
+    sandbox.expect_absent().await
 }
 
 /// A stopped sandbox rejects exec, accepts a second stop, and returns to
@@ -291,7 +297,8 @@ async fn stopped_sandbox_can_be_deleted(cx: &mut Context) -> testkit::Result {
         .create_ready()
         .await?;
     sandbox.stop().await?;
-    cx.delete_and_expect_absent(&sandbox).await
+    sandbox.delete().await?;
+    sandbox.expect_absent().await
 }
 
 /// Where the main command of [`canonical_main_environment`] records what it
@@ -422,7 +429,7 @@ fi
         )
         .await?;
 
-    let initial = cx.sandbox_details(sandbox.name()).await?;
+    let initial = sandbox.get_json().await?;
     let initial_instance = main_instance(&initial).to_string();
     testkit::ensure!(
         !initial_instance.is_empty(),
@@ -438,7 +445,7 @@ fi
         RESTART_TIMEOUT,
         Duration::from_millis(250),
         async || {
-            let details = cx.sandbox_details(sandbox.name()).await?;
+            let details = sandbox.get_json().await?;
             let instance = main_instance(&details);
             let replaced = !instance.is_empty() && instance != initial_instance;
             Ok(
@@ -471,7 +478,7 @@ fi
 }
 
 /// The ID of the sandbox's current main process instance, from
-/// [`Context::sandbox_details`]. Empty if none is reported.
+/// [`Sandbox::get_json`]. Empty if none is reported.
 fn main_instance(details: &serde_json::Value) -> &str {
     details["main_process_instance_id"]
         .as_str()
@@ -509,7 +516,8 @@ async fn detached_no_keep_sandbox_is_removed_after_main_exits(cx: &mut Context)
             .success()?;
     }
     for (sandbox, _, code) in &sandboxes {
-        cx.wait_absent(sandbox, EPHEMERAL_CLEANUP_TIMEOUT)
+        sandbox
+            .wait_absent(EPHEMERAL_CLEANUP_TIMEOUT)
             .await
             .context(format!("ephemeral sandbox exited {code}"))?;
     }

tests/suites/conformance/src/smoke.rs

@@ -21,5 +21,6 @@ async fn create_exec_delete(cx: &mut Context) -> testkit::Result {
         .await?
         .expect_stdout(&format!("{marker}\n"))?;
 
-    cx.delete_and_expect_absent(&sandbox).await
+    sandbox.delete().await?;
+    sandbox.expect_absent().await
 }

tests/suites/drivers/podman/tests/oci_identity.rs

@@ -67,7 +67,7 @@ USER {OCI_IDENTITY}
     let sandbox = cx
         .cli_sandbox("oi")
         .image(&image.tag)
-        .policy(IMAGE_IDENTITY_POLICY)
+        .policy_yaml(IMAGE_IDENTITY_POLICY)
         .command(["sleep", "infinity"])
         .create()
         .await

tests/suites/features/host-gateway/tests/host_gateway.rs

@@ -63,7 +63,7 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result
     let sandbox = cx
         .cli_sandbox("hg")
         .image(&image)
-        .policy(&policy(server.port))
+        .policy_yaml(&policy(server.port))
         .command(["sh", "-c", "exec sleep infinity"])
         .create()
         .await

tests/suites/features/oci-image/tests/oci_image.rs

@@ -225,7 +225,7 @@ USER app
             );
             // Require the condition reason, so another provisioning failure
             // cannot pass as a workspace rejection.
-            let sandbox = oci.cx.sandbox_details(&name).await?;
+            let sandbox = oci.cx.adopt(&name).await?.get_json().await?;
             let reasons: Vec<&str> = sandbox["conditions"]
                 .as_array()
                 .into_iter()
@@ -284,7 +284,7 @@ impl Oci<'_> {
         );
         let mut builder = self.cx.cli_sandbox(short).image(&image.tag);
         if let Some(policy) = policy {
-            builder = builder.policy(policy);
+            builder = builder.policy_yaml(policy);
         }
         let name = builder
             .command(["sh", "-c", &main])

tests/testkit/README.md

@@ -63,14 +63,21 @@ fails. A killed test process skips cleanup, so names are unique to the run.
 
 `Context` currently provides:
 
-- `cx.sandbox(suffix)` to build and create a sandbox with a run-unique name.
-  The returned `Sandbox` can wait for ready or finished, stop, start, exec,
-  read its config and draft policy, set a setting, and delete.
+- `cx.sandbox(suffix)` to build a sandbox with a run-unique name, then
+  `create()` it or `create_ready()` to also wait for Ready. The returned
+  `Sandbox` can wait for ready or finished, stop, start, exec or
+  `poll_exec` until the output matches, read its config and draft policy, set
+  a setting, read `sandbox get` JSON with `get_json()`, and delete. Its
+  `expect_listed`, `expect_absent` and `wait_absent` check list and get.
+  Exec output has `success()`, `failure()` and `expect_stdout()`.
+- `cx.cli_sandbox(suffix)` for a sandbox only `openshell sandbox create`
+  can make, such as one with `policy_yaml(...)` or `--no-keep`. `create()`
+  detaches and returns the `Sandbox`; `run()` returns the attached CLI
+  output.
 - `cx.track(name)` to register a sandbox created some other way, such as
   through the CLI, for cleanup. Call it before the
   create, so a create that fails after the gateway accepted it is still cleaned
   up. `cx.adopt(name)` returns a `Sandbox` handle for one the CLI created.
-- `cx.expect_listed` and `cx.expect_absent` to check a sandbox in list and get.
 - `cx.expect_compute_driver(name)` to fail unless the gateway runs only that
   compute driver. Driver suites call it first.
 - `testkit::base_image()` for the image to create sandboxes from or build on:
@@ -80,11 +87,12 @@ fails. A killed test process skips cleanup, so names are unique to the run.
   (`e2e/python/Dockerfile.workload`). The tmachine installers load it into
   every environment; `mise run e2e:workload:build` builds it locally.
 - `cx.cli([...])` to run the installed CLI against the same gateway, with
-  `success()`, `failure()`, and `json()` on the output. `cx.cli_handle()`
+  `success()`, `failure()`, `expect_stdout()`, and `json()` on the output. `cx.cli_handle()`
   returns that CLI for code that can't borrow `cx`, such as a cleanup hook.
 - `cx.on_cleanup(name, hook)` to remove something when the test ends.
 - `cx.client()` for SDK calls testkit does not wrap.
-- `poll_until` for waiting on other state.
+- `poll_until` for waiting on other state, and `with_deadline(what, timeout,
+  future)` for a step that must finish sooner than its own timeout.
 
 ## Running against a gateway
 

tests/testkit/testkit/src/cli.rs

@@ -176,6 +176,18 @@ impl CliOutput {
         Ok(self)
     }
 
+    /// Fail unless the command exited 0 and printed exactly `expected`.
+    pub fn expect_stdout(self, expected: &str) -> Result<Self> {
+        let output = self.success()?;
+        crate::ensure!(
+            output.stdout == expected,
+            "`{}` printed {:?}; expected {expected:?}",
+            output.command,
+            output.stdout
+        );
+        Ok(output)
+    }
+
     /// Stdout and stderr as one line of words, for matching an error report.
     /// The CLI wraps reports to the terminal width and draws a gutter, so
     /// match against this rather than the raw output.

tests/testkit/testkit/src/cli_sandbox.rs

@@ -48,7 +48,7 @@ impl<'a> CliSandboxBuilder<'a> {
     /// Create the sandbox with this policy YAML (`--policy`). It is written
     /// to a temporary file that lives until the CLI exits.
     #[must_use]
-    pub fn policy(mut self, yaml: &str) -> Self {
+    pub fn policy_yaml(mut self, yaml: &str) -> Self {
         self.policy = Some(yaml.to_string());
         self
     }
@@ -79,7 +79,7 @@ impl<'a> CliSandboxBuilder<'a> {
     /// return the sandbox. It need not be Ready yet.
     pub async fn create(mut self) -> Result<Sandbox> {
         self.args.push("--detach".to_string());
-        self.run()
+        self.invoke()
             .await?
             .success()
             .context(format!("create sandbox {}", self.name))?;
@@ -89,7 +89,11 @@ impl<'a> CliSandboxBuilder<'a> {
     /// Create the sandbox attached to its main command and return the CLI's
     /// output, whatever its exit code. For a test of the main command's
     /// result, or of a create that must fail.
-    pub async fn run(&self) -> Result<CliOutput> {
+    pub async fn run(self) -> Result<CliOutput> {
+        self.invoke().await
+    }
+
+    async fn invoke(&self) -> Result<CliOutput> {
         self.cx.track(&self.name);
         let policy = self.policy.as_deref().map(PolicyFile::new).transpose()?;
         let mut args = vec!["sandbox", "create", "--name", &self.name, "--no-tty"];