test(conformance): use ExecOutput::success for exec checks

Replace hand-written exit-code checks in the smoke, lifecycle, persistent_workspace, file_transfer and policy_advisor suites. Polls and checks that expect a failure keep reading the exit code.

Signed-off-by: Matthew Grossman <mgrossman@nvidia.com>

Author: matthewgrossman

tests/suites/conformance/src/file_transfer.rs

@@ -169,14 +169,11 @@ impl<'a> Transfer<'a> {
 
     /// Run a shell script in the sandbox and require it to succeed.
     async fn sh(&self, script: &str) -> testkit::Result {
-        let output = self.sandbox.exec(["sh", "-c", script]).await?;
-        testkit::ensure!(
-            output.exit_code == 0,
-            "`{script}` exited with {}; stderr: {}",
-            output.exit_code,
-            output.stderr
-        );
-        Ok(())
+        self.sandbox
+            .exec(["sh", "-c", script])
+            .await?
+            .success()
+            .map(drop)
     }
 
     async fn upload(

tests/suites/conformance/src/lifecycle.rs

@@ -24,13 +24,7 @@ async fn long_running_exec_delete(cx: &mut Context) -> testkit::Result {
     sandbox.wait_ready().await?;
 
     let marker = format!("openshell-e2e-{}", cx.run_id());
-    let output = sandbox.exec(["echo", marker.as_str()]).await?;
-    testkit::ensure!(
-        output.exit_code == 0,
-        "exec exited with {}; stderr: {}",
-        output.exit_code,
-        output.stderr
-    );
+    let output = sandbox.exec(["echo", marker.as_str()]).await?.success()?;
     testkit::ensure!(
         output.stdout == format!("{marker}\n"),
         "exec stdout was {:?}; expected {:?}",
@@ -276,13 +270,12 @@ async fn default_command_sandbox_is_listed_ready(cx: &mut Context) -> testkit::R
     cx.expect_listed(&sandbox, SandboxPhase::Ready).await?;
 
     let marker = format!("openshell-e2e-{}", cx.run_id());
-    let output = sandbox.exec(["echo", marker.as_str()]).await?;
+    let output = sandbox.exec(["echo", marker.as_str()]).await?.success()?;
     testkit::ensure!(
-        output.exit_code == 0 && output.stdout == format!("{marker}\n"),
-        "exec exited {} with stdout {:?}; stderr: {}",
-        output.exit_code,
+        output.stdout == format!("{marker}\n"),
+        "exec stdout was {:?}; expected {:?}",
         output.stdout,
-        output.stderr
+        format!("{marker}\n")
     );
     // The gateway's default image is OpenShell's Ubuntu base image.
     crate::image::expect_ubuntu(&sandbox).await?;
@@ -319,13 +312,15 @@ async fn restart_stopped_sandbox(cx: &mut Context) -> testkit::Result {
     );
 
     sandbox.start().await?;
-    let output = sandbox.exec(["echo", "started"]).await?;
+    let output = sandbox
+        .exec(["echo", "started"])
+        .await?
+        .success()
+        .context("exec after start")?;
     testkit::ensure!(
-        output.exit_code == 0 && output.stdout == "started\n",
-        "exec after start exited {} with stdout {:?}; stderr: {}",
-        output.exit_code,
-        output.stdout,
-        output.stderr
+        output.stdout == "started\n",
+        "exec after start printed {:?}; expected \"started\\n\"",
+        output.stdout
     );
     Ok(())
 }

tests/suites/conformance/src/persistent_workspace.rs

@@ -10,7 +10,7 @@
 
 use std::time::Duration;
 
-use testkit::{Context, Poll};
+use testkit::{Context, Poll, ResultExt as _};
 
 /// Files in the workspace survive stop and start, and start runs the main
 /// command again.
@@ -31,19 +31,15 @@ async fn persistent_workspace_survives_stop_start(cx: &mut Context) -> testkit::
     sandbox.wait_ready().await?;
 
     let sentinel = format!("openshell-stop-start-{}", cx.run_id());
-    let write = sandbox
+    sandbox
         .exec([
             "sh",
             "-c",
             &format!("printf '%s\\n' '{sentinel}' > '{SENTINEL}' && sync"),
         ])
-        .await?;
-    testkit::ensure!(
-        write.exit_code == 0,
-        "writing the sentinel exited {}; stderr: {}",
-        write.exit_code,
-        write.stderr
-    );
+        .await?
+        .success()
+        .context("write the sentinel")?;
 
     sandbox.stop().await?;
     sandbox.start().await?;

tests/suites/conformance/src/policy_advisor.rs

@@ -13,7 +13,7 @@ use std::time::{Duration, Instant};
 
 use serde_json::Value;
 use testkit::sdk::raw::proto;
-use testkit::{Context, Error, Sandbox};
+use testkit::{Context, Error, ResultExt as _, Sandbox};
 
 const READY_TIMEOUT: Duration = Duration::from_mins(4);
 const PROPOSAL_TIMEOUT: Duration = Duration::from_secs(90);
@@ -161,12 +161,13 @@ async fn denied_address_produces_draft_rule(cx: &mut Context) -> testkit::Result
             "printf 'BINARY=%s\\n' \"$(readlink -f /proc/$$/exe)\"; \
              if exec 3<>/dev/tcp/1.1.1.1/443; then echo UNEXPECTED_ALLOWED; exit 1; else echo DENIED; fi",
         ])
-        .await?;
+        .await?
+        .success()
+        .context("the connection to 1.1.1.1:443 must be denied")?;
     testkit::ensure!(
-        probe.exit_code == 0 && probe.stdout.lines().any(|line| line == "DENIED"),
-        "the connection to 1.1.1.1:443 was not denied: stdout {:?}, stderr {:?}",
-        probe.stdout,
-        probe.stderr
+        probe.stdout.lines().any(|line| line == "DENIED"),
+        "the connection to 1.1.1.1:443 was not denied: stdout {:?}",
+        probe.stdout
     );
     let binary = probe
         .stdout
@@ -199,12 +200,13 @@ async fn denied_hostname_produces_draft_rule(cx: &mut Context) -> testkit::Resul
             "-c",
             "if exec 3<>/dev/tcp/pypi.org/80; then echo UNEXPECTED_ALLOWED; exit 1; else echo DENIED; fi",
         ])
-        .await?;
+        .await?
+        .success()
+        .context("the connection to pypi.org:80 must be denied")?;
     testkit::ensure!(
-        probe.exit_code == 0 && probe.stdout.lines().any(|line| line == "DENIED"),
-        "the connection to pypi.org:80 was not denied: stdout {:?}, stderr {:?}",
-        probe.stdout,
-        probe.stderr
+        probe.stdout.lines().any(|line| line == "DENIED"),
+        "the connection to pypi.org:80 was not denied: stdout {:?}",
+        probe.stdout
     );
 
     expect_draft(
@@ -289,13 +291,13 @@ async fn bash_path(sandbox: &Sandbox) -> testkit::Result<String> {
             "-c",
             "printf '%s\\n' \"$(readlink -f /proc/$$/exe)\"",
         ])
-        .await?;
+        .await?
+        .success()?;
     let path = output.stdout.trim();
     testkit::ensure!(
-        output.exit_code == 0 && is_bash_path(path),
-        "expected one absolute Bash path; got {:?}, stderr {:?}",
-        output.stdout,
-        output.stderr
+        is_bash_path(path),
+        "expected one absolute Bash path; got {:?}",
+        output.stdout
     );
     Ok(path.to_string())
 }
@@ -326,13 +328,9 @@ async fn policy_local(
             path,
             body,
         ])
-        .await?;
-    testkit::ensure!(
-        output.exit_code == 0,
-        "{method} http://policy.local{path} exited {}; stderr: {}",
-        output.exit_code,
-        output.stderr
-    );
+        .await?
+        .success()
+        .context(format!("{method} http://policy.local{path}"))?;
     let (headers, json) = output.stdout.split_once("\r\n\r\n").ok_or_else(|| {
         Error::msg(format!(
             "{method} http://policy.local{path} returned no complete HTTP response: {:?}",

tests/suites/conformance/src/smoke.rs

@@ -17,13 +17,7 @@ async fn create_exec_delete(cx: &mut Context) -> testkit::Result {
     sandbox.wait_ready().await?;
 
     let marker = format!("openshell-e2e-{}", cx.run_id());
-    let output = sandbox.exec(["echo", marker.as_str()]).await?;
-    testkit::ensure!(
-        output.exit_code == 0,
-        "exec exited with {}; stderr: {}",
-        output.exit_code,
-        output.stderr
-    );
+    let output = sandbox.exec(["echo", marker.as_str()]).await?.success()?;
     testkit::ensure!(
         output.stdout == format!("{marker}\n"),
         "exec stdout was {:?}; expected {:?}",