Skip to content

Commit 9e004a7

Browse files
authored
ci(e2e): run the tmachine conformance rows with the prebuilt openshell-conformance (#5332)
* ci(e2e): run the tmachine conformance rows with the prebuilt openshell-conformance The host lanes already share one openshell-conformance build per architecture, but the tmachine conformance rows each cross-built a nextest archive. Add --program to openshell-test-runner to run a prebuilt testkit program in the guest, and have Branch E2E pass the build-conformance artifact to the conformance rows of e2e-suites.yml. Signed-off-by: Evan Lezar <elezar@nvidia.com> * ci(e2e): apply runner filters to a prebuilt test program Signed-off-by: Evan Lezar <elezar@nvidia.com> --------- Signed-off-by: Evan Lezar <elezar@nvidia.com>
1 parent 00222c2 commit 9e004a7

6 files changed

Lines changed: 214 additions & 68 deletions

File tree

‎.github/workflows/branch-e2e.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -296,14 +296,15 @@ jobs:
296296

297297
# Run driver-independent conformance tests.
298298
conformance-integration:
299-
needs: prepare-integration
299+
needs: [prepare-integration, build-conformance]
300300
permissions:
301301
actions: read
302302
contents: read
303303
packages: read
304304
uses: ./.github/workflows/e2e-suites.yml
305305
with:
306306
family: conformance
307+
prebuilt-conformance: true
307308
source-sha: ${{ needs.prepare-integration.outputs.source_sha }}
308309
integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
309310
secrets:

‎.github/workflows/e2e-suites.yml‎

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,10 @@
77
# optionally, the package in it, the gateway name the installer registers, and
88
# the runner label for the environment's architecture. `openshell-test-runner`
99
# builds the tests, boots the environment, runs them inside it and checks the
10-
# results. The `family` input selects the rows of one family, including the
11-
# workspaces under it (drivers selects drivers/podman); leave it empty to run
12-
# every row.
10+
# results; a conformance row can run the prebuilt `openshell-conformance`
11+
# instead (`prebuilt-conformance`). The `family` input selects the rows of one
12+
# family, including the workspaces under it (drivers selects drivers/podman);
13+
# leave it empty to run every row.
1314

1415
name: E2E Suites
1516

@@ -29,6 +30,11 @@ on:
2930
description: Immutable ID of the prepared integration inputs artifact
3031
required: true
3132
type: string
33+
prebuilt-conformance:
34+
description: Run the conformance rows with the openshell-conformance-x86_64-unknown-linux-musl artifact of this run instead of building the suites in every job
35+
required: false
36+
type: boolean
37+
default: false
3238
secrets:
3339
CACHIX_AUTH_TOKEN:
3440
description: Token used to push Nix builds, such as the test runner, to Cachix
@@ -161,8 +167,18 @@ jobs:
161167
artifact-ids: ${{ inputs['integration-inputs-artifact-id'] }}
162168
path: artifacts
163169

170+
# The program is built once per run (see build-conformance in
171+
# branch-e2e.yml), so the conformance rows need not each build the suites.
172+
- name: Download prebuilt openshell-conformance
173+
if: inputs.prebuilt-conformance && matrix.family == 'conformance'
174+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
175+
with:
176+
name: openshell-conformance-x86_64-unknown-linux-musl
177+
path: artifacts/prebuilt-conformance
178+
164179
- name: Run the suites in this environment
165180
env:
181+
PREBUILT_CONFORMANCE: ${{ inputs.prebuilt-conformance && matrix.family == 'conformance' }}
166182
ENVIRONMENT: ${{ matrix.environment }}
167183
INSTALLER: ${{ matrix.installer }}
168184
FAMILY: ${{ matrix.family }}
@@ -172,15 +188,21 @@ jobs:
172188
# Retry dependency preparation, then run the suites once. The runner
173189
# builds the suites for the guest, so it needs no prepared archives.
174190
nix build --no-link .#openshell-test-runner || nix build --no-link .#openshell-test-runner
175-
packages=()
176-
if [ -n "${PACKAGE}" ]; then
177-
packages=(--package "${PACKAGE}")
191+
# A prebuilt program replaces the build, and so the package choice.
192+
selection=()
193+
if [ "${PREBUILT_CONFORMANCE}" = "true" ]; then
194+
program="artifacts/prebuilt-conformance/openshell-conformance"
195+
chmod +x "${program}"
196+
"${program}" list > /dev/null
197+
selection=(--program "${program}")
198+
elif [ -n "${PACKAGE}" ]; then
199+
selection=(--package "${PACKAGE}")
178200
fi
179201
nix run .#openshell-test-runner -- \
180202
--environment "${ENVIRONMENT}" \
181203
--installer "${INSTALLER}" \
182204
--family "${FAMILY}" \
183-
"${packages[@]}" \
205+
"${selection[@]}" \
184206
--gateway "${GATEWAY}" \
185207
--results artifacts/test-results
186208

‎TESTING.md‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,15 @@ complete, for example because the environment didn't come up. It writes
248248
`artifacts/test-results/` (`--results`), even when the run timed out.
249249

250250
`--filter <filterset>` narrows the tests with a nextest filterset, and
251-
`--keep` leaves the VM running until you press Enter. Two runs on one host
251+
`--keep` leaves the VM running until you press Enter. `--program <path>` runs a
252+
prebuilt testkit program, such as a static `openshell-conformance` for the
253+
guest's architecture, instead of building the family's tests; it can't be
254+
combined with `--package`. With it, `--filter` (repeatable) takes parts of test
255+
names, as the program does, such as `--filter policy_advisor::`, rather than a
256+
nextest filterset. The runner runs the program's `ready`
257+
and `run` commands in the guest and still writes `junit.xml` and `nextest.log`
258+
(the program's output) to the results directory. Branch E2E builds
259+
`openshell-conformance` once and passes it to every conformance row this way. Two runs on one host
252260
need different `TMACHINE_SSH_PORT` values (default 2222). The installer
253261
inputs, such as the Debian package and the images, come from `artifacts/`,
254262
built by `nix run .#build-artifacts` or downloaded by CI.

‎tests/runner/src/bundle.rs‎

Lines changed: 52 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@
88
//! `nextest` config with the runner's `JUnit` profile, and `testkit-ready`,
99
//! which waits for the gateway before the tests run. The guest needs only the
1010
//! `cargo-nextest` binary to run it.
11+
//!
12+
//! A family that ships a prebuilt test program, such as `openshell-conformance`,
13+
//! can run it instead: the bundle then holds that program alone, built once
14+
//! elsewhere for the guest target, and the runner builds nothing.
1115
1216
use std::path::{Path, PathBuf};
1317
use std::process::Command;
@@ -34,13 +38,44 @@ slow-timeout = { period = "30s", terminate-after = 4 }
3438
path = "junit.xml"
3539
"#;
3640

41+
/// The prebuilt program's name in a program bundle.
42+
pub const PROGRAM_BINARY: &str = "test-program";
43+
44+
/// What a bundle runs the tests with.
45+
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
46+
pub enum Kind {
47+
/// A nextest archive, run with `cargo-nextest`.
48+
Nextest,
49+
/// A prebuilt testkit program (`list`, `ready` and `run`), run directly.
50+
Program,
51+
}
52+
3753
/// The bundle directory, removed when this is dropped.
38-
pub struct Bundle(TempDir);
54+
pub struct Bundle(TempDir, Kind);
3955

4056
impl Bundle {
4157
pub fn path(&self) -> &Path {
4258
self.0.path()
4359
}
60+
61+
pub const fn kind(&self) -> Kind {
62+
self.1
63+
}
64+
}
65+
66+
/// A bundle holding only `program`, a testkit program already built for the
67+
/// guest target. `program` must be a static binary, as the guest's libc is
68+
/// not known.
69+
pub fn program(program: &Path) -> Result<Bundle> {
70+
ensure!(
71+
program.is_file(),
72+
"--program {} is not a file",
73+
program.display()
74+
);
75+
let dir = tempfile::tempdir().context("create bundle directory")?;
76+
std::fs::copy(program, dir.path().join(PROGRAM_BINARY))
77+
.with_context(|| format!("copy {}", program.display()))?;
78+
Ok(Bundle(dir, Kind::Program))
4479
}
4580

4681
/// The Rust target for guests. tmachine only boots guests of the host's
@@ -137,7 +172,7 @@ pub fn build(workspace: &Path, packages: &[String], target: &str, ready: &Path)
137172
std::fs::write(stage.join("nextest.toml"), NEXTEST_CONFIG)?;
138173
std::fs::copy(ready, stage.join(READY_BINARY))
139174
.with_context(|| format!("copy {}", ready.display()))?;
140-
Ok(Bundle(dir))
175+
Ok(Bundle(dir, Kind::Nextest))
141176
}
142177

143178
/// Copy every `Cargo.toml` under `workspace` into `stage`, keeping relative
@@ -176,6 +211,21 @@ fn run(command: &mut Command) -> Result<()> {
176211
mod tests {
177212
use super::*;
178213

214+
#[test]
215+
fn program_bundle_holds_only_the_program() {
216+
let dir = tempfile::tempdir().unwrap();
217+
let path = dir.path().join("openshell-conformance");
218+
std::fs::write(&path, b"binary").unwrap();
219+
let bundle = program(&path).unwrap();
220+
assert_eq!(bundle.kind(), Kind::Program);
221+
let names: Vec<_> = std::fs::read_dir(bundle.path())
222+
.unwrap()
223+
.map(|entry| entry.unwrap().file_name())
224+
.collect();
225+
assert_eq!(names, [PROGRAM_BINARY]);
226+
assert!(program(&dir.path().join("missing")).is_err());
227+
}
228+
179229
#[test]
180230
fn built_executable_finds_the_named_binary() {
181231
let messages = br#"{"reason":"compiler-artifact","target":{"name":"testkit"},"executable":null}

‎tests/runner/src/execute.rs‎

Lines changed: 85 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,12 @@ use anyhow::{Context, Result, ensure};
2020
use tmachine::Session;
2121
use tmachine::config::Config;
2222

23-
use crate::bundle::{Bundle, PROFILE, READY_BINARY};
23+
use crate::bundle::{Bundle, Kind, PROFILE, PROGRAM_BINARY, READY_BINARY};
2424
use crate::test_env;
2525

2626
const GUEST_DIR: &str = "/var/tmp/openshell-tests";
27-
/// The nextest log's name, in the guest directory and in the results.
27+
/// The test log's name, in the guest directory and in the results. A program
28+
/// bundle's log keeps the name, so consumers of the results see one layout.
2829
const NEXTEST_LOG: &str = "nextest.log";
2930
/// How long the gateway has to become healthy after the environment is up.
3031
// Matches the installers' own wait: an HA k3s gateway booted from a cached
@@ -41,6 +42,9 @@ const READY_SLACK: Duration = Duration::from_mins(1);
4142
const NEXTEST_TEST_FAILED: i32 = 100;
4243
/// nextest's exit code when `--no-tests fail` found no tests to run.
4344
const NEXTEST_NO_TESTS_RUN: i32 = 4;
45+
/// A testkit program's exit code when a test failed. It exits 2 when tests
46+
/// could not run.
47+
const PROGRAM_TEST_FAILED: i32 = 1;
4448
/// How much of the end of the nextest log to print after a failure. nextest
4549
/// ends its output with a summary that names the failed tests.
4650
const LOG_TAIL_LINES: usize = 40;
@@ -91,6 +95,18 @@ impl Status {
9195
)),
9296
}
9397
}
98+
99+
/// The status of a testkit program's `run` that exited with `status`.
100+
fn of_program(status: ExitStatus, log: &Path) -> Self {
101+
match status.code() {
102+
Some(0) => Self::Passed,
103+
Some(PROGRAM_TEST_FAILED) => Self::Failed,
104+
_ => Self::Error(format!(
105+
"the test program exited with {status}; see {}",
106+
log.display()
107+
)),
108+
}
109+
}
94110
}
95111

96112
impl fmt::Display for Status {
@@ -112,8 +128,9 @@ pub struct Run<'a> {
112128
/// The gateway name the installer registers with the CLI.
113129
pub gateway: &'a str,
114130
pub bundle: &'a Bundle,
115-
/// A nextest filterset (`-E`) narrowing the tests in the bundle.
116-
pub filter: Option<&'a str>,
131+
/// Narrowing the tests in the bundle: nextest filtersets (`-E`) for a
132+
/// nextest bundle, parts of test names for a program bundle.
133+
pub filters: &'a [String],
117134
/// Where `junit.xml`, `nextest.log` and, after a failure, the guest's
118135
/// `diagnostics/` go.
119136
pub results: &'a Path,
@@ -244,15 +261,20 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result<Status> {
244261
.exec(&["mkdir", "-p", &format!("{guest}/extract")])
245262
.await?,
246263
)?;
264+
let kind = run.bundle.kind();
247265

248266
// A booted guest restarts its gateway, so wait for it once here rather
249267
// than in every test. Failing here is an environment error, not a test
250268
// failure.
269+
let program = format!("{guest}/{PROGRAM_BINARY}");
251270
let ready_binary = format!("{guest}/{READY_BINARY}");
252271
let ready_timeout = READY_TIMEOUT_SECS.to_string();
253272
let mut ready = vec!["env"];
254273
ready.extend(env.iter().map(String::as_str));
255-
ready.extend([ready_binary.as_str(), ready_timeout.as_str()]);
274+
match kind {
275+
Kind::Nextest => ready.extend([ready_binary.as_str(), ready_timeout.as_str()]),
276+
Kind::Program => ready.extend([program.as_str(), "ready", "--timeout", &ready_timeout]),
277+
}
256278
checked(
257279
"wait for the gateway",
258280
session
@@ -282,36 +304,52 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result<Status> {
282304
)?;
283305
}
284306

307+
let junit = format!("{guest}/junit.xml");
285308
let archive = format!("{guest}/tests.tar.zst");
286309
let extract = format!("{guest}/extract");
287310
let config = format!("{guest}/nextest.toml");
288-
let mut nextest = vec!["env"];
289-
nextest.extend(env.iter().map(String::as_str));
290-
nextest.extend([
291-
"cargo-nextest",
292-
"nextest",
293-
"run",
294-
"--archive-file",
295-
&archive,
296-
"--workspace-remap",
297-
guest,
298-
"--extract-to",
299-
&extract,
300-
"--extract-overwrite",
301-
"--config-file",
302-
&config,
303-
"--profile",
304-
PROFILE,
305-
"--no-capture",
306-
// Tests share one gateway and the guest has 4 GiB; run them serially
307-
// whatever --no-capture implies.
308-
"--test-threads",
309-
"1",
310-
"--no-tests",
311-
"fail",
312-
]);
313-
if let Some(filter) = run.filter {
314-
nextest.extend(["-E", filter]);
311+
let mut tests = vec!["env"];
312+
tests.extend(env.iter().map(String::as_str));
313+
match kind {
314+
Kind::Nextest => {
315+
tests.extend([
316+
"cargo-nextest",
317+
"nextest",
318+
"run",
319+
"--archive-file",
320+
&archive,
321+
"--workspace-remap",
322+
guest,
323+
"--extract-to",
324+
&extract,
325+
"--extract-overwrite",
326+
"--config-file",
327+
&config,
328+
"--profile",
329+
PROFILE,
330+
"--no-capture",
331+
// Tests share one gateway and the guest has 4 GiB; run them
332+
// serially whatever --no-capture implies.
333+
"--test-threads",
334+
"1",
335+
"--no-tests",
336+
"fail",
337+
]);
338+
for filter in run.filters {
339+
tests.extend(["-E", filter]);
340+
}
341+
}
342+
// The program runs its tests one at a time, and writes the report
343+
// where `collect_results` looks for it.
344+
Kind::Program => {
345+
tests.extend([program.as_str(), "run", "--junit", &junit]);
346+
// `run` takes name filters as arguments, after `--` so that one
347+
// starting with a dash isn't read as an option.
348+
if !run.filters.is_empty() {
349+
tests.push("--");
350+
tests.extend(run.filters.iter().map(String::as_str));
351+
}
352+
}
315353
}
316354
// nextest writes its output to a file in the guest, so a run that times
317355
// out or loses SSH still leaves a log for `collect_results`. The guest
@@ -323,12 +361,13 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result<Status> {
323361
test_env::export_script(test_env::ENV_DIR)
324362
);
325363
let mut command = vec!["sh", "-c", &script, &log];
326-
command.extend(nextest);
364+
command.extend(tests);
327365
let output = session.exec_timeout(&command, RUN_TIMEOUT).await?;
328-
Ok(Status::of_nextest(
329-
output.status,
330-
&run.results.join(NEXTEST_LOG),
331-
))
366+
let log = run.results.join(NEXTEST_LOG);
367+
Ok(match kind {
368+
Kind::Nextest => Status::of_nextest(output.status, &log),
369+
Kind::Program => Status::of_program(output.status, &log),
370+
})
332371
}
333372

334373
/// Copy the nextest log and the JUnit report out of the guest, whether or not
@@ -488,6 +527,15 @@ mod tests {
488527
assert!(matches!(status(1), Status::Error(_)));
489528
}
490529

530+
#[test]
531+
fn status_follows_the_program_exit_code() {
532+
let status = |code| Status::of_program(ExitStatus::from_raw(code << 8), Path::new("log"));
533+
assert_eq!(status(0), Status::Passed);
534+
assert_eq!(status(PROGRAM_TEST_FAILED), Status::Failed);
535+
// Exit 2: tests could not run.
536+
assert!(matches!(status(2), Status::Error(_)));
537+
}
538+
491539
#[test]
492540
fn k3s_diagnostics_keep_everything_the_old_playbook_collected() {
493541
for wanted in [

0 commit comments

Comments
 (0)