diff --git a/deploy/stacks/nvcf-compute-plane/tests/register-cluster.sh b/deploy/stacks/nvcf-compute-plane/tests/register-cluster.sh index d5b8cc32a..dc322bf00 100755 --- a/deploy/stacks/nvcf-compute-plane/tests/register-cluster.sh +++ b/deploy/stacks/nvcf-compute-plane/tests/register-cluster.sh @@ -94,6 +94,24 @@ fi values="${test_dir}/compute-plane/registration/gpu-a-register-values.yaml" grep -q '^clusterID: generated-id$' "${values}" +no_config_record="${test_dir}/cli-args-no-config" +FAKE_CLI_RECORD="${no_config_record}" make -C "${test_dir}/compute-plane" register-cluster \ + CLUSTER_NAME=gpu-b \ + NVCF_CLI="${fake_cli}" +no_config_args=() +while IFS= read -r arg; do + no_config_args+=("${arg}") +done < "${no_config_record}" +if [[ "${no_config_args[0]}" != "self-hosted" ]]; then + printf 'register-cluster added arguments before self-hosted without NVCF_CLI_CONFIG: %q\n' \ + "${no_config_args[0]}" >&2 + exit 1 +fi +if printf '%s\n' "${no_config_args[@]}" | grep -Fxq -- '--config'; then + echo "register-cluster passed --config without NVCF_CLI_CONFIG" >&2 + exit 1 +fi + rm "${profile}" "${record}" if FAKE_CLI_RECORD="${record}" make -C "${test_dir}/compute-plane" register-cluster \ CLUSTER_NAME=gpu-a \ diff --git a/src/clis/nvcf-cli/internal/openbao/client.go b/src/clis/nvcf-cli/internal/openbao/client.go index 3c12bf6ac..7cbe658b3 100644 --- a/src/clis/nvcf-cli/internal/openbao/client.go +++ b/src/clis/nvcf-cli/internal/openbao/client.go @@ -396,15 +396,8 @@ func (c *Client) generateUserJWTTokenWithSubject(ctx context.Context, vaultToken // mount, for example services/all/pki/root/cert/ca. The returned value is PEM // text suitable for a public trust bundle. func (c *Client) ReadPKICertificatePEM(ctx context.Context, pkiPath string) (string, error) { - rootToken, err := c.getOpenBaoRootToken() - if err != nil { - return "", fmt.Errorf("retrieving OpenBao root token: %w", err) - } readURL := strings.TrimRight(c.config.OpenBaoURL, "/") + "/v1/" + strings.Trim(pkiPath, "/") + "/cert/ca" - curlArgs := []string{ - "curl", "-sS", readURL, - "-H", "X-Vault-Token: " + rootToken, - } + curlArgs := []string{"curl", "-sS", readURL} return readPKICertificatePEM(ctx, 3, 2*time.Second, func(ctx context.Context) (string, error) { return c.executeKubectlRun(ctx, "openbao-pki-root-ca", curlArgs) }) diff --git a/src/clis/nvcf-cli/internal/openbao/client_test.go b/src/clis/nvcf-cli/internal/openbao/client_test.go index 09f6abfa0..8ccd8c38a 100644 --- a/src/clis/nvcf-cli/internal/openbao/client_test.go +++ b/src/clis/nvcf-cli/internal/openbao/client_test.go @@ -21,7 +21,9 @@ import ( "context" "encoding/json" "errors" + "os" "os/exec" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -164,3 +166,39 @@ func TestExecuteKubectlRunPreservesCommandError(t *testing.T) { var execErr *exec.Error require.ErrorAs(t, err, &execErr) } + +func TestReadPKICertificatePEMUsesPublicEndpointWithoutRootToken(t *testing.T) { + testDir := t.TempDir() + commandLog := filepath.Join(testDir, "kubectl.log") + kubectlPath := filepath.Join(testDir, "kubectl") + kubectlScript := `#!/bin/sh +printf '%s\n' "$*" >> "$KUBECTL_COMMAND_LOG" +case " $* " in + *" get secret "*) exit 91 ;; + *" X-Vault-Token: "*) exit 92 ;; +esac +printf '%s\n' '{"data":{"certificate":"-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n"}}' +` + require.NoError(t, os.WriteFile(kubectlPath, []byte(kubectlScript), 0o755)) + t.Setenv("PATH", testDir+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("KUBECTL_COMMAND_LOG", commandLog) + + client := NewClient(&Config{ + OpenBaoURL: "http://openbao-openbao.nvcf.svc.cluster.local:8200", + OpenBaoNamespace: "openbao", + OpenBaoSecretName: "openbao-root-token", + ClusterNamespace: "nvcf", + UtilityImage: "curlimages/curl:latest", + }, nil) + + got, err := client.ReadPKICertificatePEM(context.Background(), "services/all/pki/root") + require.NoError(t, err) + assert.Equal(t, openBaoTestCertPEM, got) + + logBody, err := os.ReadFile(commandLog) + require.NoError(t, err) + commands := string(logBody) + assert.NotContains(t, commands, " get secret ") + assert.NotContains(t, commands, "X-Vault-Token") + assert.Contains(t, commands, "/v1/services/all/pki/root/cert/ca") +} diff --git a/tests/bdd/AGENTS.md b/tests/bdd/AGENTS.md index 26dc9d0c9..19547bced 100644 --- a/tests/bdd/AGENTS.md +++ b/tests/bdd/AGENTS.md @@ -103,6 +103,11 @@ logic into `dsl/`. token never appears in argv or per-command logs. Do not introduce step handlers that capture secrets into env vars; relying on the state file keeps the JWT out of `.cmd` lines. +- The live runner installs SIGINT and SIGTERM cleanup before scenarios run. + Interrupt cleanup cancels the active step and its Unix process group, waits + for that step to stop writing, then restores the same file and environment + ledgers while preventing later steps from starting. Ledger-backed generated + registry credentials must not remain after an interrupted run. - Pre-suite destructive cleanup is governed by the single env var `BDD_CLEANUP_MODE`. Valid values: `stack-single`, `stack-multi`, `topology-single`, `topology-multi`, or unset. Unknown values fail @@ -138,7 +143,7 @@ logic into `dsl/`. deletion that catches topology infrastructure (`eg` in `envoy-gateway-system`, the namespace itself, `cert-manager`). -## CLI vs Helmfile install paths (two intentionally distinct workflows) +## CLI vs Helmfile install paths The suite exercises two operator workflows that share a stack but differ in how the control plane is installed. Future changes must keep the CLI install @@ -190,14 +195,15 @@ multi-cluster feature: refused` against an in-cluster hostname. 2. Wrong kubectl context when `make register-cluster` runs. The - `nvcf-cli cluster register` command auto-discovers OIDC issuer - and JWKS from the CURRENT context by spawning a probe Job in - that cluster, then registers that identity with ICMS. If the + `self-hosted compute-plane register` command discovers OIDC issuer + and JWKS from its selected compute context, then registers that + identity with ICMS. If the context is the cp cluster, ICMS records the cp cluster's JWKS for the compute cluster's row. The compute cluster's NVCA agent then 401s against ICMS at runtime ("Signed JWT rejected: ... no matching key(s) found"). Switch the context to the - compute cluster BEFORE `make register-cluster`, not after. + compute context explicitly through `COMPUTE_KUBE_CONTEXT` (or a + compute-scoped kubeconfig) before `make register-cluster`, not after. ## Tests diff --git a/tests/bdd/PLAN.md b/tests/bdd/PLAN.md index 51173200d..e53c0aecf 100644 --- a/tests/bdd/PLAN.md +++ b/tests/bdd/PLAN.md @@ -170,6 +170,7 @@ original order. Repeated options and empty values are preserved. | `Then Helm release {string} in namespace {string} using context {string} should contain values:` (YAML docstring) | Runs one explicit-context `helm get values -o yaml` for the named release and asserts that its values contain the supplied YAML subset. Extra map keys are allowed; lists remain order- and length-sensitive. Failure messages name the release and first differing path without printing release values. | | `Then Kubernetes resource {string} in namespace {string} using context {string} should contain:` (YAML docstring) | The resource is explicit `kind/name`. Runs one `kubectl get -o yaml` against the named context and asserts that the resource YAML contains the supplied YAML subset. Extra map keys are allowed; lists remain order- and length-sensitive. Failure messages name the resource and first differing path without printing resource values. | | `Then the rendered manifests in {string} should contain:` (table) | Requires a `text` header and one or more fixed strings. Recursively inspects regular files under the repo-relative directory and fails if any listed string is absent. `${VAR}` expansion applies to the path and table values. | +| `Then the rendered manifests in {string} should contain Kubernetes resource {string}` | Parses rendered YAML documents and requires an actual top-level resource matching the explicit `kind/name`. Nested references such as `Certificate.spec.issuerRef` do not satisfy the assertion. `${VAR}` expansion applies to the path, kind, and name. | | `Then the rendered manifests in {string} under directories matching {string} should contain:` (table) | Positive rendered-manifest assertion scoped to files below a directory whose name matches the supplied shell pattern, such as `*-nats`. The render directory, directory-name pattern, and table values support `${VAR}` expansion. | | `Then the rendered manifests in {string} should not contain:` (table) | Requires a `text` header and one or more fixed strings. Recursively inspects regular files under the repo-relative directory and fails if any listed string appears. `${VAR}` expansion applies to the path and table values. | | `Then these Helm releases should be deployed using context {string}:` (table) | Requires `name` and `namespace` headers, with an optional `revision` header. Runs one explicit-context, all-namespaces `helm list` and asserts that every listed release has status `deployed`; non-empty revision cells are also matched. | @@ -238,6 +239,15 @@ contract verified in `src/clis/nvcf-cli/cmd/`): ``` ${NVCF_CLI} --config self-hosted --control-plane-stack deploy/stacks/self-managed --compute-plane-stack deploy/stacks/nvcf-compute-plane --env local --plain compute-plane register --control-plane-profile --cluster-name --kube-context k3d- --region us-west-1 --output ``` +- Helmfile control-plane profile handoff (single cluster): + ``` + ${NVCF_CLI} --config self-hosted --control-plane-stack deploy/stacks/self-managed --env control-plane profile export --cluster-name + make -C deploy/stacks/nvcf-compute-plane register-cluster CLUSTER_NAME= CONTROL_PLANE_PROFILE= COMPUTE_KUBE_CONTEXT=k3d- NVCF_CLI=${NVCF_CLI} + ``` + The profile export runs after the selected Helmfile environment is installed + so endpoint and PKI trust data describe that deployment. A single-cluster + export omits both persistent context flags; the CLI accepts a split-cluster + pair or neither, and the bootstrap has already selected the local context. - `self-hosted compute-plane install`: ``` ${NVCF_CLI} --config self-hosted --control-plane-stack deploy/stacks/self-managed --compute-plane-stack deploy/stacks/nvcf-compute-plane --env local --plain compute-plane install --values --kube-context k3d- --cluster-name @@ -257,6 +267,11 @@ restoration ledger: - At suite teardown, the runner restores every registered path to its pre-suite state. Files that did not exist before are deleted; files that did are rewritten with the original bytes and mode. +- Live entry points cancel and quiesce the active step before restoring the + file and environment ledgers and exiting on SIGINT or SIGTERM. On Unix, the + command runner cancels the step's process group so shell, make, and kubectl + descendants cannot outlive restoration. This includes generated registry + credential files. - `Config.LedgerDir` (`out//originals/`) is reserved for an on-disk variant if very large fixtures ever push memory limits. Today the directory is created but unused. diff --git a/tests/bdd/dsl/manifests.go b/tests/bdd/dsl/manifests.go index 8912e3d4f..ff46039db 100644 --- a/tests/bdd/dsl/manifests.go +++ b/tests/bdd/dsl/manifests.go @@ -18,9 +18,15 @@ limitations under the License. package dsl import ( + "bytes" "encoding/base64" "encoding/json" "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" "gopkg.in/yaml.v3" ) @@ -30,6 +36,84 @@ const ( ngcDockerUsername = "$oauthtoken" ) +// RenderedManifestsContainResource parses rendered YAML documents below root +// and requires one top-level Kubernetes resource with the requested kind and +// metadata.name. Nested references such as Certificate.spec.issuerRef do not +// satisfy the assertion. +func RenderedManifestsContainResource(root string, resource KubernetesResource) error { + root = strings.TrimSpace(Interpolate(root)) + resource.Kind = strings.TrimSpace(Interpolate(resource.Kind)) + resource.Name = strings.TrimSpace(Interpolate(resource.Name)) + if root == "" { + return fmt.Errorf("rendered manifests directory is empty") + } + if resource.Kind == "" { + return fmt.Errorf("Kubernetes resource kind is empty") + } + if resource.Name == "" { + return fmt.Errorf("Kubernetes resource name is empty") + } + info, err := os.Stat(root) + if err != nil { + return fmt.Errorf("inspect rendered manifests directory %q: %w", root, err) + } + if !info.IsDir() { + return fmt.Errorf("rendered manifests path %q is not a directory", root) + } + + yamlFilesInspected := 0 + found := false + err = filepath.WalkDir(root, func(filePath string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return fmt.Errorf("inspect rendered manifest %q: %w", filePath, walkErr) + } + if !entry.Type().IsRegular() { + return nil + } + extension := strings.ToLower(filepath.Ext(filePath)) + if extension != ".yaml" && extension != ".yml" { + return nil + } + yamlFilesInspected++ + + manifestBody, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("read rendered manifest %q: %w", filePath, err) + } + + decoder := yaml.NewDecoder(bytes.NewReader(manifestBody)) + for document := 1; ; document++ { + var manifest struct { + Kind string `yaml:"kind"` + Metadata struct { + Name string `yaml:"name"` + } `yaml:"metadata"` + } + if err := decoder.Decode(&manifest); err != nil { + if err == io.EOF { + break + } + return fmt.Errorf("parse rendered manifest %q document %d: invalid YAML", filePath, document) + } + if manifest.Kind == resource.Kind && manifest.Metadata.Name == resource.Name { + found = true + return fs.SkipAll + } + } + return nil + }) + if err != nil { + return err + } + if found { + return nil + } + if yamlFilesInspected == 0 { + return fmt.Errorf("rendered manifests directory %q contains no YAML files", root) + } + return fmt.Errorf("rendered manifests in %q do not contain Kubernetes resource %s/%s", root, resource.Kind, resource.Name) +} + // NamespaceManifest returns a v1/Namespace YAML manifest body. The // returned slice is the file contents the caller writes to disk and // hands to kubectl apply. diff --git a/tests/bdd/dsl/manifests_test.go b/tests/bdd/dsl/manifests_test.go index faa0c5176..e0814cd64 100644 --- a/tests/bdd/dsl/manifests_test.go +++ b/tests/bdd/dsl/manifests_test.go @@ -18,10 +18,61 @@ limitations under the License. package dsl import ( + "os" + "path/filepath" "strings" "testing" ) +func TestRenderedManifestsContainResourceRejectsIssuerRefFragments(t *testing.T) { + root := t.TempDir() + body := `apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: llm-router-serving-cert +spec: + issuerRef: + kind: ClusterIssuer + name: nvcf-openbao-pki +` + if err := os.WriteFile(filepath.Join(root, "certificate.yaml"), []byte(body), 0o644); err != nil { + t.Fatalf("write rendered Certificate: %v", err) + } + + err := RenderedManifestsContainResource(root, KubernetesResource{ + Kind: "ClusterIssuer", + Name: "nvcf-openbao-pki", + }) + if err == nil { + t.Fatal("issuerRef fragments were mistaken for a rendered ClusterIssuer resource") + } +} + +func TestRenderedManifestsContainResourceFindsTopLevelResource(t *testing.T) { + root := t.TempDir() + body := `apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: llm-router-serving-cert +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: nvcf-openbao-pki +` + if err := os.WriteFile(filepath.Join(root, "pki.yaml"), []byte(body), 0o644); err != nil { + t.Fatalf("write rendered PKI resources: %v", err) + } + + err := RenderedManifestsContainResource(root, KubernetesResource{ + Kind: "ClusterIssuer", + Name: "nvcf-openbao-pki", + }) + if err != nil { + t.Fatalf("find rendered ClusterIssuer: %v", err) + } +} + func TestNamespaceManifestShape(t *testing.T) { body, err := NamespaceManifest("nvcf") if err != nil { diff --git a/tests/bdd/features/single-cluster-helmfile-llm-pki.feature b/tests/bdd/features/single-cluster-helmfile-llm-pki.feature new file mode 100644 index 000000000..6860a892c --- /dev/null +++ b/tests/bdd/features/single-cluster-helmfile-llm-pki.feature @@ -0,0 +1,244 @@ +@ncp-local @single-cluster @helmfile @pki +Feature: Install a local single-cluster NVCF stack with PKI-secured LLM transport + As a self-managed NVCF operator, + I want the Helmfile workflow with the LLM PKI addon enabled, + so that an LLM function answers invocations over a QUIC tunnel whose + trust chain is issued by the stack's own PKI. + + # Owns its own Helmfile environment (local-bdd-pki) for install-time PKI + # values. The exported profile carries the OpenBao root CA and fingerprint + # into registration values. The shared compute fixture uses secure QUIC so + # the registered bundle trust remains active. + + Rule: Helmfile installs the control plane with the LLM PKI addon + + Background: + Given these environment variables are set: + | name | + | NGC_API_KEY | + | NVCF_CLI | + | REPO_ROOT | + | SAMPLE_NGC_ORG | + | SAMPLE_NGC_TEAM | + And I copy the file "tests/bdd/fixtures/self-managed-local-bdd.yaml" to "deploy/stacks/self-managed/environments/local-bdd-pki.yaml" + # PKI render requirements: dnsNames must cover the router's + # advertised hostname (a single replica advertises its plain + # service DNS name), allowedDomains constrains the OpenBao + # signing role, and the provisioning hook needs the + # nvcf-openbao-migrations tag (no default propagates from env). + And I update yaml file "deploy/stacks/self-managed/environments/local-bdd-pki.yaml" with keys: + | global.imagePullSecrets[0].name | nvcr-pull-secret | + | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | addons.llm.pki.enabled | true | + | addons.llm.pki.dnsNames[0] | llm-request-router.nvcf.svc.cluster.local | + | addons.llm.pki.allowedDomains | nvcf.svc.cluster.local | + | addons.llm.pki.image.tag | 0.16.2 | + | observability.profile | disabled | + And I copy the file "tests/bdd/fixtures/nvcf-compute-plane-local-bdd.yaml" to "deploy/stacks/nvcf-compute-plane/environments/local-bdd-pki.yaml" + And I update yaml file "deploy/stacks/nvcf-compute-plane/environments/local-bdd-pki.yaml" with keys: + | global.imagePullSecrets[0].name | nvcr-pull-secret | + | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | + | observability.profile | disabled | + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-pki-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential + # Conflict precheck: ncp-local-cp's k3d serverlb claims + # 0.0.0.0:8080/8443/10081, NATS on 4222, and the worker + # callback port 10086, overlapping host ports single-cluster + # ncp-local needs. Fail loudly so the operator runs + # `make -C tools/ncp-local-cluster destroy-multicluster` + # before retrying. `k3d cluster get` exits 1 when absent (k3d v5). + When I run command "k3d cluster get ncp-local-cp" + And the command exit code should be 1 + And a single-cluster ncp-local cluster is running + And the "nvcr-pull-secret" image pull secret exists in namespaces: + | cassandra-system | + | nats-system | + | nvcf | + | api-keys | + | ess | + | sis | + | vault-system | + | nvca-operator | + | cert-manager | + + @llm-pki-render + Scenario: Operator validates the PKI-enabled environment renders + When I run command "make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-pki" + Then the command exit code should be 0 + And the rendered manifests in "deploy/stacks/self-managed/out" should contain Kubernetes resource "ClusterIssuer/nvcf-openbao-pki" + And the rendered manifests in "deploy/stacks/self-managed/out" should contain: + | text | + | name: ADDONS_LLM_ENABLED | + | value: "true" | + | llm-request-router.nvcf.svc.cluster.local | + | name: NVCF_SERVICE_PKI_ALLOWED_DOMAINS | + | value: "nvcf.svc.cluster.local" | + | nvcf-openbao-migrations:0.16.2 | + + @llm-pki-install + Scenario: Operator installs the control plane with the PKI addon enabled + When I run command "make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd-pki" + + Then the command exit code should be 0 + + Then these Helm releases should be deployed using context "k3d-ncp-local": + | name | namespace | + | nats | nats-system | + | cert-manager | cert-manager | + | openbao-server | vault-system | + | nvcf-pki | cert-manager | + | cassandra | cassandra-system | + | api-keys | api-keys | + | sis | sis | + | api | nvcf | + | nvct-api | nvcf | + | invocation-service | nvcf | + | grpc-proxy | nvcf | + | ess-api | ess | + | notary-service | nvcf | + | admin-issuer-proxy | api-keys | + | reval | nvcf | + | nats-auth-callout-service | nats-system | + | ingress | envoy-gateway-system | + | llm-request-router | nvcf | + | llm-api-gateway | nvcf | + + # The issuer and the stargate leaf are functional gates for the + # secure tunnel: the router cannot serve TLS before cert-manager + # writes the stargate-quic-tls Secret. + When I run command "kubectl --context k3d-ncp-local wait clusterissuer nvcf-openbao-pki --for=condition=Ready --timeout=5m" + Then the command exit code should be 0 + + When I run command "kubectl --context k3d-ncp-local wait certificate stargate-quic-tls -n nvcf --for=condition=Ready --timeout=5m" + Then the command exit code should be 0 + + # Registration consumes the generated profile. Export it only after the + # PKI is ready so the profile carries both management and transport trust. + When I run command: + """ + ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml self-hosted --control-plane-stack deploy/stacks/self-managed --env local-bdd-pki control-plane profile export --cluster-name ncp-local + """ + Then the command exit code should be 0 + And file "deploy/stacks/self-managed/out/control-plane-profile.yaml" should exist + And yaml file "deploy/stacks/self-managed/out/control-plane-profile.yaml" should contain: + """ + managementTls: + trustMode: bundle + transportTls: + trustMode: bundle + """ + And yaml file "deploy/stacks/self-managed/out/control-plane-profile.yaml" should have non-empty keys: + | key | + | managementTls.caBundlePem | + | transportTls.trustBundleFingerprint | + | transportTls.trustBundlePem | + + # The profile carries endpoints and trust, not credentials. Initialize + # the harness-isolated, config-scoped CLI state before compute-plane + # registration. Discard stdout because init prints the minted token; + # stderr and the exit status remain available for diagnostics. + And command has succeeded: + """ + /bin/sh -c '${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml init >/dev/null' + """ + + Rule: The compute plane installs with bundle trust distributed from OpenBao + + Background: + Given these environment variables are set: + | name | + | NVCF_CLI | + | REPO_ROOT | + # This rule depends on the earlier control-plane install scenario + # in the same feature run. The @llm-pki-nvca scenario is not a + # standalone tag target. + + @llm-pki-nvca + Scenario: Operator registers the cluster and installs NVCA with bundle trust + When I run command: + """ + make -C deploy/stacks/nvcf-compute-plane register-cluster CLUSTER_NAME=ncp-local CONTROL_PLANE_PROFILE=${REPO_ROOT}/deploy/stacks/self-managed/out/control-plane-profile.yaml COMPUTE_KUBE_CONTEXT=k3d-ncp-local NVCF_CLI=${NVCF_CLI} NVCF_CLI_CONFIG=${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml + """ + Then the command exit code should be 0 + And file "deploy/stacks/nvcf-compute-plane/registration/ncp-local-register-values.yaml" should exist + + When I run command: + """ + make -C deploy/stacks/nvcf-compute-plane install CLUSTER_NAME=ncp-local HELMFILE_ENV=local-bdd-pki COMPUTE_KUBE_CONTEXT=k3d-ncp-local NVCF_CLI=${NVCF_CLI} + """ + Then the command exit code should be 0 + + Then these Helm releases should be deployed using context "k3d-ncp-local": + | name | namespace | + | nvca-operator | nvca-operator | + + When I run command "helm get values nvca-operator --namespace nvca-operator --kube-context k3d-ncp-local -o yaml" + Then the command exit code should be 0 + And the command output should contain "stargateQUICInsecure: false" + And the command output should contain "trustMode: bundle" + And the command output should contain "trustBundleFingerprint: sha256:" + + When I run command "kubectl --context k3d-ncp-local rollout status deployment/nvca-operator -n nvca-operator --timeout=10m" + Then the command exit code should be 0 + + When I run command "kubectl --context k3d-ncp-local wait nvcfbackend ncp-local -n nvca-operator --for=jsonpath={.status.agentStatus}=healthy --timeout=10m" + Then the command exit code should be 0 + + Rule: An LLM function answers invocations over the secured tunnel + + Background: + Given these environment variables are set: + | name | + | NVCF_CLI | + | REPO_ROOT | + | SAMPLE_NGC_ORG | + | SAMPLE_NGC_TEAM | + + # Depends on the earlier scenarios in this feature run; not a + # standalone tag target. Same body as the non-PKI LLM scenario: + # the invoke succeeding over the secure tunnel is the trust-chain + # proof. + @llm-function-type + Scenario: Operator creates, deploys, and invokes an LLM-type function over the secured tunnel + When I run command: + """ + ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml function create --name bdd-pki-openai-compatible-sample --image nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/nvcf-openai-compatible-sample:local --function-type LLM --inference-url /v1/chat/completions --inference-port 8000 --health-uri /health --health-port 8000 --health-timeout PT30S --llm-model 'name=openai-compatible-sample,uris=/v1/chat/completions|/v1/embeddings,routingMethod=round_robin' + """ + Then the command exit code should be 0 + + When I run command: + """ + ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml function deploy create --gpu H100 --instance-type NCP.GPU.H100_1x --backend ncp-local --regions us-west-1 --min-instances 1 --max-instances 1 --timeout 900 + """ + Then the command exit code should be 0 + + When I run command: + """ + ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml api-key generate --description bdd-pki-openai-compatible-sample --for function --scopes invoke_function,list_functions,queue_details,list_functions_details + """ + Then the command exit code should be 0 + + When I run command: + """ + ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke --inference-url /v1/chat/completions --model-name openai-compatible-sample --request-body '{"messages":[{"role":"user","content":"bdd-pki-llm"}]}' --timeout 120 + """ + Then the command exit code should be 0 + And the command output should contain "chat.completion" + And the command output should contain "fixed 128-byte response" + + # curl reports only the status code so the assertion cannot + # match response-body noise. + When I run command: + """ + curl -s --connect-timeout 5 --max-time 30 -o /dev/null -w "%{http_code}" -X POST http://llm.localhost:8080/v1/chat/completions -H "Content-Type: application/json" -H "traceparent: 00-00000000000000000000000000001076-0000000000001076-01" -d '{"model":"unauthenticated/check","messages":[]}' + """ + Then the command exit code should be 0 + And the command output should contain "401" + + # Leave the GPU capacity free, same as the non-PKI feature. + When I run command: + """ + ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml function delete --deployment-only + """ + Then the command exit code should be 0 diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 64ed32d51..5b889615a 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -517,6 +517,111 @@ func TestSingleClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { } } +// TestSingleClusterHelmfileLLMPKIFeatureFileWiresToSteps runs the +// LLM PKI Helmfile feature against a fake runner, with canned results +// for the LLM invoke and the no-auth curl. +func TestSingleClusterHelmfileLLMPKIFeatureFileWiresToSteps(t *testing.T) { + t.Setenv("NGC_API_KEY", "test-key") + t.Setenv("SAMPLE_NGC_ORG", "test-org") + t.Setenv("SAMPLE_NGC_TEAM", "test-team") + t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") + t.Setenv("REPO_ROOT", "/repo-root-placeholder") + suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{ + "helm list --all-namespaces --kube-context k3d-ncp-local -o json": {ExitCode: 0, Stdout: helmListAllNamespacesJSON()}, + "helm get values nvca-operator --namespace nvca-operator --kube-context k3d-ncp-local -o yaml": { + ExitCode: 0, + Stdout: "agentConfig:\n mergeConfig: |\n workload:\n stargateQUICInsecure: false\n transportTLS:\n trustMode: bundle\n trustBundleFingerprint: sha256:test\n", + }, + "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" + + " --inference-url /v1/chat/completions --model-name openai-compatible-sample" + + " --request-body '{\"messages\":[{\"role\":\"user\",\"content\":\"bdd-pki-llm\"}]}' --timeout 120": { + ExitCode: 0, + Stdout: "Function invocation completed!\n\nResponse:\n" + + `{"object":"chat.completion","choices":[{"message":{"content":"This is a fixed 128-byte response from an NVCF-hosted OpenAI-compatible sample, used for routing and response-contract validation, not token-generation capacity."}}]}` + + "\n", + }, + `curl -s --connect-timeout 5 --max-time 30 -o /dev/null -w "%{http_code}" -X POST ` + + `http://llm.localhost:8080/v1/chat/completions -H "Content-Type: application/json" ` + + `-H "traceparent: 00-00000000000000000000000000001076-0000000000001076-01" ` + + `-d '{"model":"unauthenticated/check","messages":[]}'`: { + ExitCode: 0, + Stdout: "401", + }, + // Conflict precheck: feature asserts the conflicting + // multi-cluster control-plane is absent. + "k3d cluster get ncp-local-cp": {ExitCode: 1}, + })) + seedHelmfileLocalBDDFixture(t, suite.Config.RepoRoot) + seedComputePlaneLocalBDDFixture(t, suite.Config.RepoRoot) + seedStackSecretsTemplate(t, suite.Config.RepoRoot) + writeProfileHandoffArtifact(t, suite.Config.RepoRoot) + writeHelmfileRegisterValues(t, suite.Config.RepoRoot) + seedPKIRenderOutput(t, suite.Config.RepoRoot) + + sc := steps.NewScenarioContext(suite) + featurePath := mustResolveFeaturePath(t, "single-cluster-helmfile-llm-pki.feature") + var out strings.Builder + status := godog.TestSuite{ + Name: "single-cluster-helmfile-llm-pki-wiring", + ScenarioInitializer: func(ctx *godog.ScenarioContext) { + steps.RegisterAll(ctx, sc) + }, + Options: &godog.Options{ + Format: "pretty", + Paths: []string{featurePath}, + Strict: true, + Output: &out, + }, + }.Run() + if status != 0 { + t.Fatalf("godog suite status = %d\n%s", status, out.String()) + } + runs := suite.Runner.(*fakeRunner).runs + if !commandRanThatContains(runs, "install HELMFILE_ENV=local-bdd-pki") { + t.Fatal("PKI helmfile install make target was never invoked") + } + profileExport := "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml" + + " self-hosted --control-plane-stack deploy/stacks/self-managed --env local-bdd-pki" + + " control-plane profile export --cluster-name ncp-local" + initCommand := "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml init >/dev/null" + profileExportIndex := -1 + initIndex := -1 + registerIndex := -1 + for index, command := range runs { + if strings.Contains(command, profileExport) { + profileExportIndex = index + } + if strings.Contains(command, initCommand) { + initIndex = index + } + if strings.Contains(command, "register-cluster CLUSTER_NAME=ncp-local") { + registerIndex = index + if !strings.Contains(command, "CONTROL_PLANE_PROFILE=/repo-root-placeholder/deploy/stacks/self-managed/out/control-plane-profile.yaml") { + t.Fatalf("compute-plane registration did not use the exported profile: %s", command) + } + if !strings.Contains(command, "COMPUTE_KUBE_CONTEXT=k3d-ncp-local") { + t.Fatalf("compute-plane registration did not select the local cluster context: %s", command) + } + if !strings.Contains(command, "NVCF_CLI_CONFIG=/repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml") { + t.Fatalf("compute-plane registration did not select the initialized CLI config: %s", command) + } + } + } + if profileExportIndex < 0 { + t.Fatal("selected Helmfile environment was not exported to a control-plane profile") + } + if initIndex < 0 { + t.Fatal("local admin credentials were not initialized before compute-plane registration") + } + if registerIndex < 0 { + t.Fatal("compute-plane register-cluster make target was never invoked") + } + if profileExportIndex >= initIndex || initIndex >= registerIndex { + t.Fatal("profile export and credential initialization did not precede compute-plane registration") + } + assertFunctionDeploymentsUseInstanceType(t, runs, "NCP.GPU.H100_1x", 1) +} + // TestObservabilityControlFeatureFileWiresToSteps runs the live-install // observability-control feature against a fake runner. It checks the // single-cluster Helmfile path renders and verifies the profile-selected @@ -1196,6 +1301,7 @@ func helmListAllNamespacesJSON() string { {"name":"nats","namespace":"nats-system","status":"deployed"}, {"name":"cert-manager","namespace":"cert-manager","status":"deployed"}, {"name":"openbao-server","namespace":"vault-system","status":"deployed"}, +{"name":"nvcf-pki","namespace":"cert-manager","status":"deployed"}, {"name":"cassandra","namespace":"cassandra-system","status":"deployed"}, {"name":"api-keys","namespace":"api-keys","status":"deployed"}, {"name":"sis","namespace":"sis","status":"deployed"}, @@ -1303,6 +1409,32 @@ image: docker.io/natsio/nats-server-config-reloader:0.23.0 } } +// seedPKIRenderOutput writes the representative PKI resources asserted by the +// focused Helmfile feature wiring test. +func seedPKIRenderOutput(t *testing.T, repoRoot string) { + t.Helper() + manifest := `kind: ClusterIssuer +metadata: + name: "nvcf-openbao-pki" +spec: + dnsNames: + - llm-request-router.nvcf.svc.cluster.local +env: + - name: ADDONS_LLM_ENABLED + value: "true" + - name: NVCF_SERVICE_PKI_ALLOWED_DOMAINS + value: "nvcf.svc.cluster.local" +image: nvcr.io/test-org/test-team/nvcf-openbao-migrations:0.16.2 +` + filePath := filepath.Join(repoRoot, "deploy", "stacks", "self-managed", "out", "01-pki", "templates", "pki.yaml") + if err := os.MkdirAll(filepath.Dir(filePath), 0o755); err != nil { + t.Fatalf("mkdir rendered PKI manifest dir: %v", err) + } + if err := os.WriteFile(filePath, []byte(manifest), 0o644); err != nil { + t.Fatalf("write rendered PKI manifest: %v", err) + } +} + // seedHelmfileLocalBDDMultiFixture writes the multi-cluster variant // the multi-cluster helmfile feature copies onto the env file. Its // workerEndpoints and nvcaOperator.selfManaged URLs use the service @@ -1719,6 +1851,15 @@ func TestSingleClusterHelmfile(t *testing.T) { runLiveFeature(t, "single-cluster-helmfile.feature") } +// TestSingleClusterHelmfileLLMPKI is the live entry point for the +// PKI-secured LLM transport Helmfile feature. Skipped under -short. +func TestSingleClusterHelmfileLLMPKI(t *testing.T) { + if testing.Short() { + t.Skip("live run skipped under -short") + } + runLiveFeature(t, "single-cluster-helmfile-llm-pki.feature") +} + // TestObservabilityControl is the live entry point for the control // observability profile feature. Skipped under -short. func TestObservabilityControl(t *testing.T) { @@ -1815,7 +1956,9 @@ func runLiveFeatureTags(t *testing.T, feature, tags string) { if err != nil { t.Fatalf("new suite: %v", err) } + stopSignalCleanup := suite.InstallSignalCleanup() defer func() { + stopSignalCleanup() if err := suite.Teardown(); err != nil { t.Errorf("teardown: %v", err) } @@ -1826,6 +1969,18 @@ func runLiveFeatureTags(t *testing.T, feature, tags string) { Name: "bdd-live-" + feature, ScenarioInitializer: func(ctx *godog.ScenarioContext) { steps.RegisterAll(ctx, sc) + stepHooks := ctx.StepContext() + stepHooks.Before(func(stepContext context.Context, _ *godog.Step) (context.Context, error) { + return suite.BeginSignalSafeStep(stepContext) + }) + stepHooks.After(func( + stepContext context.Context, + _ *godog.Step, + _ godog.StepResultStatus, + _ error, + ) (context.Context, error) { + return suite.EndSignalSafeStep(stepContext), nil + }) }, Options: &godog.Options{ Format: "pretty", diff --git a/tests/bdd/harness/process_group_other.go b/tests/bdd/harness/process_group_other.go new file mode 100644 index 000000000..058c7f560 --- /dev/null +++ b/tests/bdd/harness/process_group_other.go @@ -0,0 +1,26 @@ +//go:build !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd + +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package harness + +import "os/exec" + +// exec.CommandContext's direct-child cancellation is the available fallback +// on platforms without Unix process groups. +func configureCommandCancellation(_ *exec.Cmd) {} diff --git a/tests/bdd/harness/process_group_unix.go b/tests/bdd/harness/process_group_unix.go new file mode 100644 index 000000000..e49163efe --- /dev/null +++ b/tests/bdd/harness/process_group_unix.go @@ -0,0 +1,44 @@ +//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd + +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package harness + +import ( + "errors" + "os" + "os/exec" + "syscall" +) + +// configureCommandCancellation gives each BDD command its own process group. +// Context cancellation kills the group so make, kubectl, and shell descendants +// cannot outlive the step and mutate ledger-backed inputs after signal cleanup. +func configureCommandCancellation(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { + if cmd.Process == nil { + return os.ErrProcessDone + } + err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + if errors.Is(err, syscall.ESRCH) { + return os.ErrProcessDone + } + return err + } +} diff --git a/tests/bdd/harness/runner.go b/tests/bdd/harness/runner.go index f891443f6..37c0df793 100644 --- a/tests/bdd/harness/runner.go +++ b/tests/bdd/harness/runner.go @@ -124,6 +124,7 @@ func (r *execRunner) run(ctx context.Context, commandText string, options runOpt return Result{}, errors.New("empty command") } cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) + configureCommandCancellation(cmd) cmd.Dir = r.cwd var stdout, stderr bytes.Buffer cmd.Stdout = &stdout diff --git a/tests/bdd/harness/runner_test.go b/tests/bdd/harness/runner_test.go index 841581790..99d590095 100644 --- a/tests/bdd/harness/runner_test.go +++ b/tests/bdd/harness/runner_test.go @@ -75,6 +75,30 @@ func TestCommandRunnerRunWithTTYReadHonorsContext(t *testing.T) { } } +func TestCommandRunnerContextCancellationStopsChildProcesses(t *testing.T) { + testDir := t.TempDir() + markerPath := filepath.Join(testDir, "late-child-write") + scriptPath := filepath.Join(testDir, "spawn-child.sh") + script := `#!/bin/sh +nohup sh -c 'sleep 0.4; touch "$1"' sh "$1" >/dev/null 2>&1 & +wait +` + if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil { + t.Fatalf("write child process fixture: %v", err) + } + + runner := NewCommandRunner(testDir, "") + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + if _, err := runner.Run(ctx, scriptPath+" "+markerPath); err == nil { + t.Fatal("child-spawning command unexpectedly completed") + } + time.Sleep(500 * time.Millisecond) + if _, err := os.Stat(markerPath); !os.IsNotExist(err) { + t.Fatalf("child process survived context cancellation and wrote %s: %v", markerPath, err) + } +} + func TestCommandRunnerNonZeroExit(t *testing.T) { runner := NewCommandRunner(t.TempDir(), "") result, err := runner.Run(context.Background(), "false") diff --git a/tests/bdd/harness/suite.go b/tests/bdd/harness/suite.go index 2df85f501..9094c685e 100644 --- a/tests/bdd/harness/suite.go +++ b/tests/bdd/harness/suite.go @@ -23,7 +23,10 @@ import ( "fmt" "os" "os/exec" + "os/signal" "path/filepath" + "sync" + "syscall" "testing" ) @@ -37,6 +40,12 @@ type Suite struct { Ledger *Ledger EnvLedger *EnvLedger Cache *CommandCache + + signalMu sync.Mutex + signalContext context.Context + signalCancel context.CancelFunc + stepMu sync.RWMutex + teardownMu sync.Mutex } // NewSuite resolves Config, creates the run-id directory tree, builds @@ -118,9 +127,99 @@ func (s *Suite) snapshotCLIStateFile(contextName string) error { // Teardown restores every file the Ledger tracked and every env var // the EnvLedger tracked. Live entry points should defer it. func (s *Suite) Teardown() error { + s.teardownMu.Lock() + defer s.teardownMu.Unlock() return errors.Join(s.Ledger.RestoreAll(), s.EnvLedger.RestoreAll()) } +type signalSafeStepKey struct{} + +type signalSafeStep struct { + parent context.Context + once sync.Once + cleanup func() +} + +// BeginSignalSafeStep marks one live BDD step active and returns a context that +// is canceled when SIGINT or SIGTERM starts cleanup. EndSignalSafeStep must be +// called with the returned context. Signal cleanup waits for every active step +// to finish, so a file-writing step cannot recreate a ledger-backed credential +// after restoration. +func (s *Suite) BeginSignalSafeStep(ctx context.Context) (context.Context, error) { + s.signalMu.Lock() + signalContext := s.signalContext + s.signalMu.Unlock() + if signalContext == nil { + return ctx, errors.New("signal cleanup is not installed") + } + + s.stepMu.RLock() + stepContext, cancelStep := context.WithCancel(ctx) + stopSignalCancel := context.AfterFunc(signalContext, cancelStep) + step := &signalSafeStep{parent: ctx, cleanup: func() { + stopSignalCancel() + cancelStep() + s.stepMu.RUnlock() + }} + return context.WithValue(stepContext, signalSafeStepKey{}, step), nil +} + +// EndSignalSafeStep releases the live-step guard installed by +// BeginSignalSafeStep and returns the uncanceled parent context for the next +// Godog step. Repeated calls are harmless. +func (s *Suite) EndSignalSafeStep(ctx context.Context) context.Context { + step, ok := ctx.Value(signalSafeStepKey{}).(*signalSafeStep) + if !ok { + return ctx + } + step.once.Do(step.cleanup) + return step.parent +} + +// InstallSignalCleanup restores ledger-backed files and environment variables +// before an interrupted live run exits. It first cancels the active step, then +// waits for that step to quiesce before restoring. The returned stop function +// joins the signal goroutine and must run before normal Teardown. +func (s *Suite) InstallSignalCleanup() func() { + signals := make(chan os.Signal, 1) + stop := make(chan struct{}) + done := make(chan struct{}) + s.signalMu.Lock() + s.signalContext, s.signalCancel = context.WithCancel(context.Background()) + cancelSteps := s.signalCancel + s.signalMu.Unlock() + signal.Notify(signals, os.Interrupt, syscall.SIGTERM) + var stopOnce sync.Once + go func() { + defer close(done) + select { + case sig := <-signals: + cancelSteps() + s.stepMu.Lock() + if err := s.Teardown(); err != nil { + fmt.Fprintf(os.Stderr, "BDD interrupt cleanup failed: %v\n", err) + } + os.Exit(signalExitCode(sig)) + case <-stop: + } + }() + return func() { + stopOnce.Do(func() { + signal.Stop(signals) + close(stop) + <-done + cancelSteps() + }) + } +} + +func signalExitCode(sig os.Signal) int { + if value, ok := sig.(syscall.Signal); ok { + return 128 + int(value) + } + return 1 +} + // buildCLI invokes `go build` directly via exec.Command rather than // routing through the CommandRunner so paths with spaces in the repo // root cannot be silently mis-tokenized. The build runs inside the diff --git a/tests/bdd/harness/suite_test.go b/tests/bdd/harness/suite_test.go new file mode 100644 index 000000000..1c2f50e44 --- /dev/null +++ b/tests/bdd/harness/suite_test.go @@ -0,0 +1,170 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package harness + +import ( + "bytes" + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "syscall" + "testing" + "time" +) + +func TestSignalSafeStepsPreserveNormalContinuation(t *testing.T) { + suite := &Suite{} + stopSignalCleanup := suite.InstallSignalCleanup() + defer stopSignalCleanup() + + type contextKey string + parent := context.WithValue(context.Background(), contextKey("scenario"), "retained") + first, err := suite.BeginSignalSafeStep(parent) + if err != nil { + t.Fatalf("begin first signal-safe step: %v", err) + } + + continuation := suite.EndSignalSafeStep(first) + if err := continuation.Err(); err != nil { + t.Fatalf("normal continuation was canceled: %v", err) + } + if got := continuation.Value(contextKey("scenario")); got != "retained" { + t.Fatalf("normal continuation lost scenario value: got %v", got) + } + if repeated := suite.EndSignalSafeStep(first); repeated != continuation { + t.Fatal("repeated end did not return the same continuation context") + } + + second, err := suite.BeginSignalSafeStep(continuation) + if err != nil { + t.Fatalf("begin second signal-safe step: %v", err) + } + defer suite.EndSignalSafeStep(second) + if err := second.Err(); err != nil { + t.Fatalf("second step inherited canceled context: %v", err) + } +} + +func TestSuiteRestoresGeneratedSecretOnSignals(t *testing.T) { + for _, testCase := range []struct { + name string + signal os.Signal + exitCode int + }{ + {name: "SIGINT", signal: os.Interrupt, exitCode: 130}, + {name: "SIGTERM", signal: syscall.SIGTERM, exitCode: 143}, + } { + t.Run(testCase.name, func(t *testing.T) { + testDir := t.TempDir() + secretPath := filepath.Join(testDir, "local-bdd-pki-secrets.yaml") + readyPath := filepath.Join(testDir, "ready") + var output bytes.Buffer + cmd := exec.Command(os.Args[0], "-test.run=^TestSuiteSignalCleanupHelper$") + cmd.Env = append(os.Environ(), + "NVCF_BDD_SIGNAL_HELPER=1", + "NVCF_BDD_SIGNAL_SECRET="+secretPath, + "NVCF_BDD_SIGNAL_READY="+readyPath, + ) + cmd.Stdout = &output + cmd.Stderr = &output + if err := cmd.Start(); err != nil { + t.Fatalf("start signal cleanup helper: %v", err) + } + + deadline := time.Now().Add(5 * time.Second) + for { + if _, err := os.Stat(readyPath); err == nil { + break + } else if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("stat helper readiness file: %v", err) + } + if time.Now().After(deadline) { + _ = cmd.Process.Kill() + _ = cmd.Wait() + t.Fatalf("signal cleanup helper did not become ready:\n%s", output.String()) + } + time.Sleep(10 * time.Millisecond) + } + + if err := cmd.Process.Signal(testCase.signal); err != nil { + t.Fatalf("send %s: %v", testCase.name, err) + } + waitDone := make(chan error, 1) + go func() { + waitDone <- cmd.Wait() + }() + var err error + select { + case err = <-waitDone: + case <-time.After(5 * time.Second): + _ = cmd.Process.Kill() + <-waitDone + t.Fatalf("signal cleanup helper did not exit after %s:\n%s", testCase.name, output.String()) + } + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + t.Fatalf("signal cleanup helper error = %v, want exit %d\n%s", err, testCase.exitCode, output.String()) + } + if got := exitErr.ExitCode(); got != testCase.exitCode { + t.Fatalf("signal cleanup helper exit code = %d, want %d\n%s", got, testCase.exitCode, output.String()) + } + if _, err := os.Stat(secretPath); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("generated registry credential remained after %s cleanup: %v", testCase.name, err) + } + }) + } +} + +func TestSuiteSignalCleanupHelper(t *testing.T) { + if os.Getenv("NVCF_BDD_SIGNAL_HELPER") != "1" { + return + } + secretPath := os.Getenv("NVCF_BDD_SIGNAL_SECRET") + readyPath := os.Getenv("NVCF_BDD_SIGNAL_READY") + suite := &Suite{ + Ledger: NewLedger(filepath.Join(filepath.Dir(secretPath), "originals")), + EnvLedger: NewEnvLedger(), + } + if err := suite.Ledger.Snapshot(secretPath); err != nil { + t.Fatalf("snapshot generated secret: %v", err) + } + stopSignalCleanup := suite.InstallSignalCleanup() + defer stopSignalCleanup() + + stepContext, err := suite.BeginSignalSafeStep(context.Background()) + if err != nil { + t.Fatalf("begin signal-safe step: %v", err) + } + if err := os.WriteFile(secretPath, []byte("registryCredential: generated\n"), 0o600); err != nil { + t.Fatalf("write generated secret: %v", err) + } + if err := os.WriteFile(readyPath, []byte("ready\n"), 0o600); err != nil { + t.Fatalf("write helper readiness file: %v", err) + } + + <-stepContext.Done() + // Simulate a file-writing step completing after cancellation. Signal + // cleanup must wait for this active step, then restore the absent snapshot. + if err := os.WriteFile(secretPath, []byte("registryCredential: recreated-after-cancel\n"), 0o600); err != nil { + t.Fatalf("rewrite generated secret after cancellation: %v", err) + } + suite.EndSignalSafeStep(stepContext) + select {} +} diff --git a/tests/bdd/steps/assertion_steps.go b/tests/bdd/steps/assertion_steps.go index cb68d5967..dcd5bac6a 100644 --- a/tests/bdd/steps/assertion_steps.go +++ b/tests/bdd/steps/assertion_steps.go @@ -45,6 +45,7 @@ func registerAssertionSteps(ctx *godog.ScenarioContext, sc *ScenarioContext) { ctx.Step(`^the json output should contain rows:$`, sc.jsonOutputShouldContainRows) ctx.Step(`^Helm release "([^"]*)" in namespace "([^"]*)" using context "([^"]*)" should contain values:$`, sc.helmReleaseShouldContainValues) ctx.Step(`^the rendered manifests in "([^"]*)" should contain:$`, sc.renderedManifestsShouldContain) + ctx.Step(`^the rendered manifests in "([^"]*)" should contain Kubernetes resource "([^"/]+)/([^"]+)"$`, sc.renderedManifestsShouldContainKubernetesResource) ctx.Step(`^the rendered manifests in "([^"]*)" under directories matching "([^"]*)" should contain:$`, sc.renderedManifestsUnderMatchingDirectoriesShouldContain) ctx.Step(`^the rendered manifests in "([^"]*)" should not contain:$`, sc.renderedManifestsShouldNotContain) ctx.Step(`^these Helm releases should be deployed using context "([^"]*)":$`, sc.helmReleasesShouldBeDeployed) @@ -183,6 +184,13 @@ func (sc *ScenarioContext) renderedManifestsShouldContain(path string, table *go return dsl.FilesContain(sc.resolvePath(dsl.Interpolate(path)), "", needles) } +func (sc *ScenarioContext) renderedManifestsShouldContainKubernetesResource(path, kind, name string) error { + return dsl.RenderedManifestsContainResource( + sc.resolvePath(dsl.Interpolate(path)), + dsl.KubernetesResource{Kind: kind, Name: name}, + ) +} + func (sc *ScenarioContext) renderedManifestsUnderMatchingDirectoriesShouldContain(path, pattern string, table *godog.Table) error { needles, err := tableToSingleColumn(table, "text") if err != nil {