diff --git a/tests/bdd/AGENTS.md b/tests/bdd/AGENTS.md index 274ccbd41..4e0d08a23 100644 --- a/tests/bdd/AGENTS.md +++ b/tests/bdd/AGENTS.md @@ -26,9 +26,9 @@ assertion as the escape hatch for uncommon or command-specific behavior. `CommandCache`, `Suite`. Step handlers depend on these; nothing else does. - `dsl/` owns pure helpers: `${VAR}` interpolation, dotted-path YAML - upsert and read, YAML subtree match/contain, kubectl manifest - builders, JSON row matching. Every helper is unit-testable in - isolation. No I/O coordination, no Godog dependency. + upsert and read, YAML subtree match/contain, self-managed secrets + rendering, kubectl manifest builders, JSON row matching. Every helper + is unit-testable in isolation. No I/O coordination, no Godog dependency. - `steps/` owns Godog step handlers and `ScenarioContext`. Each handler is one or two lines plus a delegate to a `dsl` helper or `Suite.Runner`. @@ -45,9 +45,9 @@ logic into `dsl/`. a bare `$word` is left literal. Implementations must not use `os.ExpandEnv`. Expansion lives in `dsl.Interpolate`. - File-mutating steps (`I copy the file`, `I update yaml file`, - `I substitute`) snapshot the destination through `Suite.Ledger` - before the first write. Suite teardown restores every snapshotted - path. + `I prepare self-managed secrets file`, `I substitute a block`) + snapshot the destination through `Suite.Ledger` before the first write. + Suite teardown restores every snapshotted path. - `Given command has succeeded:` keys on the fully resolved command text. Two scenarios whose pre-interpolation text matches but whose env vars differ must miss the cache. The cache lives in diff --git a/tests/bdd/PLAN.md b/tests/bdd/PLAN.md index 19c309e52..95e930777 100644 --- a/tests/bdd/PLAN.md +++ b/tests/bdd/PLAN.md @@ -10,7 +10,7 @@ domain-specific validation logic. The vocabulary is restricted to four categories: -1. File operations: copy, edit YAML, substitute strings. +1. File operations: copy, edit YAML, substitute blocks, prepare secrets. 2. Environment preconditions: env vars set, files exist, infrastructure reachable. 3. Command execution: exec a shell command and capture exit code, stdout, @@ -105,7 +105,7 @@ refactor in every consumer; that is a feature. |------|-------| | `And I copy the file {string} to {string}` | Both paths are repo-relative. | | `And I update yaml file {string} with keys:` (two-column table of dotted-path and value) | Path supports dotted notation and `[n]` indices (e.g. `global.imagePullSecrets[0].name`). Missing intermediate maps and missing list indices are upserted: writing `global.imagePullSecrets[0].name` against a file that has neither `global.imagePullSecrets` nor any list entry creates both. Existing scalars at intermediate positions cause the step to fail rather than silently overwrite a non-map. Value cells expand `${VAR}` from `os.Environ`. | -| `And I substitute {string} in file {string} with base64 of {string}` | Used for credential rendering; the third arg expands `${VAR}` then base64-encodes. The handler never logs the substituted value. | +| `And I prepare self-managed secrets file {string} from template {string} using the current NGC registry credential` | The destination and template are explicit repo-relative paths with `${VAR}` interpolation. Replaces the template's registry credential placeholder with base64 of the current `$oauthtoken:` credential and writes the destination with mode `0600`. The destination is ledger-backed, and secret material never enters Gherkin, command logs, or failure messages. | | `And I substitute a block in file {string}:` (docstring) | The docstring contains an old block and replacement block separated by exactly one `---` line. `${VAR}` interpolation applies before an exact, ledger-backed replacement. Missing or malformed old blocks fail. | ### Command execution (When) @@ -213,7 +213,8 @@ contract verified in `src/clis/nvcf-cli/cmd/`): Every step that writes into a path under the repo working tree (`I copy the file ... to ...`, `I update yaml file ...`, -`I substitute ... in file ...`) registers that path with the runner's +`I prepare self-managed secrets file ...`, `I substitute a block ...`) +registers that path with the runner's restoration ledger: - Before the first write, the runner snapshots the file (exists/not, @@ -448,9 +449,14 @@ const ( // names the first path that differed. func MatchYAMLSubtree(filePath, keyPath, expectedYAML string, mode MatchMode) error +// RenderSelfManagedSecrets replaces the secrets-template registry credential +// placeholder with base64 of `$oauthtoken:`. It fails without +// returning raw or encoded credential material when the key or placeholder is +// missing. +func RenderSelfManagedSecrets(template []byte, apiKey string) ([]byte, error) + // SubstituteFile replaces every occurrence of placeholder with -// replacement in the named file. Used for credential rendering. -// Never logs placeholder or replacement. +// replacement in the named file. Never logs placeholder or replacement. func SubstituteFile(path, placeholder, replacement string) error // JSONContainsRows parses raw as a JSON array of objects, and for each diff --git a/tests/bdd/dsl/secrets.go b/tests/bdd/dsl/secrets.go new file mode 100644 index 000000000..d297d1c13 --- /dev/null +++ b/tests/bdd/dsl/secrets.go @@ -0,0 +1,41 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dsl + +import ( + "bytes" + "encoding/base64" + "fmt" +) + +const selfManagedRegistryCredentialPlaceholder = "REPLACE_WITH_BASE64_DOCKER_CREDENTIAL" + +// RenderSelfManagedSecrets replaces the registry credential placeholder with +// base64 of the Docker username and current NGC API key. Errors never include +// the raw or encoded credential. +func RenderSelfManagedSecrets(template []byte, apiKey string) ([]byte, error) { + if apiKey == "" { + return nil, fmt.Errorf("NGC_API_KEY is not set") + } + placeholder := []byte(selfManagedRegistryCredentialPlaceholder) + if !bytes.Contains(template, placeholder) { + return nil, fmt.Errorf("self-managed secrets template is missing the registry credential placeholder") + } + credential := base64.StdEncoding.EncodeToString([]byte("$oauthtoken:" + apiKey)) + return bytes.ReplaceAll(template, placeholder, []byte(credential)), nil +} diff --git a/tests/bdd/dsl/secrets_test.go b/tests/bdd/dsl/secrets_test.go new file mode 100644 index 000000000..fef67ac55 --- /dev/null +++ b/tests/bdd/dsl/secrets_test.go @@ -0,0 +1,63 @@ +/* +SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dsl + +import ( + "encoding/base64" + "strings" + "testing" +) + +func TestRenderSelfManagedSecretsUsesDockerCredentialFormat(t *testing.T) { + apiKey := "test-api-key" + template := []byte("first: " + selfManagedRegistryCredentialPlaceholder + "\nsecond: " + selfManagedRegistryCredentialPlaceholder + "\n") + + got, err := RenderSelfManagedSecrets(template, apiKey) + if err != nil { + t.Fatalf("render: %v", err) + } + wantCredential := base64.StdEncoding.EncodeToString([]byte("$oauthtoken:" + apiKey)) + if strings.Count(string(got), wantCredential) != 2 { + t.Fatalf("rendered credential count = %d, want 2", strings.Count(string(got), wantCredential)) + } + if strings.Contains(string(got), apiKey) { + t.Fatal("raw API key leaked into rendered secrets") + } +} + +func TestRenderSelfManagedSecretsRejectsMissingAPIKey(t *testing.T) { + _, err := RenderSelfManagedSecrets([]byte(selfManagedRegistryCredentialPlaceholder), "") + if err == nil || !strings.Contains(err.Error(), "NGC_API_KEY is not set") { + t.Fatalf("error = %v, want missing-key error", err) + } +} + +func TestRenderSelfManagedSecretsFailureHidesCredentialMaterial(t *testing.T) { + apiKey := "sensitive-test-api-key" + encoded := base64.StdEncoding.EncodeToString([]byte("$oauthtoken:" + apiKey)) + + _, err := RenderSelfManagedSecrets([]byte("registryCredential: missing\n"), apiKey) + if err == nil { + t.Fatal("expected missing-placeholder error") + } + for _, secret := range []string{apiKey, encoded} { + if strings.Contains(err.Error(), secret) { + t.Fatalf("error leaked credential material: %v", err) + } + } +} diff --git a/tests/bdd/features/multi-cluster-eks-helmfile.feature b/tests/bdd/features/multi-cluster-eks-helmfile.feature index 0220d23bf..3d3de57e7 100644 --- a/tests/bdd/features/multi-cluster-eks-helmfile.feature +++ b/tests/bdd/features/multi-cluster-eks-helmfile.feature @@ -73,8 +73,7 @@ Feature: Install a multi-cluster NVCF stack across two pre-provisioned EKS clust bash -c 'set -eo pipefail; printf %s "$NGC_API_KEY" | helm registry login nvcr.io --username "\$oauthtoken" --password-stdin' """ # Create NGC dockerconfig registry credentials. - And I copy the file "deploy/stacks/self-managed/secrets/secrets.yaml.template" to "deploy/stacks/self-managed/secrets/eks-bdd-multi-secrets.yaml" - And I substitute "REPLACE_WITH_BASE64_DOCKER_CREDENTIAL" in file "deploy/stacks/self-managed/secrets/eks-bdd-multi-secrets.yaml" with base64 of "$oauthtoken:${NGC_API_KEY}" + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/eks-bdd-multi-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential # Both clusters must be reachable before we start. And I run command "kubectl --context ${EKS_CONTEXT} get nodes -o name" And the command exit code should be 0 @@ -412,6 +411,8 @@ Feature: Install a multi-cluster NVCF stack across two pre-provisioned EKS clust Then the command exit code should be 0 And the command output should contain "bdd-echo" + # Failing until GitHub issue #1098 is resolved and the fix from GitHub + # issue #1032 is consumed by the self-managed stack. @nvct-task-api Scenario: User launches an NVCT task on the compute cluster and waits for completion Given command has succeeded: diff --git a/tests/bdd/features/multi-cluster-helmfile.feature b/tests/bdd/features/multi-cluster-helmfile.feature index 23bf3f7c5..02cf36bd9 100644 --- a/tests/bdd/features/multi-cluster-helmfile.feature +++ b/tests/bdd/features/multi-cluster-helmfile.feature @@ -50,8 +50,7 @@ Feature: Install a local multi-cluster NVCF stack with Helmfile | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | | observability.profile | disabled | - And I copy the file "deploy/stacks/self-managed/secrets/secrets.yaml.template" to "deploy/stacks/self-managed/secrets/local-bdd-secrets.yaml" - And I substitute "REPLACE_WITH_BASE64_DOCKER_CREDENTIAL" in file "deploy/stacks/self-managed/secrets/local-bdd-secrets.yaml" with base64 of "$oauthtoken:${NGC_API_KEY}" + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential # Conflict precheck: single-cluster ncp-local's k3d serverlb # claims 0.0.0.0:8080/8443/10081, and ncp-local-cp also # needs NATS on 4222 plus the worker callback port 10086. @@ -235,6 +234,8 @@ Feature: Install a local multi-cluster NVCF stack with Helmfile # This scenario intentionally has no Background. It depends on the # earlier control-plane install and NVCA registration scenarios in # this feature run, and is not a standalone tag target. + # Failing until GitHub issue #1098 is resolved and the fix from GitHub + # issue #1032 is consumed by the self-managed stack. @nvct-task-api Scenario: Operator launches an NVCT task and waits for it to complete When I run command: diff --git a/tests/bdd/features/multi-cluster-up.feature b/tests/bdd/features/multi-cluster-up.feature index 021186302..7eb9cf761 100644 --- a/tests/bdd/features/multi-cluster-up.feature +++ b/tests/bdd/features/multi-cluster-up.feature @@ -20,17 +20,14 @@ Feature: Bring up a local multi-cluster NVCF stack with the CLI | NGC_API_KEY | | SAMPLE_NGC_ORG | | SAMPLE_NGC_TEAM | - # self-hosted install --env local reads operator-authored local - # secrets files from both split stacks: - # deploy/stacks/self-managed/secrets/local-secrets.yaml (control - # plane). Only secrets.yaml.template is tracked in each - # stack. Author both files from the canonical templates before - # running install/register. Ledger snapshots whatever + # self-hosted install --env local reads the operator-authored + # control-plane secrets file. Only secrets.yaml.template is tracked. + # Prepare local-secrets.yaml from that template before running + # install/register. Ledger snapshots whatever # local-secrets.yaml state existed before the first write (its # prior contents or absence) and restores or removes it at suite # teardown. - And I copy the file "deploy/stacks/self-managed/secrets/secrets.yaml.template" to "deploy/stacks/self-managed/secrets/local-secrets.yaml" - And I substitute "REPLACE_WITH_BASE64_DOCKER_CREDENTIAL" in file "deploy/stacks/self-managed/secrets/local-secrets.yaml" with base64 of "$oauthtoken:${NGC_API_KEY}" + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential # --env local also reads operator-authored environment values from # both split stacks: deploy/stacks//environments/local.yaml. # Neither file is tracked, so author both from the BDD multi-cluster diff --git a/tests/bdd/features/observability-all.feature b/tests/bdd/features/observability-all.feature index d5ad373de..de987bcdc 100644 --- a/tests/bdd/features/observability-all.feature +++ b/tests/bdd/features/observability-all.feature @@ -42,8 +42,7 @@ Feature: Install local Helmfile observability for both planes | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | | global.nvcaOperator.selfManaged.otelCollector.imageRepository | nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/nvcf-otel-collector | | observability.profile | all | - And I copy the file "deploy/stacks/self-managed/secrets/secrets.yaml.template" to "deploy/stacks/self-managed/secrets/local-bdd-observability-all-secrets.yaml" - And I substitute "REPLACE_WITH_BASE64_DOCKER_CREDENTIAL" in file "deploy/stacks/self-managed/secrets/local-bdd-observability-all-secrets.yaml" with base64 of "$oauthtoken:${NGC_API_KEY}" + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-observability-all-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential # Conflict precheck: the split topology claims host ports used by the # single-cluster topology. From the repository root, run # `make -C tools/ncp-local-cluster destroy-all-ncp-local SHELL=/bin/bash` diff --git a/tests/bdd/features/observability-compute.feature b/tests/bdd/features/observability-compute.feature index 65b904226..8c8a83f38 100644 --- a/tests/bdd/features/observability-compute.feature +++ b/tests/bdd/features/observability-compute.feature @@ -44,8 +44,7 @@ Feature: Install local Helmfile observability with the compute profile | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | | global.nvcaOperator.selfManaged.otelCollector.imageRepository | nvcr.io/${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM}/nvcf-otel-collector | | observability.profile | compute | - And I copy the file "deploy/stacks/self-managed/secrets/secrets.yaml.template" to "deploy/stacks/self-managed/secrets/local-bdd-observability-compute-secrets.yaml" - And I substitute "REPLACE_WITH_BASE64_DOCKER_CREDENTIAL" in file "deploy/stacks/self-managed/secrets/local-bdd-observability-compute-secrets.yaml" with base64 of "$oauthtoken:${NGC_API_KEY}" + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-observability-compute-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential # Conflict precheck: single-cluster ncp-local claims host ports used by the # split topology. From the repository root, run # `make -C tools/ncp-local-cluster destroy CLUSTER_NAME=ncp-local` diff --git a/tests/bdd/features/observability-control.feature b/tests/bdd/features/observability-control.feature index 01a1c9a39..d85834e9a 100644 --- a/tests/bdd/features/observability-control.feature +++ b/tests/bdd/features/observability-control.feature @@ -31,8 +31,7 @@ Feature: Install local Helmfile observability with the control profile | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | | observability.profile | control | - And I copy the file "deploy/stacks/self-managed/secrets/secrets.yaml.template" to "deploy/stacks/self-managed/secrets/local-bdd-observability-control-secrets.yaml" - And I substitute "REPLACE_WITH_BASE64_DOCKER_CREDENTIAL" in file "deploy/stacks/self-managed/secrets/local-bdd-observability-control-secrets.yaml" with base64 of "$oauthtoken:${NGC_API_KEY}" + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-observability-control-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential # Conflict precheck: ncp-local-cp claims host ports that overlap with this # single-cluster topology. From tools/ncp-local-cluster, run # `make destroy CLUSTER_NAME=ncp-local-cp` before retrying. diff --git a/tests/bdd/features/observability-disabled.feature b/tests/bdd/features/observability-disabled.feature index aeddb1172..5dcb0bcf4 100644 --- a/tests/bdd/features/observability-disabled.feature +++ b/tests/bdd/features/observability-disabled.feature @@ -24,8 +24,7 @@ Feature: Render local Helmfile stacks with observability disabled | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | | observability.profile | disabled | - And I copy the file "deploy/stacks/self-managed/secrets/secrets.yaml.template" to "deploy/stacks/self-managed/secrets/local-bdd-observability-disabled-secrets.yaml" - And I substitute "REPLACE_WITH_BASE64_DOCKER_CREDENTIAL" in file "deploy/stacks/self-managed/secrets/local-bdd-observability-disabled-secrets.yaml" with base64 of "$oauthtoken:${NGC_API_KEY}" + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-observability-disabled-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential # Create the compute-plane stack environment used by the worker render. And I copy the file "tests/bdd/fixtures/nvcf-compute-plane-local-bdd.yaml" to "deploy/stacks/nvcf-compute-plane/environments/local-bdd-observability-disabled.yaml" And I update yaml file "deploy/stacks/nvcf-compute-plane/environments/local-bdd-observability-disabled.yaml" with keys: diff --git a/tests/bdd/features/single-cluster-eks-helmfile.feature b/tests/bdd/features/single-cluster-eks-helmfile.feature index cf4ebc508..2cccc24ff 100644 --- a/tests/bdd/features/single-cluster-eks-helmfile.feature +++ b/tests/bdd/features/single-cluster-eks-helmfile.feature @@ -71,8 +71,7 @@ Feature: Install a single-cluster NVCF stack on a pre-provisioned EKS cluster wi bash -c 'set -eo pipefail; printf %s "$NGC_API_KEY" | helm registry login nvcr.io --username "\$oauthtoken" --password-stdin' """ # Create NGC dockerconfig registry credentials - And I copy the file "deploy/stacks/self-managed/secrets/secrets.yaml.template" to "deploy/stacks/self-managed/secrets/eks-bdd-secrets.yaml" - And I substitute "REPLACE_WITH_BASE64_DOCKER_CREDENTIAL" in file "deploy/stacks/self-managed/secrets/eks-bdd-secrets.yaml" with base64 of "$oauthtoken:${NGC_API_KEY}" + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/eks-bdd-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential And I run command "kubectl --context ${EKS_CONTEXT} get nodes -o name" And the command exit code should be 0 diff --git a/tests/bdd/features/single-cluster-helmfile-upstream-images.feature b/tests/bdd/features/single-cluster-helmfile-upstream-images.feature index 849ece593..1baa80dda 100644 --- a/tests/bdd/features/single-cluster-helmfile-upstream-images.feature +++ b/tests/bdd/features/single-cluster-helmfile-upstream-images.feature @@ -26,9 +26,7 @@ Feature: Install a local single-cluster stack with upstream supporting images | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | | observability.profile | disabled | - And I copy the file "deploy/stacks/self-managed/secrets/secrets.yaml.template" to "deploy/stacks/self-managed/secrets/local-bdd-secrets.yaml" - # Only ${VAR} is interpolated; bare $oauthtoken stays literal. - And I substitute "REPLACE_WITH_BASE64_DOCKER_CREDENTIAL" in file "deploy/stacks/self-managed/secrets/local-bdd-secrets.yaml" with base64 of "$oauthtoken:${NGC_API_KEY}" + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential And I substitute a block in file "deploy/stacks/self-managed/global.yaml.gotmpl": """ reloader: diff --git a/tests/bdd/features/single-cluster-helmfile.feature b/tests/bdd/features/single-cluster-helmfile.feature index 0562d9103..2473ef166 100644 --- a/tests/bdd/features/single-cluster-helmfile.feature +++ b/tests/bdd/features/single-cluster-helmfile.feature @@ -31,9 +31,7 @@ Feature: Install a local single-cluster NVCF stack with Helmfile | global.helm.sources.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | | global.image.repository | ${SAMPLE_NGC_ORG}/${SAMPLE_NGC_TEAM} | | observability.profile | disabled | - And I copy the file "deploy/stacks/self-managed/secrets/secrets.yaml.template" to "deploy/stacks/self-managed/secrets/local-bdd-secrets.yaml" - # Only ${VAR} is interpolated; bare $oauthtoken stays literal. - And I substitute "REPLACE_WITH_BASE64_DOCKER_CREDENTIAL" in file "deploy/stacks/self-managed/secrets/local-bdd-secrets.yaml" with base64 of "$oauthtoken:${NGC_API_KEY}" + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential Scenario: Operator validates the authored Helmfile environment renders When I run command "make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd" diff --git a/tests/bdd/features/single-cluster-up-oneclick.feature b/tests/bdd/features/single-cluster-up-oneclick.feature index 469602ba0..8c4e60aaf 100644 --- a/tests/bdd/features/single-cluster-up-oneclick.feature +++ b/tests/bdd/features/single-cluster-up-oneclick.feature @@ -49,8 +49,7 @@ Feature: Bring up a local single-cluster NVCF stack with the self-hosted up one- # The control-plane stack also requires an operator-authored local # secrets file. Author it from the tracked template; the Ledger gives # it the same restore-or-remove behavior as the environment files. - And I copy the file "deploy/stacks/self-managed/secrets/secrets.yaml.template" to "deploy/stacks/self-managed/secrets/local-secrets.yaml" - And I substitute "REPLACE_WITH_BASE64_DOCKER_CREDENTIAL" in file "deploy/stacks/self-managed/secrets/local-secrets.yaml" with base64 of "$oauthtoken:${NGC_API_KEY}" + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential # Conflict precheck: ncp-local-cp's k3d serverlb claims # 0.0.0.0:8080/8443/10081, NATS on 4222, and the worker # callback port 10086, overlapping host ports single-cluster diff --git a/tests/bdd/features/single-cluster-up.feature b/tests/bdd/features/single-cluster-up.feature index 2eddb0254..f26aea571 100644 --- a/tests/bdd/features/single-cluster-up.feature +++ b/tests/bdd/features/single-cluster-up.feature @@ -20,17 +20,14 @@ Feature: Bring up a local single-cluster NVCF stack with the CLI | NGC_API_KEY | | SAMPLE_NGC_ORG | | SAMPLE_NGC_TEAM | - # self-hosted install --env local reads operator-authored local - # secrets files from both split stacks: - # deploy/stacks/self-managed/secrets/local-secrets.yaml (control - # plane). Only secrets.yaml.template is tracked in each - # stack. Author both files from the canonical templates before - # running install/register. Ledger snapshots whatever + # self-hosted install --env local reads the operator-authored + # control-plane secrets file. Only secrets.yaml.template is tracked. + # Prepare local-secrets.yaml from that template before running + # install/register. Ledger snapshots whatever # local-secrets.yaml state existed before the first write (its # prior contents or absence) and restores or removes it at suite # teardown, so the working tree stays clean. - And I copy the file "deploy/stacks/self-managed/secrets/secrets.yaml.template" to "deploy/stacks/self-managed/secrets/local-secrets.yaml" - And I substitute "REPLACE_WITH_BASE64_DOCKER_CREDENTIAL" in file "deploy/stacks/self-managed/secrets/local-secrets.yaml" with base64 of "$oauthtoken:${NGC_API_KEY}" + And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential # --env local also reads operator-authored environment values from # both split stacks: deploy/stacks//environments/local.yaml. # Neither file is tracked, so author both from the BDD fixtures. diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 0c408bed2..adc3bd9ed 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -224,8 +224,8 @@ selfManaged: // the suite's RepoRoot. The body is not a faithful copy of the real // stack templates (which have richer schemas with several placeholders); // it only carries the single REPLACE_WITH_BASE64_DOCKER_CREDENTIAL token -// the feature substitutes, which is sufficient to exercise the I copy -// and I substitute steps against a fake CommandRunner. +// the self-managed secrets step renders, which is sufficient to exercise +// the file preparation path against a fake CommandRunner. func seedStackSecretsTemplate(t *testing.T, repoRoot string) { t.Helper() templatePath := filepath.Join(repoRoot, "deploy", "stacks", "self-managed", "secrets", "secrets.yaml.template") diff --git a/tests/bdd/harness/ledger.go b/tests/bdd/harness/ledger.go index d1ae093e2..ba6decb71 100644 --- a/tests/bdd/harness/ledger.go +++ b/tests/bdd/harness/ledger.go @@ -118,6 +118,8 @@ func (l *Ledger) restoreOne(path string, entry ledgerEntry) error { if err := os.WriteFile(path, entry.body, entry.mode); err != nil { return fmt.Errorf("restore %s: %w", path, err) } + if err := os.Chmod(path, entry.mode); err != nil { + return fmt.Errorf("restore mode %s: %w", path, err) + } return nil } - diff --git a/tests/bdd/harness/ledger_test.go b/tests/bdd/harness/ledger_test.go index 4b8913f90..3e0ff0c33 100644 --- a/tests/bdd/harness/ledger_test.go +++ b/tests/bdd/harness/ledger_test.go @@ -108,6 +108,9 @@ func TestLedgerRestorePreservesMode(t *testing.T) { if err := os.WriteFile(path, []byte("y\n"), 0o644); err != nil { t.Fatalf("write: %v", err) } + if err := os.Chmod(path, 0o644); err != nil { + t.Fatalf("chmod: %v", err) + } if err := ledger.RestoreAll(); err != nil { t.Fatalf("restore: %v", err) } diff --git a/tests/bdd/steps/file_steps.go b/tests/bdd/steps/file_steps.go index a05d17b5c..45361f901 100644 --- a/tests/bdd/steps/file_steps.go +++ b/tests/bdd/steps/file_steps.go @@ -18,7 +18,6 @@ limitations under the License. package steps import ( - "encoding/base64" "fmt" "io" "os" @@ -35,7 +34,7 @@ import ( func registerFileSteps(ctx *godog.ScenarioContext, sc *ScenarioContext) { ctx.Step(`^I copy the file "([^"]*)" to "([^"]*)"$`, sc.iCopyFile) ctx.Step(`^I update yaml file "([^"]*)" with keys:$`, sc.iUpdateYAMLFile) - ctx.Step(`^I substitute "([^"]*)" in file "([^"]*)" with base64 of "([^"]*)"$`, sc.iSubstituteBase64) + ctx.Step(`^I prepare self-managed secrets file "([^"]*)" from template "([^"]*)" using the current NGC registry credential$`, sc.iPrepareSelfManagedSecretsFile) ctx.Step(`^I substitute a block in file "([^"]*)":$`, sc.iSubstituteBlock) ctx.Step(`^environment variable "([^"]*)" is set$`, sc.environmentVariableIsSet) ctx.Step(`^these environment variables are set:$`, sc.environmentVariablesAreSet) @@ -67,22 +66,47 @@ func (sc *ScenarioContext) iUpdateYAMLFile(path string, table *godog.Table) erro return dsl.UpdateYAMLKeys(resolved, keys) } -// iSubstituteBase64 expands ${VAR} inside source, base64-encodes the -// result, and replaces every occurrence of placeholder in path. The -// handler never returns the substituted value to its caller so the -// secret material does not leak into logs. Go's base64.StdEncoding -// emits the encoded string on a single line with no line wrapping -- -// equivalent to `base64 -w0` -- so the resulting value can be sed- -// substituted into the secrets template without the sed-substitution -// breaking on embedded newlines. -func (sc *ScenarioContext) iSubstituteBase64(placeholder, path, source string) error { - resolvedPath := sc.resolvePath(dsl.Interpolate(path)) - if err := sc.Suite.Ledger.Snapshot(resolvedPath); err != nil { +// iPrepareSelfManagedSecretsFile delegates the secrets-file operation so the +// registered Godog handler remains declarative. +func (sc *ScenarioContext) iPrepareSelfManagedSecretsFile(dest, template string) error { + return sc.prepareSelfManagedSecretsFile(dest, template) +} + +// prepareSelfManagedSecretsFile resolves paths, snapshots the destination, +// renders the current credential, and replaces the destination with mode 0600. +func (sc *ScenarioContext) prepareSelfManagedSecretsFile(dest, template string) error { + resolvedDest := sc.resolvePath(dsl.Interpolate(dest)) + resolvedTemplate := sc.resolvePath(dsl.Interpolate(template)) + if err := sc.Suite.Ledger.Snapshot(resolvedDest); err != nil { + return err + } + body, err := os.ReadFile(resolvedTemplate) + if err != nil { + return fmt.Errorf("read secrets template %s: %w", resolvedTemplate, err) + } + rendered, err := dsl.RenderSelfManagedSecrets(body, os.Getenv("NGC_API_KEY")) + if err != nil { return err } - resolvedSource := dsl.Interpolate(source) - encoded := base64.StdEncoding.EncodeToString([]byte(resolvedSource)) - return dsl.SubstituteFile(resolvedPath, placeholder, encoded) + if err := os.MkdirAll(filepath.Dir(resolvedDest), 0o755); err != nil { + return fmt.Errorf("mkdir %s: %w", filepath.Dir(resolvedDest), err) + } + out, err := os.OpenFile(resolvedDest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) + if err != nil { + return fmt.Errorf("create self-managed secrets %s: %w", resolvedDest, err) + } + if err := out.Chmod(0o600); err != nil { + _ = out.Close() + return fmt.Errorf("secure self-managed secrets %s: %w", resolvedDest, err) + } + if _, err := out.Write(rendered); err != nil { + _ = out.Close() + return fmt.Errorf("write self-managed secrets %s: %w", resolvedDest, err) + } + if err := out.Close(); err != nil { + return fmt.Errorf("close self-managed secrets %s: %w", resolvedDest, err) + } + return nil } // iSubstituteBlock snapshots path before delegating the exact multi-line diff --git a/tests/bdd/steps/steps_test.go b/tests/bdd/steps/steps_test.go index e8101928a..09c69e7f4 100644 --- a/tests/bdd/steps/steps_test.go +++ b/tests/bdd/steps/steps_test.go @@ -19,6 +19,7 @@ package steps import ( "context" + "encoding/base64" "errors" "io" "os" @@ -93,6 +94,136 @@ func TestICopyFileSnapshotsAndCopies(t *testing.T) { } } +func TestIPrepareSelfManagedSecretsFileRendersInterpolatedPaths(t *testing.T) { + sc, fake := newScenarioContext(t) + t.Setenv("NGC_API_KEY", "test-api-key") + t.Setenv("BDD_TMP_SECRETS_NAME", "local-bdd-secrets.yaml") + t.Setenv("BDD_TMP_TEMPLATE_NAME", "secrets.yaml.template") + templateRel := "templates/${BDD_TMP_TEMPLATE_NAME}" + templateAbs := filepath.Join(sc.Suite.Config.RepoRoot, "templates", "secrets.yaml.template") + if err := os.MkdirAll(filepath.Dir(templateAbs), 0o755); err != nil { + t.Fatalf("mkdir template: %v", err) + } + if err := os.WriteFile(templateAbs, []byte("registryCredential: REPLACE_WITH_BASE64_DOCKER_CREDENTIAL\n"), 0o644); err != nil { + t.Fatalf("seed template: %v", err) + } + + destRel := "secrets/${BDD_TMP_SECRETS_NAME}" + if err := sc.iPrepareSelfManagedSecretsFile(destRel, templateRel); err != nil { + t.Fatalf("prepare secrets: %v", err) + } + destAbs := filepath.Join(sc.Suite.Config.RepoRoot, "secrets", "local-bdd-secrets.yaml") + got, err := os.ReadFile(destAbs) + if err != nil { + t.Fatalf("read destination: %v", err) + } + wantCredential := base64.StdEncoding.EncodeToString([]byte("$oauthtoken:test-api-key")) + if string(got) != "registryCredential: "+wantCredential+"\n" { + t.Fatalf("destination body does not contain the expected encoded credential") + } + info, err := os.Stat(destAbs) + if err != nil { + t.Fatalf("stat destination: %v", err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("destination mode = %o, want 600", info.Mode().Perm()) + } + if len(fake.runs) != 0 { + t.Fatalf("secret preparation wrote %d command log entries, want 0", len(fake.runs)) + } +} + +func TestIPrepareSelfManagedSecretsFileRestoresExistingDestination(t *testing.T) { + sc, _ := newScenarioContext(t) + t.Setenv("NGC_API_KEY", "test-api-key") + templateRel := "secrets.yaml.template" + templateAbs := filepath.Join(sc.Suite.Config.RepoRoot, templateRel) + if err := os.WriteFile(templateAbs, []byte("registryCredential: REPLACE_WITH_BASE64_DOCKER_CREDENTIAL\n"), 0o600); err != nil { + t.Fatalf("seed template: %v", err) + } + destRel := "local-secrets.yaml" + destAbs := filepath.Join(sc.Suite.Config.RepoRoot, destRel) + original := []byte("operator-authored: original\n") + if err := os.WriteFile(destAbs, original, 0o640); err != nil { + t.Fatalf("seed destination: %v", err) + } + + if err := sc.iPrepareSelfManagedSecretsFile(destRel, templateRel); err != nil { + t.Fatalf("prepare secrets: %v", err) + } + renderedInfo, err := os.Stat(destAbs) + if err != nil { + t.Fatalf("stat rendered destination: %v", err) + } + if renderedInfo.Mode().Perm() != 0o600 { + t.Fatalf("rendered destination mode = %o, want 600", renderedInfo.Mode().Perm()) + } + if err := sc.Suite.Ledger.RestoreAll(); err != nil { + t.Fatalf("restore: %v", err) + } + got, err := os.ReadFile(destAbs) + if err != nil { + t.Fatalf("read restored destination: %v", err) + } + if string(got) != string(original) { + t.Fatalf("restored body = %q, want original", got) + } + info, err := os.Stat(destAbs) + if err != nil { + t.Fatalf("stat restored destination: %v", err) + } + if info.Mode().Perm() != 0o640 { + t.Fatalf("restored mode = %o, want 640", info.Mode().Perm()) + } +} + +func TestIPrepareSelfManagedSecretsFileRestoresAbsentDestination(t *testing.T) { + sc, _ := newScenarioContext(t) + t.Setenv("NGC_API_KEY", "test-api-key") + templateRel := "secrets.yaml.template" + templateAbs := filepath.Join(sc.Suite.Config.RepoRoot, templateRel) + if err := os.WriteFile(templateAbs, []byte("registryCredential: REPLACE_WITH_BASE64_DOCKER_CREDENTIAL\n"), 0o600); err != nil { + t.Fatalf("seed template: %v", err) + } + destRel := "generated/local-secrets.yaml" + destAbs := filepath.Join(sc.Suite.Config.RepoRoot, destRel) + + if err := sc.iPrepareSelfManagedSecretsFile(destRel, templateRel); err != nil { + t.Fatalf("prepare secrets: %v", err) + } + if err := sc.Suite.Ledger.RestoreAll(); err != nil { + t.Fatalf("restore: %v", err) + } + if _, err := os.Stat(destAbs); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("generated destination should be removed: %v", err) + } +} + +func TestIPrepareSelfManagedSecretsFileFailureHidesCredentialMaterial(t *testing.T) { + sc, fake := newScenarioContext(t) + apiKey := "sensitive-test-api-key" + t.Setenv("NGC_API_KEY", apiKey) + templateRel := "secrets.yaml.template" + templateAbs := filepath.Join(sc.Suite.Config.RepoRoot, templateRel) + if err := os.WriteFile(templateAbs, []byte("registryCredential: missing\n"), 0o600); err != nil { + t.Fatalf("seed template: %v", err) + } + + err := sc.iPrepareSelfManagedSecretsFile("local-secrets.yaml", templateRel) + if err == nil { + t.Fatal("expected missing-placeholder error") + } + encoded := base64.StdEncoding.EncodeToString([]byte("$oauthtoken:" + apiKey)) + for _, secret := range []string{apiKey, encoded} { + if strings.Contains(err.Error(), secret) { + t.Fatalf("error leaked credential material: %v", err) + } + } + if len(fake.runs) != 0 { + t.Fatalf("failed secret preparation wrote %d command log entries, want 0", len(fake.runs)) + } +} + func TestIUpdateYAMLFileWritesKeys(t *testing.T) { sc, _ := newScenarioContext(t) rel := "env.yaml" @@ -832,23 +963,6 @@ func TestRegisterAllRunsAFeatureFile(t *testing.T) { } } -func TestISubstituteBase64DoesNotReturnSecretMaterial(t *testing.T) { - sc, _ := newScenarioContext(t) - rel := "secrets.yaml" - abs := filepath.Join(sc.Suite.Config.RepoRoot, rel) - if err := os.WriteFile(abs, []byte("token: REPLACE_ME\n"), 0o600); err != nil { - t.Fatalf("seed: %v", err) - } - t.Setenv("BDD_TMP_API_KEY", "real-secret-token") - if err := sc.iSubstituteBase64("REPLACE_ME", rel, "${BDD_TMP_API_KEY}"); err != nil { - t.Fatalf("substitute: %v", err) - } - got, _ := os.ReadFile(abs) - if strings.Contains(string(got), "real-secret-token") { - t.Fatalf("raw secret leaked into file body:\n%s", got) - } -} - func TestPullSecretInNamespacesKeepsAPIKeyOutOfArgv(t *testing.T) { sc, fake := newScenarioContext(t) t.Setenv("NGC_API_KEY", "super-secret-token")