Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions deploy/stacks/nvcf-compute-plane/tests/register-cluster.sh
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,24 @@ fi
values="${test_dir}/compute-plane/registration/gpu-a-register-values.yaml"
grep -q '^clusterID: generated-id$' "${values}"

no_config_record="${test_dir}/cli-args-no-config"
FAKE_CLI_RECORD="${no_config_record}" make -C "${test_dir}/compute-plane" register-cluster \
CLUSTER_NAME=gpu-b \
NVCF_CLI="${fake_cli}"
no_config_args=()
while IFS= read -r arg; do
no_config_args+=("${arg}")
done < "${no_config_record}"
if [[ "${no_config_args[0]}" != "self-hosted" ]]; then
printf 'register-cluster added arguments before self-hosted without NVCF_CLI_CONFIG: %q\n' \
"${no_config_args[0]}" >&2
exit 1
fi
if printf '%s\n' "${no_config_args[@]}" | grep -Fxq -- '--config'; then
echo "register-cluster passed --config without NVCF_CLI_CONFIG" >&2
exit 1
fi

rm "${profile}" "${record}"
if FAKE_CLI_RECORD="${record}" make -C "${test_dir}/compute-plane" register-cluster \
CLUSTER_NAME=gpu-a \
Expand Down
9 changes: 1 addition & 8 deletions src/clis/nvcf-cli/internal/openbao/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -396,15 +396,8 @@ func (c *Client) generateUserJWTTokenWithSubject(ctx context.Context, vaultToken
// mount, for example services/all/pki/root/cert/ca. The returned value is PEM
// text suitable for a public trust bundle.
func (c *Client) ReadPKICertificatePEM(ctx context.Context, pkiPath string) (string, error) {
rootToken, err := c.getOpenBaoRootToken()
if err != nil {
return "", fmt.Errorf("retrieving OpenBao root token: %w", err)
}
readURL := strings.TrimRight(c.config.OpenBaoURL, "/") + "/v1/" + strings.Trim(pkiPath, "/") + "/cert/ca"
curlArgs := []string{
"curl", "-sS", readURL,
"-H", "X-Vault-Token: " + rootToken,
}
curlArgs := []string{"curl", "-sS", readURL}
return readPKICertificatePEM(ctx, 3, 2*time.Second, func(ctx context.Context) (string, error) {
return c.executeKubectlRun(ctx, "openbao-pki-root-ca", curlArgs)
})
Expand Down
38 changes: 38 additions & 0 deletions src/clis/nvcf-cli/internal/openbao/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,9 @@ import (
"context"
"encoding/json"
"errors"
"os"
"os/exec"
"path/filepath"
"testing"

"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -164,3 +166,39 @@ func TestExecuteKubectlRunPreservesCommandError(t *testing.T) {
var execErr *exec.Error
require.ErrorAs(t, err, &execErr)
}

func TestReadPKICertificatePEMUsesPublicEndpointWithoutRootToken(t *testing.T) {
testDir := t.TempDir()
commandLog := filepath.Join(testDir, "kubectl.log")
kubectlPath := filepath.Join(testDir, "kubectl")
kubectlScript := `#!/bin/sh
printf '%s\n' "$*" >> "$KUBECTL_COMMAND_LOG"
case " $* " in
*" get secret "*) exit 91 ;;
*" X-Vault-Token: "*) exit 92 ;;
esac
printf '%s\n' '{"data":{"certificate":"-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n"}}'
`
require.NoError(t, os.WriteFile(kubectlPath, []byte(kubectlScript), 0o755))
t.Setenv("PATH", testDir+string(os.PathListSeparator)+os.Getenv("PATH"))
t.Setenv("KUBECTL_COMMAND_LOG", commandLog)

client := NewClient(&Config{
OpenBaoURL: "http://openbao-openbao.nvcf.svc.cluster.local:8200",
OpenBaoNamespace: "openbao",
OpenBaoSecretName: "openbao-root-token",
ClusterNamespace: "nvcf",
UtilityImage: "curlimages/curl:latest",
}, nil)

got, err := client.ReadPKICertificatePEM(context.Background(), "services/all/pki/root")
require.NoError(t, err)
assert.Equal(t, openBaoTestCertPEM, got)

logBody, err := os.ReadFile(commandLog)
require.NoError(t, err)
commands := string(logBody)
assert.NotContains(t, commands, " get secret ")
assert.NotContains(t, commands, "X-Vault-Token")
assert.Contains(t, commands, "/v1/services/all/pki/root/cert/ca")
}
16 changes: 11 additions & 5 deletions tests/bdd/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,11 @@ logic into `dsl/`.
token never appears in argv or per-command logs. Do not introduce
step handlers that capture secrets into env vars; relying on the
state file keeps the JWT out of `<seq>.cmd` lines.
- The live runner installs SIGINT and SIGTERM cleanup before scenarios run.
Interrupt cleanup cancels the active step and its Unix process group, waits
for that step to stop writing, then restores the same file and environment
ledgers while preventing later steps from starting. Ledger-backed generated
registry credentials must not remain after an interrupted run.
- Pre-suite destructive cleanup is governed by the single env var
`BDD_CLEANUP_MODE`. Valid values: `stack-single`, `stack-multi`,
`topology-single`, `topology-multi`, or unset. Unknown values fail
Expand Down Expand Up @@ -138,7 +143,7 @@ logic into `dsl/`.
deletion that catches topology infrastructure (`eg` in
`envoy-gateway-system`, the namespace itself, `cert-manager`).

## CLI vs Helmfile install paths (two intentionally distinct workflows)
## CLI vs Helmfile install paths

The suite exercises two operator workflows that share a stack but differ in
how the control plane is installed. Future changes must keep the CLI install
Expand Down Expand Up @@ -190,14 +195,15 @@ multi-cluster feature:
refused` against an in-cluster hostname.

2. Wrong kubectl context when `make register-cluster` runs. The
`nvcf-cli cluster register` command auto-discovers OIDC issuer
and JWKS from the CURRENT context by spawning a probe Job in
that cluster, then registers that identity with ICMS. If the
`self-hosted compute-plane register` command discovers OIDC issuer
and JWKS from its selected compute context, then registers that
identity with ICMS. If the
context is the cp cluster, ICMS records the cp cluster's JWKS
for the compute cluster's row. The compute cluster's NVCA agent
then 401s against ICMS at runtime ("Signed JWT rejected:
... no matching key(s) found"). Switch the context to the
compute cluster BEFORE `make register-cluster`, not after.
compute context explicitly through `COMPUTE_KUBE_CONTEXT` (or a
compute-scoped kubeconfig) before `make register-cluster`, not after.

## Tests

Expand Down
15 changes: 15 additions & 0 deletions tests/bdd/PLAN.md
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ original order. Repeated options and empty values are preserved.
| `Then Helm release {string} in namespace {string} using context {string} should contain values:` (YAML docstring) | Runs one explicit-context `helm get values -o yaml` for the named release and asserts that its values contain the supplied YAML subset. Extra map keys are allowed; lists remain order- and length-sensitive. Failure messages name the release and first differing path without printing release values. |
| `Then Kubernetes resource {string} in namespace {string} using context {string} should contain:` (YAML docstring) | The resource is explicit `kind/name`. Runs one `kubectl get -o yaml` against the named context and asserts that the resource YAML contains the supplied YAML subset. Extra map keys are allowed; lists remain order- and length-sensitive. Failure messages name the resource and first differing path without printing resource values. |
| `Then the rendered manifests in {string} should contain:` (table) | Requires a `text` header and one or more fixed strings. Recursively inspects regular files under the repo-relative directory and fails if any listed string is absent. `${VAR}` expansion applies to the path and table values. |
| `Then the rendered manifests in {string} should contain Kubernetes resource {string}` | Parses rendered YAML documents and requires an actual top-level resource matching the explicit `kind/name`. Nested references such as `Certificate.spec.issuerRef` do not satisfy the assertion. `${VAR}` expansion applies to the path, kind, and name. |
| `Then the rendered manifests in {string} under directories matching {string} should contain:` (table) | Positive rendered-manifest assertion scoped to files below a directory whose name matches the supplied shell pattern, such as `*-nats`. The render directory, directory-name pattern, and table values support `${VAR}` expansion. |
| `Then the rendered manifests in {string} should not contain:` (table) | Requires a `text` header and one or more fixed strings. Recursively inspects regular files under the repo-relative directory and fails if any listed string appears. `${VAR}` expansion applies to the path and table values. |
| `Then these Helm releases should be deployed using context {string}:` (table) | Requires `name` and `namespace` headers, with an optional `revision` header. Runs one explicit-context, all-namespaces `helm list` and asserts that every listed release has status `deployed`; non-empty revision cells are also matched. |
Expand Down Expand Up @@ -238,6 +239,15 @@ contract verified in `src/clis/nvcf-cli/cmd/`):
```
${NVCF_CLI} --config <cfg> self-hosted --control-plane-stack deploy/stacks/self-managed --compute-plane-stack deploy/stacks/nvcf-compute-plane --env local --plain compute-plane register --control-plane-profile <profile-path> --cluster-name <compute> --kube-context k3d-<compute> --region us-west-1 --output <values-path>
```
- Helmfile control-plane profile handoff (single cluster):
```
${NVCF_CLI} --config <cfg> self-hosted --control-plane-stack deploy/stacks/self-managed --env <env> control-plane profile export --cluster-name <control>
make -C deploy/stacks/nvcf-compute-plane register-cluster CLUSTER_NAME=<compute> CONTROL_PLANE_PROFILE=<profile-path> COMPUTE_KUBE_CONTEXT=k3d-<compute> NVCF_CLI=${NVCF_CLI}
```
The profile export runs after the selected Helmfile environment is installed
so endpoint and PKI trust data describe that deployment. A single-cluster
export omits both persistent context flags; the CLI accepts a split-cluster
pair or neither, and the bootstrap has already selected the local context.
- `self-hosted compute-plane install`:
```
${NVCF_CLI} --config <cfg> self-hosted --control-plane-stack deploy/stacks/self-managed --compute-plane-stack deploy/stacks/nvcf-compute-plane --env local --plain compute-plane install --values <values-path> --kube-context k3d-<compute> --cluster-name <compute>
Expand All @@ -257,6 +267,11 @@ restoration ledger:
- At suite teardown, the runner restores every registered path to its
pre-suite state. Files that did not exist before are deleted; files
that did are rewritten with the original bytes and mode.
- Live entry points cancel and quiesce the active step before restoring the
file and environment ledgers and exiting on SIGINT or SIGTERM. On Unix, the
command runner cancels the step's process group so shell, make, and kubectl
descendants cannot outlive restoration. This includes generated registry
credential files.
- `Config.LedgerDir` (`out/<run-id>/originals/`) is reserved for an
on-disk variant if very large fixtures ever push memory limits.
Today the directory is created but unused.
Expand Down
84 changes: 84 additions & 0 deletions tests/bdd/dsl/manifests.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,15 @@ limitations under the License.
package dsl

import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"

"gopkg.in/yaml.v3"
)
Expand All @@ -30,6 +36,84 @@ const (
ngcDockerUsername = "$oauthtoken"
)

// RenderedManifestsContainResource parses rendered YAML documents below root
// and requires one top-level Kubernetes resource with the requested kind and
// metadata.name. Nested references such as Certificate.spec.issuerRef do not
// satisfy the assertion.
func RenderedManifestsContainResource(root string, resource KubernetesResource) error {
root = strings.TrimSpace(Interpolate(root))
resource.Kind = strings.TrimSpace(Interpolate(resource.Kind))
resource.Name = strings.TrimSpace(Interpolate(resource.Name))
if root == "" {
return fmt.Errorf("rendered manifests directory is empty")
}
if resource.Kind == "" {
return fmt.Errorf("Kubernetes resource kind is empty")
}
if resource.Name == "" {
return fmt.Errorf("Kubernetes resource name is empty")
}
info, err := os.Stat(root)
if err != nil {
return fmt.Errorf("inspect rendered manifests directory %q: %w", root, err)
}
if !info.IsDir() {
return fmt.Errorf("rendered manifests path %q is not a directory", root)
}

yamlFilesInspected := 0
found := false
err = filepath.WalkDir(root, func(filePath string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return fmt.Errorf("inspect rendered manifest %q: %w", filePath, walkErr)
}
if !entry.Type().IsRegular() {
return nil
}
extension := strings.ToLower(filepath.Ext(filePath))
if extension != ".yaml" && extension != ".yml" {
return nil
}
yamlFilesInspected++

manifestBody, err := os.ReadFile(filePath)
if err != nil {
return fmt.Errorf("read rendered manifest %q: %w", filePath, err)
}

decoder := yaml.NewDecoder(bytes.NewReader(manifestBody))
for document := 1; ; document++ {
var manifest struct {
Kind string `yaml:"kind"`
Metadata struct {
Name string `yaml:"name"`
} `yaml:"metadata"`
}
if err := decoder.Decode(&manifest); err != nil {
if err == io.EOF {
break
}
return fmt.Errorf("parse rendered manifest %q document %d: invalid YAML", filePath, document)
}
if manifest.Kind == resource.Kind && manifest.Metadata.Name == resource.Name {
found = true
return fs.SkipAll
}
}
return nil
})
if err != nil {
return err
}
if found {
return nil
}
if yamlFilesInspected == 0 {
return fmt.Errorf("rendered manifests directory %q contains no YAML files", root)
}
return fmt.Errorf("rendered manifests in %q do not contain Kubernetes resource %s/%s", root, resource.Kind, resource.Name)
}

// NamespaceManifest returns a v1/Namespace YAML manifest body. The
// returned slice is the file contents the caller writes to disk and
// hands to kubectl apply.
Expand Down
51 changes: 51 additions & 0 deletions tests/bdd/dsl/manifests_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,61 @@ limitations under the License.
package dsl

import (
"os"
"path/filepath"
"strings"
"testing"
)

func TestRenderedManifestsContainResourceRejectsIssuerRefFragments(t *testing.T) {
root := t.TempDir()
body := `apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: llm-router-serving-cert
spec:
issuerRef:
kind: ClusterIssuer
name: nvcf-openbao-pki
`
if err := os.WriteFile(filepath.Join(root, "certificate.yaml"), []byte(body), 0o644); err != nil {
t.Fatalf("write rendered Certificate: %v", err)
}

err := RenderedManifestsContainResource(root, KubernetesResource{
Kind: "ClusterIssuer",
Name: "nvcf-openbao-pki",
})
if err == nil {
t.Fatal("issuerRef fragments were mistaken for a rendered ClusterIssuer resource")
}
}

func TestRenderedManifestsContainResourceFindsTopLevelResource(t *testing.T) {
root := t.TempDir()
body := `apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: llm-router-serving-cert
---
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
name: nvcf-openbao-pki
`
if err := os.WriteFile(filepath.Join(root, "pki.yaml"), []byte(body), 0o644); err != nil {
t.Fatalf("write rendered PKI resources: %v", err)
}

err := RenderedManifestsContainResource(root, KubernetesResource{
Kind: "ClusterIssuer",
Name: "nvcf-openbao-pki",
})
if err != nil {
t.Fatalf("find rendered ClusterIssuer: %v", err)
}
}

func TestNamespaceManifestShape(t *testing.T) {
body, err := NamespaceManifest("nvcf")
if err != nil {
Expand Down
Loading
Loading