diff --git a/.github/workflows/coherence-matrix.yaml b/.github/workflows/coherence-matrix.yaml index 23ce8d3a8..be5d2403f 100644 --- a/.github/workflows/coherence-matrix.yaml +++ b/.github/workflows/coherence-matrix.yaml @@ -1,4 +1,4 @@ -# Copyright 2021, 2025, Oracle Corporation and/or its affiliates. All rights reserved. +# Copyright 2021, 2026, Oracle Corporation and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at # http://oss.oracle.com/licenses/upl. @@ -49,8 +49,8 @@ jobs: - "12.2.1-4-SNAPSHOT" include: - matrixName: "15.1.1-0-SNAPSHOT" - coherenceVersion: "15.1.1-0-3-SNAPSHOT" - coherenceImage: "iad.ocir.io/odx-stateservice/test/coherence:15.1.1-0-3-SNAPSHOT-java17" + coherenceVersion: "15.1.1-0-5-SNAPSHOT" + coherenceImage: "iad.ocir.io/odx-stateservice/test/coherence:15.1.1-0-5-SNAPSHOT-java17" javaVersion: 17 coherenceIsJava8: false baseImage: "gcr.io/distroless/java17-debian12" @@ -229,6 +229,23 @@ jobs: docker pull gcr.io/distroless/java21-debian12 docker pull "${COHERENCE_IMAGE}" + - name: Health mutator protocol regressions + shell: bash + run: | + make generate + go test ./pkg/probe ./controllers/finalizer ./controllers/statefulset + ./mvnw -B -f java -pl coherence-operator -am test -Dtest=SecretLoginModuleTest,HealthSSLProviderConfigTest -Dsurefire.failIfNoSpecifiedTests=false + if [[ "${{ matrix.matrixName }}" == "15.1.1-0-SNAPSHOT" ]]; then + test/health-mutator/extract-coherence-jar.sh \ + "${{ matrix.coherenceImage }}" \ + "${{ matrix.coherenceVersion }}" \ + /tmp/health-mutator/coherence.jar + ./mvnw -B -f java -pl coherence-operator -am test \ + -Dtest=HealthSecuritySupportTest \ + -Dhealth.mutator.coherence.jar=/tmp/health-mutator/coherence.jar \ + -Dsurefire.failIfNoSpecifiedTests=false + fi + - name: Coherence Certification Tests shell: bash run: | diff --git a/.github/workflows/compatibility-tests.yaml b/.github/workflows/compatibility-tests.yaml index 4a7e8589e..b4a219fe5 100644 --- a/.github/workflows/compatibility-tests.yaml +++ b/.github/workflows/compatibility-tests.yaml @@ -1,4 +1,4 @@ -# Copyright 2019, 2025, Oracle Corporation and/or its affiliates. All rights reserved. +# Copyright 2019, 2026, Oracle Corporation and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at # http://oss.oracle.com/licenses/upl. @@ -164,6 +164,12 @@ jobs: sudo docker builder prune -a df -h + - name: Health mutator protocol regressions + shell: bash + run: | + go test ./pkg/probe ./controllers/finalizer ./controllers/statefulset + ./mvnw -B -f java -pl coherence-operator -am test -Dtest=SecretLoginModuleTest,HealthSSLProviderConfigTest -Dsurefire.failIfNoSpecifiedTests=false + - name: Compatibility Tests shell: bash run: | diff --git a/.github/workflows/health-mutator-tests.yaml b/.github/workflows/health-mutator-tests.yaml new file mode 100644 index 000000000..8a7b65d2b --- /dev/null +++ b/.github/workflows/health-mutator-tests.yaml @@ -0,0 +1,167 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at +# http://oss.oracle.com/licenses/upl. +name: Health mutator tests +on: + pull_request: + workflow_dispatch: +permissions: + contents: read +jobs: + client-and-admission: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: .go-version + - name: Generate API and schema + run: make generate manifests + - name: Client and controller tests + run: | + go test ./api/... ./controllers/... ./pkg/probe + go test ./pkg/runner -run '^(TestManagedHealth|TestHealthSecurityCheck)' + - name: Install isolated API server + run: | + go install sigs.k8s.io/controller-runtime/tools/setup-envtest@32e5e9e948a572779280969aaaf7db92f76d8252 + echo "KUBEBUILDER_ASSETS=$(setup-envtest use -p path 1.30.x!)" >> "$GITHUB_ENV" + - name: Admission tests + run: HEALTH_MUTATOR_RUNNER_SUITE=true go test -v ./test/health-mutator + java-support: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + java: ['8', '17', '25'] + steps: + - uses: actions/checkout@v4 + - name: Install build JDK + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '17' + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: ${{ matrix.java }} + # Compile Java 8 bytecode with JDK 17 because the Spring build dependency requires JDK 17. + # Surefire runs these focused tests on the selected Java runtime. They do not exercise the + # Coherence 21.12.5 code path that requires the legacy JDK 17 toolchain on Java 24 and later. + - name: Test packaged validator and TLS provider mapping + run: | + test_java="${JAVA_HOME}/bin/java" + JAVA_HOME="${JAVA_HOME_17_X64}" ./mvnw -B -f java -pl coherence-operator -am test \ + -P=-legacy-coherence-toolchain \ + -Djvm="${test_java}" \ + -Dtest=SecretLoginModuleTest,HealthSSLProviderConfigTest \ + -Dsurefire.failIfNoSpecifiedTests=false + - name: Upload reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: health-mutator-java-support-${{ matrix.java }} + path: java/coherence-operator/target/surefire-reports/ + runtime-e2e: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + COHERENCE_VERSION_LTS: 15.1.1-0-5 + KIND_CLUSTER: health-mutator + OPERATOR_IMAGE_REGISTRY: health-mutator.local + OPERATOR_IMAGE_TAG: ${{ github.sha }} + TEST_APPLICATION_IMAGE: health-mutator.local/operator-test:${{ github.sha }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: .go-version + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '17' + cache: maven + - name: Install runtime test tools + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends --yes gettext-base + - name: Build candidate and test application images + run: make build-operator build-basic-test-image + - name: Create isolated Kind cluster + run: make kind-single-worker + - name: Load digest-pinned images + id: images + shell: bash + run: | + operator_image="${OPERATOR_IMAGE_REGISTRY}/coherence-operator:${OPERATOR_IMAGE_TAG}" + + build/tools/bin/kind load docker-image --name "${KIND_CLUSTER}" "${operator_image}" + build/tools/bin/kind load docker-image --name "${KIND_CLUSTER}" "${TEST_APPLICATION_IMAGE}" + + first_node=$(build/tools/bin/kind get nodes --name "${KIND_CLUSTER}" | head -n 1) + operator_manifest_digest=$(docker exec "${first_node}" ctr --namespace k8s.io images list | \ + awk -v ref="${operator_image}" '$1 == ref {print $3}') + fixture_manifest_digest=$(docker exec "${first_node}" ctr --namespace k8s.io images list | \ + awk -v ref="${TEST_APPLICATION_IMAGE}" '$1 == ref {print $3}') + operator_digest="${operator_image%:*}@${operator_manifest_digest}" + fixture_digest="${TEST_APPLICATION_IMAGE%:*}@${fixture_manifest_digest}" + if [[ "${operator_digest}" != *@sha256:* || "${fixture_digest}" != *@sha256:* ]]; then + echo "The locally built images do not have immutable manifest digests" >&2 + exit 1 + fi + + while read -r node; do + docker exec "${node}" ctr --namespace k8s.io images tag --force \ + "${operator_image}" "${operator_digest}" + docker exec "${node}" ctr --namespace k8s.io images tag --force \ + "${TEST_APPLICATION_IMAGE}" "${fixture_digest}" + done < <(build/tools/bin/kind get nodes --name "${KIND_CLUSTER}") + + echo "operator_digest=${operator_digest}" >> "${GITHUB_OUTPUT}" + echo "fixture_digest=${fixture_digest}" >> "${GITHUB_OUTPUT}" + - name: Deploy candidate Operator + shell: bash + run: | + make prepare-deploy + ( + cd build/_output/config/manager + "${GITHUB_WORKSPACE}/build/tools/bin/kustomize" edit set image \ + "controller=${{ steps.images.outputs.operator_digest }}" + ) + kubectl create namespace operator-test --dry-run=client -o yaml | kubectl apply -f - + build/tools/bin/kustomize build build/_output/config/default | kubectl apply -f - + kubectl -n operator-test scale \ + deployment/coherence-operator-controller-manager --replicas=1 + kubectl -n operator-test rollout status \ + deployment/coherence-operator-controller-manager --timeout=10m + deployed_image=$(kubectl -n operator-test get \ + deployment/coherence-operator-controller-manager \ + -o jsonpath='{.spec.template.spec.containers[0].image}') + if [[ "${deployed_image}" != "${{ steps.images.outputs.operator_digest }}" ]]; then + echo "The deployed Operator image does not match the locally built digest" >&2 + exit 1 + fi + - name: Run health mutator runtime acceptance + shell: bash + run: | + context="kind-${KIND_CLUSTER}" + namespace=health-mutator-e2e + material="${RUNNER_TEMP}/health-mutator-e2e" + bash test/health-mutator/apply.sh \ + "${context}" "${namespace}" "${{ steps.images.outputs.fixture_digest }}" "${material}" + bash test/health-mutator/verify.sh "${context}" "${namespace}" "${material}" + - name: Collect failure diagnostics + if: failure() + shell: bash + run: | + context="kind-${KIND_CLUSTER}" + kubectl --context "${context}" get nodes -o wide || true + kubectl --context "${context}" get pods,statefulsets,pvc -A -o wide || true + kubectl --context "${context}" -n health-mutator-e2e \ + get coherence.coherence.oracle.com health-secure -o yaml || true + kubectl --context "${context}" -n operator-test logs \ + -l control-plane=coherence --all-containers --prefix --tail=1000 || true + kubectl --context "${context}" get events -A \ + --sort-by=.metadata.creationTimestamp || true + - name: Delete isolated Kind cluster + if: always() + run: make kind-stop diff --git a/.gitignore b/.gitignore index 221a2f8e1..04b7b1edd 100644 --- a/.gitignore +++ b/.gitignore @@ -65,7 +65,6 @@ bundle.Dockerfile node_modules .coh-history .cohql-history -runner venv .oca/ diff --git a/Makefile b/Makefile index 94bd942ed..58f668ad8 100644 --- a/Makefile +++ b/Makefile @@ -823,10 +823,12 @@ config/crd/bases/coherence.oracle.com_coherence.yaml: $(TOOLS_BIN)/kustomize $(A $(CONTROLLER_GEN) "crd:crdVersions={v1}" \ rbac:roleName=manager-role paths="{./api/...,./controllers/...}" \ output:crd:dir=config/crd/bases + go run ./utils/crd-fixup/ config/crd/bases/*.yaml cp -R config/crd/ config/crd-small $(CONTROLLER_GEN) "crd:crdVersions={v1},maxDescLen=0" \ rbac:roleName=manager-role paths="{./api/...,./controllers/...}" \ output:crd:dir=config/crd-small/bases + go run ./utils/crd-fixup/ config/crd-small/bases/*.yaml $(YQ) eval -i '.metadata.labels["app.kubernetes.io/version"] = "$(VERSION)"' config/crd/bases/coherence.oracle.com_coherence.yaml $(YQ) eval -i '.metadata.labels["app.kubernetes.io/version"] = "$(VERSION)"' config/crd/bases/coherence.oracle.com_coherencejob.yaml $(YQ) eval -i '.metadata.labels["app.kubernetes.io/version"] = "$(VERSION)"' config/crd-small/bases/coherence.oracle.com_coherence.yaml @@ -872,6 +874,7 @@ docs/about/04_coherence_spec.adoc: $(API_GO_FILES) utils/docgen/main.go api/v1/coherence_types.go \ api/v1/coherenceresource_types.go \ api/v1/coherencejobresource_types.go \ + api/v1/health_mutator.go \ > docs/about/04_coherence_spec.adoc # ---------------------------------------------------------------------------------------------------------------------- diff --git a/api/v1/coherence_types.go b/api/v1/coherence_types.go index 3c7c9e935..692bdc52f 100644 --- a/api/v1/coherence_types.go +++ b/api/v1/coherence_types.go @@ -2319,6 +2319,7 @@ func (in *ServiceSpec) createServiceSpec() corev1.ServiceSpec { // ScalingSpec is the configuration to control safe scaling. // +k8s:openapi-gen=true +// +kubebuilder:validation:XValidation:rule="!has(self.probe) || !has(self.probe.http) || !has(self.probe.http.method) || self.probe.http.method == 'GET'",message="HA requires a read-only GET action" type ScalingSpec struct { // ScalingPolicy describes how the replicas of the deployment will be scaled. // The default if not specified is based upon the value of the StorageEnabled field. @@ -2343,8 +2344,14 @@ type ScalingSpec struct { // StatusHA checking is primarily used during scaling of a deployment, a deployment must be in a safe Phase HA // state before scaling takes place. If StatusHA handler is disabled for a deployment (by specifically setting // Enabled to false then no check will take place and a deployment will be assumed to be safe). +// Empty and gRPC-only handlers remain successful no-ops for backward compatibility. The Operator emits a Warning +// event because these handlers do not execute a request or command and do not prove StatusHA or service suspension. // +k8s:openapi-gen=true +// +kubebuilder:validation:XValidation:rule="!has(self.http) || (!has(self.exec) && !has(self.httpGet) && !has(self.tcpSocket) && !has(self.grpc))",message="http cannot be combined with a legacy probe action" type Probe struct { + // HTTP is a method-aware request executed by the Operator. + // +optional + HTTP *HTTPAction `json:"http,omitempty"` corev1.ProbeHandler `json:",inline"` // Number of seconds after which the handler times out (only applies to http and tcp handlers). // Defaults to 1 second. Minimum value is 1. @@ -2352,6 +2359,81 @@ type Probe struct { TimeoutSeconds *int `json:"timeoutSeconds,omitempty"` } +// HTTPAction is an Operator HTTP action. Only HTTP 200 is successful; redirects are rejected. +// +kubebuilder:validation:XValidation:rule="!has(self.basicAuth) || (has(self.scheme) && self.scheme == 'HTTPS')",message="Basic authentication requires HTTPS" +// +kubebuilder:validation:XValidation:rule="!has(self.basicAuth) || !has(self.host) || size(self.host) == 0",message="Basic authentication requires the target Pod address; host must be empty" +// +kubebuilder:validation:XValidation:rule="!has(self.basicAuth) || !has(self.httpHeaders) || self.httpHeaders.all(h, h.name.lowerAscii() != 'authorization')",message="Authorization conflicts with basicAuth" +// +kubebuilder:validation:XValidation:rule="!has(self.tls) || (has(self.scheme) && self.scheme == 'HTTPS')",message="TLS settings require HTTPS" +// +kubebuilder:validation:XValidation:rule="!has(self.host) || (!self.host.contains('@') && !self.host.contains('/') && !self.host.contains('?') && !self.host.contains('#'))",message="host must not contain URL credentials or delimiters" +// +kubebuilder:validation:XValidation:rule="!has(self.path) || (self.path.startsWith('/') && !self.path.startsWith('//'))",message="path must be an absolute endpoint path" +// +kubebuilder:validation:XValidation:rule="!has(self.httpHeaders) || self.httpHeaders.all(h, h.name.lowerAscii() != 'authorization') || (has(self.scheme) && self.scheme == 'HTTPS')",message="Authorization requires HTTPS" +type HTTPAction struct { + HTTPEndpoint `json:",inline"` + // HTTPHeaders contains non-secret custom request headers. + // +optional + // +kubebuilder:validation:MaxItems=32 + HTTPHeaders []HTTPActionHeader `json:"httpHeaders,omitempty"` + // Method defaults to GET. + // +optional + // +kubebuilder:validation:Enum=GET;PUT + Method string `json:"method,omitempty"` + // BasicAuth references credentials in the resource namespace. It can only be used with the target Pod address, + // so Host must be empty when BasicAuth is configured. + // +optional + BasicAuth *HTTPBasicAuth `json:"basicAuth,omitempty"` + // TLS configures verified server authentication. + // +optional + TLS *HTTPClientTLS `json:"tls,omitempty"` +} + +// HTTPEndpoint identifies a target without Kubernetes GET-only semantics. +type HTTPEndpoint struct { + // Path is the URL path to request. It must be absolute and defaults to "/". + // +optional + // +kubebuilder:validation:MaxLength=2048 + Path string `json:"path,omitempty"` + // Port is the numeric or named port on the target Pod. + Port intstr.IntOrString `json:"port"` + // Host is the optional connection host. It defaults to the target Pod's address. + // +optional + // +kubebuilder:validation:MaxLength=253 + Host string `json:"host,omitempty"` + // Scheme is the protocol used for the request and defaults to HTTP. + // +optional + // +kubebuilder:validation:Enum=HTTP;HTTPS + Scheme corev1.URIScheme `json:"scheme,omitempty"` +} + +// HTTPActionHeader is a bounded non-secret HTTP request header. +type HTTPActionHeader struct { + // Name is the HTTP header name. + // +kubebuilder:validation:MaxLength=256 + Name string `json:"name"` + // Value is the HTTP header value. + // +kubebuilder:validation:MaxLength=8192 + Value string `json:"value"` +} + +// HTTPBasicAuth contains namespace-local credential references. +// +kubebuilder:validation:XValidation:rule="has(self.username.name) && size(self.username.name) > 0 && size(self.username.key) > 0 && (!has(self.username.optional) || !self.username.optional)",message="username requires a non-optional named Secret key" +// +kubebuilder:validation:XValidation:rule="has(self.password.name) && size(self.password.name) > 0 && size(self.password.key) > 0 && (!has(self.password.optional) || !self.password.optional)",message="password requires a non-optional named Secret key" +type HTTPBasicAuth struct { + // Username references the Secret key containing the Basic authentication username. + Username corev1.SecretKeySelector `json:"username"` + // Password references the Secret key containing the Basic authentication password. + Password corev1.SecretKeySelector `json:"password"` +} + +// HTTPClientTLS supplies a PEM trust bundle and optional verified DNS name. +// +kubebuilder:validation:XValidation:rule="has(self.caSecret.name) && size(self.caSecret.name) > 0 && size(self.caSecret.key) > 0 && (!has(self.caSecret.optional) || !self.caSecret.optional)",message="TLS requires a non-optional named CA Secret key" +type HTTPClientTLS struct { + // CASecret references the Secret key containing the PEM-encoded CA certificate bundle. + CASecret corev1.SecretKeySelector `json:"caSecret"` + // ServerName is the optional DNS name used to verify the server certificate. + // +optional + ServerName string `json:"serverName,omitempty"` +} + // GetTimeout returns the timeout value in seconds. func (in *Probe) GetTimeout() time.Duration { if in == nil || in.TimeoutSeconds == nil || *in.TimeoutSeconds <= 0 { diff --git a/api/v1/coherencejobresource_types.go b/api/v1/coherencejobresource_types.go index 0e45b6726..180e8f596 100644 --- a/api/v1/coherencejobresource_types.go +++ b/api/v1/coherencejobresource_types.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2025, Oracle and/or its affiliates. + * Copyright (c) 2020, 2026, Oracle and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl. */ @@ -504,10 +504,11 @@ type CoherenceJobResourceSpec struct { // +optional JobAnnotations map[string]string `json:"jobAnnotations,omitempty"` - // ReadyAction is a probe that will be executed when one or more Pods - // reach the ready state. The probe will be executed on every Pod that - // is ready. One the required number of ready Pods is reached the probe - // will also be executed on every Pod that becomes ready after that time. + // ReadyAction is attempted for each Ready Pod after ReadyCount is reached. + // The Operator does not automatically make another attempt while a Pod + // remains Ready. If a Pod stops being Ready and subsequently becomes Ready + // again, another attempt is made. The most recently recorded result is + // available in status.jobProbes. // +optional ReadyAction *CoherenceJobProbe `json:"readyAction,omitempty"` diff --git a/api/v1/coherenceresource_status_test.go b/api/v1/coherenceresource_status_test.go new file mode 100644 index 000000000..fa19e7f57 --- /dev/null +++ b/api/v1/coherenceresource_status_test.go @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ + +package v1 + +import ( + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +func TestUpdateFromJobClearsProbeErrorAfterLaterReadyTransition(t *testing.T) { + firstReady := metav1.NewTime(time.Now()) + secondReady := metav1.NewTime(firstReady.Add(time.Second)) + secondProbe := metav1.NewTime(secondReady.Add(time.Second)) + status := CoherenceResourceStatus{JobProbes: []CoherenceJobProbeStatus{{ + Pod: "job-0", + LastReadyTime: &firstReady, + LastProbeTime: &firstReady, + Success: ptr.To(false), + Error: ptr.To("HTTP probe failed: transport timeout"), + }}} + + updated := status.UpdateFromJob(&CoherenceJob{}, nil, []CoherenceJobProbeStatus{{ + Pod: "job-0", + LastReadyTime: &secondReady, + LastProbeTime: &secondProbe, + Success: ptr.To(true), + }}) + if !updated { + t.Fatal("successful attempt after a later Ready transition did not update status") + } + if len(status.JobProbes) != 1 { + t.Fatalf("expected the existing probe status to be updated, got %d entries", len(status.JobProbes)) + } + probe := status.MaybeFindJobProbeStatus("job-0") + if probe == nil || probe.Success == nil || !*probe.Success || probe.Error != nil { + t.Fatalf("unexpected probe status: %+v", probe) + } +} diff --git a/api/v1/coherenceresource_types.go b/api/v1/coherenceresource_types.go index a49ddf666..f64e7368d 100644 --- a/api/v1/coherenceresource_types.go +++ b/api/v1/coherenceresource_types.go @@ -466,8 +466,12 @@ func (in *Coherence) UpdateStatusVersion(v string) { // CoherenceStatefulSetResourceSpec defines the specification of a Coherence resource. A Coherence resource is // typically one or more Pods that perform the same functionality, for example storage members. // +k8s:openapi-gen=true +// +kubebuilder:validation:XValidation:rule="!has(self.healthMutatorConnection) || !self.healthMutatorConnection.secure || !has(self.suspendProbe)",message="managed secure mode excludes custom suspend probes" type CoherenceStatefulSetResourceSpec struct { - CoherenceResourceSpec `json:",inline"` + // HealthMutatorConnection configures the selected health server and suspension client. + // +optional + HealthMutatorConnection *HealthMutatorConnection `json:"healthMutatorConnection,omitempty"` + CoherenceResourceSpec `json:",inline"` // The optional name of the Coherence cluster that this Coherence resource belongs to. // If this value is set the Pods controlled by this Coherence resource will form a cluster // with other Pods controlled by Coherence resources with the same cluster name. @@ -554,8 +558,10 @@ type CoherenceStatefulSetResourceSpec struct { // using Coherence persistence features. // +optional AllowUnsafeDelete *bool `json:"allowUnsafeDelete,omitempty"` - // Actions to execute once all the Pods are ready after an initial deployment + // Actions to execute once all the Pods are ready after an initial deployment. + // A maximum of 128 actions may be configured. // +optional + // +kubebuilder:validation:MaxItems=128 Actions []Action `json:"actions,omitempty"` // List of sources to populate environment variables in the container. // The keys defined within a source must be a C_IDENTIFIER. All invalid keys @@ -639,6 +645,7 @@ func (in *CoherenceStatefulSetResourceSpec) CreateStatefulSet(deployment *Cohere replicas := in.GetReplicas() podTemplate := in.CreatePodTemplateSpec(deployment) + in.ConfigureHealthMutators(&podTemplate) // Work out the StatefulSet rolling upgrade strategy based on the // value of the Coherence spec RollingUpdateStrategy field @@ -763,8 +770,9 @@ func (in *CoherenceStatefulSetResourceSpec) GetDefaultSuspendProbe() *Probe { probe := Probe{ TimeoutSeconds: timeout, - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ + HTTP: &HTTPAction{ + Method: "PUT", + HTTPEndpoint: HTTPEndpoint{ Path: "/suspend", Port: intstr.FromString(PortNameHealth), }, @@ -1018,6 +1026,7 @@ func (in *CoherenceResourceStatus) UpdateFromJob(deployment *CoherenceJob, jobSt ps.LastProbeTime = s.LastProbeTime ps.LastReadyTime = s.LastReadyTime ps.Success = s.Success + ps.Error = s.Error updated = true } } @@ -1134,9 +1143,9 @@ func (in *CoherenceResourceStatus) FindJobProbeStatus(pod string) CoherenceJobPr func (in *CoherenceResourceStatus) MaybeFindJobProbeStatus(pod string) *CoherenceJobProbeStatus { if in != nil { - for _, status := range in.JobProbes { - if status.Pod == pod { - return &status + for i := range in.JobProbes { + if in.JobProbes[i].Pod == pod { + return &in.JobProbes[i] } } } diff --git a/api/v1/create_statefulset_applicationspec_test.go b/api/v1/create_statefulset_applicationspec_test.go index 30f02995d..ffcaad04b 100644 --- a/api/v1/create_statefulset_applicationspec_test.go +++ b/api/v1/create_statefulset_applicationspec_test.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2025, Oracle and/or its affiliates. + * Copyright (c) 2020, 2026, Oracle and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl. */ @@ -195,6 +195,27 @@ func TestCreateStatefulSetWithApplicationEntryPoint(t *testing.T) { assertStatefulSetCreation(t, deployment, stsExpected) } +func TestCreateStatefulSetWithApplicationEntryPointAndImageEntryPointDelivery(t *testing.T) { + spec := coh.CoherenceResourceSpec{ + Application: &coh.ApplicationSpec{ + EntryPoint: []string{"foo", "bar"}, + UseImageEntryPoint: ptr.To(true), + }, + } + + // Create the test deployment + deployment := createTestDeployment(spec) + // Create expected StatefulSet + stsExpected := createMinimalExpectedStatefulSet(deployment) + stsExpected.Spec.Template.Spec.Containers[0].Command = []string{"foo", "bar"} + stsExpected.Spec.Template.Spec.Containers[0].Args = nil + stsExpected.Spec.Template.Spec.Containers[0].Env = append(stsExpected.Spec.Template.Spec.Containers[0].Env, + corev1.EnvVar{Name: coh.EnvVarJdkOptions, Value: "@/coherence-operator/utils/coherence-entrypoint-args.txt"}) + + // assert that the StatefulSet is as expected + assertStatefulSetCreation(t, deployment, stsExpected) +} + func TestCreateStatefulSetUseImageEntryPointWithAltJavaOptEnvVar(t *testing.T) { spec := coh.CoherenceResourceSpec{ Application: &coh.ApplicationSpec{ diff --git a/api/v1/health_mutator.go b/api/v1/health_mutator.go new file mode 100644 index 000000000..4fe9832b4 --- /dev/null +++ b/api/v1/health_mutator.go @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ +package v1 + +import ( + "encoding/json" + "fmt" + corev1 "k8s.io/api/core/v1" +) + +// HealthMutatorAnnotation records connection references for each Pod generation. +const HealthMutatorAnnotation = "coherence.oracle.com/health-mutator-connection" + +// HealthMutatorEnv carries non-secret managed configuration to the runner. +const HealthMutatorEnv = "COHERENCE_HEALTH_MUTATOR_CONNECTION" + +// HealthMutatorConnection configures authenticated health operations for StatefulSets. +// +kubebuilder:validation:XValidation:rule="!self.secure || (has(self.basicAuth) && has(self.tls) && has(self.serverTLS))",message="secure mode requires credentials, client TLS and server TLS" +// +kubebuilder:validation:XValidation:rule="self.secure || (!has(self.basicAuth) && !has(self.tls) && !has(self.serverTLS) && !has(self.jaas))",message="security settings require secure mode" +// +kubebuilder:validation:XValidation:rule="!self.secure || (has(self.serverTLS.enabled) && self.serverTLS.enabled && (!has(self.serverTLS.requireClientCert) || !self.serverTLS.requireClientCert))",message="secure mode requires one-way server TLS" +// +kubebuilder:validation:XValidation:rule="!self.secure || (has(self.tls.serverName) && size(self.tls.serverName) > 0)",message="secure mode requires the verified server name" +// +kubebuilder:validation:XValidation:rule="!self.secure || (has(self.serverTLS.keyStore) && size(self.serverTLS.keyStore) > 0 && has(self.serverTLS.keyStorePasswordFile) && size(self.serverTLS.keyStorePasswordFile) > 0 && has(self.serverTLS.keyPasswordFile) && size(self.serverTLS.keyPasswordFile) > 0)",message="secure mode requires a server keystore and password files" +type HealthMutatorConnection struct { + // Secure enables authenticated PUT over verified TLS. + // +kubebuilder:default=false + Secure bool `json:"secure"` + // BasicAuth supplies credentials for authenticated health requests. + // +optional + BasicAuth *HTTPBasicAuth `json:"basicAuth,omitempty"` + // TLS configures CA trust and server name verification for health requests. + // +optional + TLS *HTTPClientTLS `json:"tls,omitempty"` + // ServerTLS configures the member's one-way TLS identity. TrustStore fields + // are accepted for compatibility but are not used by HealthSSLProvider. + // +optional + ServerTLS *SSLSpec `json:"serverTLS,omitempty"` + // JAAS selects how the member's CoherenceREST login context is configured. + // +optional + JAAS *HealthJAAS `json:"jaas,omitempty"` +} + +// HealthJAAS selects the packaged validator or an application-owned JAAS configuration. +type HealthJAAS struct { + // Mode is "packaged" to generate the packaged JAAS configuration or "existing" to use the + // application's configuration. It defaults to "packaged". + // +optional + // +kubebuilder:default=packaged + // +kubebuilder:validation:Enum=packaged;existing + Mode string `json:"mode,omitempty"` +} + +// Validate checks the complete managed connection before use. +func (in *HealthMutatorConnection) Validate() error { + if in == nil { + return nil + } + if !in.Secure { + if in.BasicAuth != nil || in.TLS != nil || in.ServerTLS != nil || in.JAAS != nil { + return fmt.Errorf("security settings require secure mode") + } + return nil + } + if in.BasicAuth == nil || in.TLS == nil || in.TLS.ServerName == "" || in.ServerTLS == nil || in.ServerTLS.Enabled == nil || !*in.ServerTLS.Enabled || (in.ServerTLS.RequireClientCert != nil && *in.ServerTLS.RequireClientCert) { + return fmt.Errorf("secure health mutators require credentials and one-way TLS with a verified server name") + } + a := &Probe{HTTP: &HTTPAction{HTTPEndpoint: HTTPEndpoint{Scheme: corev1.URISchemeHTTPS}, BasicAuth: in.BasicAuth, TLS: in.TLS}} + if err := a.ValidateAction(); err != nil { + return err + } + if in.JAAS != nil && in.JAAS.Mode != "" && in.JAAS.Mode != "packaged" && in.JAAS.Mode != "existing" { + return fmt.Errorf("unsupported health JAAS mode") + } + for _, value := range []*string{in.ServerTLS.KeyStore, in.ServerTLS.KeyStorePasswordFile, in.ServerTLS.KeyPasswordFile} { + if value == nil || *value == "" { + return fmt.Errorf("server keystore and password files are required") + } + } + return nil +} + +// ConfigureHealthMutators supplies only references and read-only mounts to the member container. +func (in *CoherenceStatefulSetResourceSpec) ConfigureHealthMutators(pod *corev1.PodTemplateSpec) { + connection := in.HealthMutatorConnection + if connection == nil || !connection.Secure { + return + } + data, _ := json.Marshal(connection) + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + pod.Annotations[HealthMutatorAnnotation] = string(data) + var containers []*corev1.Container + for i := range pod.Spec.Containers { + if pod.Spec.Containers[i].Name == ContainerNameCoherence { + containers = append(containers, &pod.Spec.Containers[i]) + } + } + for i := range pod.Spec.InitContainers { + if pod.Spec.InitContainers[i].Name == ContainerNameOperatorConfig { + containers = append(containers, &pod.Spec.InitContainers[i]) + } + } + mounted := map[string]bool{} + for _, container := range containers { + container.Env = append(container.Env, corev1.EnvVar{Name: HealthMutatorEnv, Value: string(data)}) + mount := func(name, secret, path string, items []corev1.KeyToPath) { + if !mounted[name] { + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{Name: name, VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: secret, Items: items}}}) + mounted[name] = true + } + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{Name: name, MountPath: path, ReadOnly: true}) + } + if connection.BasicAuth != nil { + mount("health-mutator-username", connection.BasicAuth.Username.Name, "/coherence/health-auth/username", []corev1.KeyToPath{{Key: connection.BasicAuth.Username.Key, Path: "value"}}) + mount("health-mutator-password", connection.BasicAuth.Password.Name, "/coherence/health-auth/password", []corev1.KeyToPath{{Key: connection.BasicAuth.Password.Key, Path: "value"}}) + } + if connection.ServerTLS != nil && connection.ServerTLS.Secrets != nil { + mount("health-mutator-tls", *connection.ServerTLS.Secrets, "/coherence/health-tls", nil) + } + probes := []struct { + probe *corev1.Probe + generated bool + }{ + {container.ReadinessProbe, isGeneratedHealthProbe(in.ReadinessProbe)}, + {container.LivenessProbe, isGeneratedHealthProbe(in.LivenessProbe)}, + {container.StartupProbe, in.StartupProbe != nil && isGeneratedHealthProbe(in.StartupProbe)}, + } + for _, item := range probes { + if item.generated && item.probe != nil && item.probe.HTTPGet != nil && item.probe.HTTPGet.Port.IntValue() == int(in.GetHealthPort()) { + item.probe.HTTPGet.Scheme = corev1.URISchemeHTTPS + } + } + } +} + +func isGeneratedHealthProbe(probe *ReadinessProbeSpec) bool { + return probe == nil || (probe.Exec == nil && probe.HTTPGet == nil && probe.TCPSocket == nil) +} diff --git a/api/v1/health_mutator_test.go b/api/v1/health_mutator_test.go new file mode 100644 index 000000000..e4d54adb5 --- /dev/null +++ b/api/v1/health_mutator_test.go @@ -0,0 +1,321 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ +package v1 + +import ( + "encoding/json" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + "reflect" + "strings" + "testing" +) + +func TestDisabledManagedHealthDoesNotChangePodMetadata(t *testing.T) { + for name, connection := range map[string]*HealthMutatorConnection{ + "omitted": nil, + "false": {Secure: false}, + } { + t.Run(name, func(t *testing.T) { + pod := &corev1.PodTemplateSpec{} + spec := &CoherenceStatefulSetResourceSpec{HealthMutatorConnection: connection} + spec.ConfigureHealthMutators(pod) + if pod.Annotations != nil { + t.Fatalf("disabled managed Health allocated annotations: %#v", pod.Annotations) + } + }) + } + + annotations := map[string]string{"example.com/owner": "application"} + pod := &corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Annotations: annotations}} + spec := &CoherenceStatefulSetResourceSpec{HealthMutatorConnection: &HealthMutatorConnection{Secure: false}} + spec.ConfigureHealthMutators(pod) + if !reflect.DeepEqual(pod.Annotations, map[string]string{"example.com/owner": "application"}) { + t.Fatalf("disabled managed Health changed unrelated annotations: %#v", pod.Annotations) + } +} + +func TestManagedHealthStatefulSetTemplateTransitions(t *testing.T) { + deployment := &Coherence{ + ObjectMeta: metav1.ObjectMeta{Name: "health-transition"}, + Spec: CoherenceStatefulSetResourceSpec{ + CoherenceResourceSpec: CoherenceResourceSpec{StartupProbe: &ReadinessProbeSpec{}}, + }, + } + + omitted := deployment.Spec.CreateStatefulSet(deployment).Spec.Template + assertDisabledManagedHealthTemplate(t, &omitted) + + deployment.Spec.HealthMutatorConnection = &HealthMutatorConnection{Secure: false} + disabled := deployment.Spec.CreateStatefulSet(deployment).Spec.Template + assertDisabledManagedHealthTemplate(t, &disabled) + if !reflect.DeepEqual(omitted, disabled) { + t.Fatal("omitted and secure:false managed Health generated different Pod templates") + } + + secureConnection := validManagedHealthConnection() + if err := secureConnection.Validate(); err != nil { + t.Fatalf("test secure connection is invalid: %v", err) + } + deployment.Spec.HealthMutatorConnection = secureConnection + secure := deployment.Spec.CreateStatefulSet(deployment).Spec.Template + assertSecureManagedHealthTemplate(t, &secure) + if reflect.DeepEqual(disabled, secure) { + t.Fatal("enabling managed Health did not change the Pod template") + } + + deployment.Spec.HealthMutatorConnection = &HealthMutatorConnection{Secure: false} + disabledAfterSecure := deployment.Spec.CreateStatefulSet(deployment).Spec.Template + assertDisabledManagedHealthTemplate(t, &disabledAfterSecure) + if !reflect.DeepEqual(disabled, disabledAfterSecure) { + t.Fatal("regenerating after disabling managed Health did not restore the disabled template") + } + + deployment.Spec.HealthMutatorConnection = validManagedHealthConnection() + secureAfterDisabled := deployment.Spec.CreateStatefulSet(deployment).Spec.Template + if !reflect.DeepEqual(secure, secureAfterDisabled) { + t.Fatal("re-enabling managed Health did not restore the secure Pod template") + } +} + +func validManagedHealthConnection() *HealthMutatorConnection { + username := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "health-auth"}, Key: "username"} + password := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "health-auth"}, Key: "password"} + ca := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "health-ca"}, Key: "ca.crt"} + return &HealthMutatorConnection{ + Secure: true, + BasicAuth: &HTTPBasicAuth{Username: username, Password: password}, + TLS: &HTTPClientTLS{CASecret: ca, ServerName: "health.example"}, + ServerTLS: &SSLSpec{ + Enabled: ptr.To(true), + Secrets: ptr.To("health-server-tls"), + KeyStore: ptr.To("server.jks"), + KeyStorePasswordFile: ptr.To("store-password"), + KeyPasswordFile: ptr.To("key-password"), + }, + } +} + +func assertDisabledManagedHealthTemplate(t *testing.T, pod *corev1.PodTemplateSpec) { + t.Helper() + if _, found := pod.Annotations[HealthMutatorAnnotation]; found { + t.Fatal("disabled managed Health generated a connection annotation") + } + assertNoManagedHealthWiring(t, &pod.Spec) + coherence := FindContainerInPodTemplate(ContainerNameCoherence, pod) + if coherence == nil { + t.Fatal("missing Coherence container") + } + for name, probe := range map[string]*corev1.Probe{ + "readiness": coherence.ReadinessProbe, + "liveness": coherence.LivenessProbe, + "startup": coherence.StartupProbe, + } { + if probe == nil || probe.HTTPGet == nil || probe.HTTPGet.Scheme != corev1.URISchemeHTTP { + t.Fatalf("disabled managed Health did not retain the generated HTTP %s probe", name) + } + } +} + +func assertNoManagedHealthWiring(t *testing.T, pod *corev1.PodSpec) { + t.Helper() + for _, volume := range pod.Volumes { + if strings.HasPrefix(volume.Name, "health-mutator-") { + t.Fatalf("disabled managed Health generated volume %q", volume.Name) + } + } + containers := append(append([]corev1.Container{}, pod.InitContainers...), pod.Containers...) + for _, container := range containers { + for _, env := range container.Env { + if env.Name == HealthMutatorEnv { + t.Fatalf("disabled managed Health configured %s on container %q", HealthMutatorEnv, container.Name) + } + } + for _, mount := range container.VolumeMounts { + if strings.HasPrefix(mount.Name, "health-mutator-") { + t.Fatalf("disabled managed Health generated mount %q on container %q", mount.Name, container.Name) + } + } + } +} + +func assertSecureManagedHealthTemplate(t *testing.T, pod *corev1.PodTemplateSpec) { + t.Helper() + if value := pod.Annotations[HealthMutatorAnnotation]; !strings.Contains(value, `"secure":true`) { + t.Fatalf("secure managed Health annotation is missing its connection: %q", value) + } + for _, name := range []string{"health-mutator-username", "health-mutator-password", "health-mutator-tls"} { + found := false + for _, volume := range pod.Spec.Volumes { + if volume.Name == name { + found = true + break + } + } + if !found { + t.Fatalf("secure managed Health did not generate volume %q", name) + } + } + for _, container := range []*corev1.Container{ + FindContainerInPodTemplate(ContainerNameCoherence, pod), + FindInitContainerInPodTemplate(ContainerNameOperatorConfig, pod), + } { + if container == nil { + t.Fatal("missing managed Health target container") + } + if !hasEnv(container.Env, HealthMutatorEnv) { + t.Fatalf("secure managed Health did not configure %s on container %q", HealthMutatorEnv, container.Name) + } + for _, name := range []string{"health-mutator-username", "health-mutator-password", "health-mutator-tls"} { + if !hasVolumeMount(container.VolumeMounts, name) { + t.Fatalf("secure managed Health did not mount %q on container %q", name, container.Name) + } + } + } + coherence := FindContainerInPodTemplate(ContainerNameCoherence, pod) + for name, probe := range map[string]*corev1.Probe{ + "readiness": coherence.ReadinessProbe, + "liveness": coherence.LivenessProbe, + "startup": coherence.StartupProbe, + } { + if probe == nil || probe.HTTPGet == nil || probe.HTTPGet.Scheme != corev1.URISchemeHTTPS { + t.Fatalf("secure managed Health did not convert the generated %s probe to HTTPS", name) + } + } +} + +func hasEnv(env []corev1.EnvVar, name string) bool { + for _, value := range env { + if value.Name == name { + return true + } + } + return false +} + +func hasVolumeMount(mounts []corev1.VolumeMount, name string) bool { + for _, mount := range mounts { + if mount.Name == name { + return true + } + } + return false +} + +func TestManagedHealthWiringIsStatefulSetOnly(t *testing.T) { + selector := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "auth-v1"}, Key: "value"} + connection := &HealthMutatorConnection{Secure: true, BasicAuth: &HTTPBasicAuth{Username: selector, Password: selector}, TLS: &HTTPClientTLS{CASecret: selector, ServerName: "health.example"}, ServerTLS: &SSLSpec{Enabled: ptr.To(true), Secrets: ptr.To("stores-v1"), KeyStore: ptr.To("server.jks"), KeyStorePasswordFile: ptr.To("password"), KeyPasswordFile: ptr.To("password")}} + if err := connection.Validate(); err != nil { + t.Fatal(err) + } + spec := &CoherenceStatefulSetResourceSpec{HealthMutatorConnection: connection} + pod := &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: ContainerNameCoherence}, {Name: "sidecar"}}}} + pod.Spec.InitContainers = []corev1.Container{{Name: ContainerNameOperatorConfig}, {Name: ContainerNameOperatorInit}} + spec.ConfigureHealthMutators(pod) + if len(pod.Spec.Volumes) != 3 || len(pod.Spec.InitContainers[0].VolumeMounts) != 3 || len(pod.Spec.InitContainers[1].VolumeMounts) != 0 { + t.Fatal("configuration init container wiring is incorrect") + } + if len(pod.Spec.Containers[0].VolumeMounts) != 3 || len(pod.Spec.Containers[1].VolumeMounts) != 0 { + t.Fatal("credential mounts must be confined to the member") + } + if !strings.Contains(pod.Annotations[HealthMutatorAnnotation], "auth-v1") { + t.Fatal("missing active generation references") + } + data, _ := json.Marshal(&CoherenceJobResourceSpec{}) + if strings.Contains(string(data), "healthMutatorConnection") { + t.Fatal("managed fields leaked into Job") + } + defaultProbe := spec.GetDefaultSuspendProbe() + if defaultProbe.HTTP == nil || defaultProbe.HTTP.Method != "PUT" || defaultProbe.HTTP.Path != "/suspend" { + t.Fatal("invalid generated suspend request") + } +} + +func TestManagedHealthServerTLSRequiresOnlyIdentityMaterial(t *testing.T) { + selector := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "auth-v1"}, Key: "value"} + connection := &HealthMutatorConnection{ + Secure: true, + BasicAuth: &HTTPBasicAuth{Username: selector, Password: selector}, + TLS: &HTTPClientTLS{CASecret: selector, ServerName: "health.example"}, + ServerTLS: &SSLSpec{ + Enabled: ptr.To(true), + KeyStore: ptr.To("server.jks"), + KeyStorePasswordFile: ptr.To("store-password"), + KeyPasswordFile: ptr.To("key-password"), + }, + } + if err := connection.Validate(); err != nil { + t.Fatalf("one-way TLS without a truststore was rejected: %v", err) + } + for name, clear := range map[string]func(*SSLSpec){ + "keystore": func(ssl *SSLSpec) { ssl.KeyStore = nil }, + "keystore password": func(ssl *SSLSpec) { ssl.KeyStorePasswordFile = nil }, + "key password": func(ssl *SSLSpec) { ssl.KeyPasswordFile = nil }, + } { + t.Run(name, func(t *testing.T) { + invalid := connection.DeepCopy() + clear(invalid.ServerTLS) + if err := invalid.Validate(); err == nil { + t.Fatal("missing server identity material was accepted") + } + }) + } + invalid := connection.DeepCopy() + invalid.ServerTLS.RequireClientCert = ptr.To(true) + if err := invalid.Validate(); err == nil { + t.Fatal("client certificate requirement was accepted") + } + connection.ServerTLS.TrustStore = ptr.To("legacy-trust.jks") + connection.ServerTLS.TrustStorePasswordFile = ptr.To("legacy-password") + if err := connection.Validate(); err != nil { + t.Fatalf("previously valid truststore fields were rejected: %v", err) + } +} + +func TestManagedHealthUsesHTTPSForGeneratedCustomPortProbes(t *testing.T) { + selector := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "auth-v1"}, Key: "value"} + connection := &HealthMutatorConnection{Secure: true, BasicAuth: &HTTPBasicAuth{Username: selector, Password: selector}, TLS: &HTTPClientTLS{CASecret: selector, ServerName: "health.example"}, ServerTLS: &SSLSpec{Enabled: ptr.To(true), Secrets: ptr.To("stores-v1"), KeyStore: ptr.To("server.jks"), KeyStorePasswordFile: ptr.To("password"), KeyPasswordFile: ptr.To("password")}} + healthPort := int32(8089) + spec := &CoherenceStatefulSetResourceSpec{ + HealthMutatorConnection: connection, + CoherenceResourceSpec: CoherenceResourceSpec{ + HealthPort: &healthPort, + StartupProbe: &ReadinessProbeSpec{}, + }, + } + container := spec.CreateCoherenceContainer(&Coherence{}) + pod := &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{container}}} + spec.ConfigureHealthMutators(pod) + for name, probe := range map[string]*corev1.Probe{ + "readiness": pod.Spec.Containers[0].ReadinessProbe, + "liveness": pod.Spec.Containers[0].LivenessProbe, + "startup": pod.Spec.Containers[0].StartupProbe, + } { + if probe == nil || probe.HTTPGet == nil || probe.HTTPGet.Port.IntValue() != int(healthPort) || probe.HTTPGet.Scheme != corev1.URISchemeHTTPS { + t.Fatalf("%s probe did not use HTTPS on custom health port", name) + } + } +} + +func TestManagedHealthPreservesCustomKubeletProbe(t *testing.T) { + healthPort := int32(8089) + custom := &corev1.HTTPGetAction{Path: "/custom", Port: intstr.FromInt32(healthPort), Scheme: corev1.URISchemeHTTP} + spec := &CoherenceStatefulSetResourceSpec{ + HealthMutatorConnection: &HealthMutatorConnection{Secure: true}, + CoherenceResourceSpec: CoherenceResourceSpec{ + HealthPort: &healthPort, + ReadinessProbe: &ReadinessProbeSpec{ProbeHandler: ProbeHandler{HTTPGet: custom}}, + }, + } + container := spec.CreateCoherenceContainer(&Coherence{}) + pod := &corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{container}}} + spec.ConfigureHealthMutators(pod) + if pod.Spec.Containers[0].ReadinessProbe.HTTPGet.Scheme != corev1.URISchemeHTTP { + t.Fatal("managed security changed an explicit custom readiness probe") + } +} diff --git a/api/v1/probe_validation.go b/api/v1/probe_validation.go new file mode 100644 index 000000000..448a5a3eb --- /dev/null +++ b/api/v1/probe_validation.go @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ +package v1 + +import ( + "fmt" + corev1 "k8s.io/api/core/v1" + "strings" +) + +// ValidateAction validates the new Operator HTTP action while preserving the +// historical behavior of legacy probe actions. +func (in *Probe) ValidateAction() error { + if in == nil { + return fmt.Errorf("probe action is required") + } + a := in.HTTP + if a == nil { + return nil + } + if in.Exec != nil || in.HTTPGet != nil || in.TCPSocket != nil || in.GRPC != nil { + return fmt.Errorf("HTTP cannot be combined with a legacy probe action") + } + if a.Method != "" && a.Method != "GET" && a.Method != "PUT" { + return fmt.Errorf("HTTP method must be GET or PUT") + } + if a.Scheme != "" && a.Scheme != corev1.URISchemeHTTP && a.Scheme != corev1.URISchemeHTTPS { + return fmt.Errorf("unsupported HTTP scheme") + } + if strings.ContainsAny(a.Host, "/@?#") || (!strings.HasPrefix(a.Path, "/") && a.Path != "") || strings.HasPrefix(a.Path, "//") { + return fmt.Errorf("invalid HTTP endpoint") + } + for _, h := range a.HTTPHeaders { + if strings.EqualFold(h.Name, "Authorization") && a.Scheme != corev1.URISchemeHTTPS { + return fmt.Errorf("authorization requires HTTPS") + } + } + valid := func(s corev1.SecretKeySelector) bool { + return s.Name != "" && s.Key != "" && (s.Optional == nil || !*s.Optional) + } + if a.BasicAuth != nil { + if a.Host != "" { + return fmt.Errorf("basic authentication requires the target Pod address; host must be empty") + } + if a.Scheme != corev1.URISchemeHTTPS { + return fmt.Errorf("basic authentication requires HTTPS") + } + if !valid(a.BasicAuth.Username) || !valid(a.BasicAuth.Password) { + return fmt.Errorf("required credential Secret selectors are incomplete") + } + for _, h := range a.HTTPHeaders { + if strings.EqualFold(h.Name, "Authorization") { + return fmt.Errorf("authorization conflicts with basicAuth") + } + } + } + if a.TLS != nil && (a.Scheme != corev1.URISchemeHTTPS || !valid(a.TLS.CASecret)) { + return fmt.Errorf("TLS requires HTTPS and a required CA Secret selector") + } + return nil +} diff --git a/api/v1/probe_validation_test.go b/api/v1/probe_validation_test.go new file mode 100644 index 000000000..d0e5c3fe9 --- /dev/null +++ b/api/v1/probe_validation_test.go @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ +package v1 + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +func TestBasicAuthRequiresTargetPodAddress(t *testing.T) { + selector := func(key string) corev1.SecretKeySelector { + return corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "auth"}, Key: key} + } + probe := &Probe{HTTP: &HTTPAction{ + HTTPEndpoint: HTTPEndpoint{Port: intstr.FromInt(6676), Scheme: corev1.URISchemeHTTPS}, + BasicAuth: &HTTPBasicAuth{Username: selector("username"), Password: selector("password")}, + }} + if err := probe.ValidateAction(); err != nil { + t.Fatalf("Pod-targeted Basic authentication was rejected: %v", err) + } + + probe.HTTP.TLS = &HTTPClientTLS{CASecret: selector("ca.crt"), ServerName: "health.example"} + if err := probe.ValidateAction(); err != nil { + t.Fatalf("TLS server name was treated as a connection host: %v", err) + } + + probe.HTTP.Host = "health.example" + if err := probe.ValidateAction(); err == nil || err.Error() != "basic authentication requires the target Pod address; host must be empty" { + t.Fatalf("custom host with Basic authentication was not rejected: %v", err) + } + + probe.HTTP.BasicAuth = nil + if err := probe.ValidateAction(); err != nil { + t.Fatalf("unauthenticated custom host was rejected: %v", err) + } +} diff --git a/controllers/coherence_controller.go b/controllers/coherence_controller.go index 3b606a690..62ca66b60 100644 --- a/controllers/coherence_controller.go +++ b/controllers/coherence_controller.go @@ -8,6 +8,7 @@ package controllers import ( "context" + "encoding/json" "fmt" "strconv" "time" @@ -45,6 +46,8 @@ import ( const ( // The name of this controller controllerName = "controllers.Coherence" + // The event and condition reason used when managed health configuration fails runtime validation. + invalidHealthMutatorConfigurationReason = "InvalidHealthMutatorConfiguration" // The error message template to use to indicate a reconcile failure. reconcileFailedMessage string = "failed to reconcile Coherence resource '%s' in namespace '%s'\n%s" @@ -161,6 +164,10 @@ func (in *CoherenceReconciler) Reconcile(ctx context.Context, request ctrl.Reque // This is an add request or update request + if err := validateHealthMutatorConfiguration(deployment); err != nil { + return in.rejectHealthMutatorConfiguration(ctx, deployment, err, log) + } + if deployment.Spec.AllowUnsafeDelete != nil && *deployment.Spec.AllowUnsafeDelete { if controllerutil.ContainsFinalizer(deployment, coh.CoherenceFinalizer) { err := in.finalizerManager.EnsureFinalizerRemoved(ctx, deployment) @@ -342,6 +349,66 @@ func (in *CoherenceReconciler) Reconcile(ctx context.Context, request ctrl.Reque return result, nil } +func validateHealthMutatorConfiguration(deployment *coh.Coherence) error { + connection := deployment.Spec.HealthMutatorConnection + if err := connection.Validate(); err != nil { + return err + } + if connection == nil || !connection.Secure { + return nil + } + if deployment.Spec.SuspendProbe != nil { + return fmt.Errorf("managed secure health mode excludes custom suspend probes") + } + return validateManagedHealthLauncher(deployment.Spec.Application) +} + +func validateManagedHealthLauncher(application *coh.ApplicationSpec) error { + if application == nil { + return nil + } + + useImageEntryPoint := application.UseImageEntryPoint != nil && *application.UseImageEntryPoint + if !useImageEntryPoint { + if len(application.EntryPoint) > 0 { + return fmt.Errorf("managed secure health with application.entryPoint requires application.useImageEntryPoint=true and a supported generated JVM argument delivery option") + } + return nil + } + + useJdkJavaOptions := application.UseJdkJavaOptions == nil || *application.UseJdkJavaOptions + hasAlternateJdkJavaOptions := application.AlternateJdkJavaOptions != nil && *application.AlternateJdkJavaOptions != "" + if !useJdkJavaOptions && !hasAlternateJdkJavaOptions { + return fmt.Errorf("managed secure health with application.useImageEntryPoint=true requires application.useJdkJavaOptions=true (or omitted), or a non-empty application.alternateJdkJavaOptions that the application passes to the member JVM") + } + return nil +} + +func (in *CoherenceReconciler) rejectHealthMutatorConfiguration(ctx context.Context, deployment *coh.Coherence, cause error, log logr.Logger) (ctrl.Result, error) { + message := fmt.Sprintf("Invalid health mutator configuration: %s", cause.Error()) + condition := coh.Condition{ + Type: coh.ConditionTypeFailed, + Status: coreV1.ConditionTrue, + Reason: coh.ConditionReason(invalidHealthMutatorConfigurationReason), + Message: message, + } + updated := deployment.DeepCopy() + changed := updated.Status.SetCondition(deployment, condition) + changed = updated.Status.Conditions.SetCondition(condition) || changed + + log.Error(cause, "Invalid health mutator configuration") + if !changed { + return ctrl.Result{}, nil + } + + in.GetEventRecorder().Eventf(deployment, nil, coreV1.EventTypeWarning, + invalidHealthMutatorConfigurationReason, "ValidateHealthMutatorConfiguration", message) + if err := in.GetClient().Status().Patch(ctx, updated, client.MergeFrom(deployment)); err != nil { + return ctrl.Result{}, errors.Wrap(err, "updating status for invalid health mutator configuration") + } + return ctrl.Result{}, nil +} + func (in *CoherenceReconciler) SetupWithManager(mgr ctrl.Manager, cs clients.ClientSet) error { SetupMonitoringResources(mgr) @@ -390,6 +457,7 @@ func (in *CoherenceReconciler) SetupWithManager(mgr ctrl.Manager, cs clients.Cli return ctrl.NewControllerManagedBy(mgr). For(template). + Watches(&coreV1.Secret{}, handler.EnqueueRequestsFromMapFunc(in.healthSecretRequests)). Named("coherence"). WithOptions(controller.Options{MaxConcurrentReconciles: 1}). Complete(in) @@ -428,3 +496,72 @@ func SetupMonitoringResources(mgr ctrl.Manager) { } mgr.GetScheme().AddKnownTypes(gv, &monitoringv1.ServiceMonitor{}, &monitoringv1.ServiceMonitorList{}) } + +// healthSecretRequests retries resources whose desired or live Pod generation references this Secret. +func (in *CoherenceReconciler) healthSecretRequests(ctx context.Context, secret client.Object) []reconcile.Request { + return mapHealthSecretRequests(ctx, in.GetClient(), secret) +} + +func mapHealthSecretRequests(ctx context.Context, kube client.Client, secret client.Object) []reconcile.Request { + resources := &coh.CoherenceList{} + if err := kube.List(ctx, resources, client.InNamespace(secret.GetNamespace())); err != nil { + return nil + } + matches := func(c *coh.HealthMutatorConnection) bool { + if c == nil { + return false + } + return (c.BasicAuth != nil && (c.BasicAuth.Username.Name == secret.GetName() || c.BasicAuth.Password.Name == secret.GetName())) || + (c.TLS != nil && c.TLS.CASecret.Name == secret.GetName()) || + (c.ServerTLS != nil && c.ServerTLS.Secrets != nil && *c.ServerTLS.Secrets == secret.GetName()) + } + matched := make(map[string]bool, len(resources.Items)) + unmatched := make(map[string]*coh.Coherence, len(resources.Items)) + for i := range resources.Items { + resource := &resources.Items[i] + if matches(resource.Spec.HealthMutatorConnection) { + matched[resource.Name] = true + } else { + unmatched[resource.Name] = resource + } + } + + if len(unmatched) > 0 { + pods := &coreV1.PodList{} + err := kube.List(ctx, pods, + client.InNamespace(secret.GetNamespace()), + client.MatchingLabels{coh.LabelComponent: coh.LabelComponentCoherencePod}) + if err == nil { + for i := range pods.Items { + pod := &pods.Items[i] + resource, found := unmatched[pod.Labels[coh.LabelCoherenceDeployment]] + if !found || !matchesLabels(pod.Labels, resource.Spec.CreatePodSelectorLabels(resource)) { + continue + } + connection := &coh.HealthMutatorConnection{} + if json.Unmarshal([]byte(pod.Annotations[coh.HealthMutatorAnnotation]), connection) == nil && matches(connection) { + matched[resource.Name] = true + delete(unmatched, resource.Name) + } + } + } + } + + var requests []reconcile.Request + for i := range resources.Items { + resource := &resources.Items[i] + if matched[resource.Name] { + requests = append(requests, reconcile.Request{NamespacedName: types.NamespacedName{Name: resource.Name, Namespace: resource.Namespace}}) + } + } + return requests +} + +func matchesLabels(actual, required map[string]string) bool { + for key, value := range required { + if actual[key] != value { + return false + } + } + return true +} diff --git a/controllers/coherence_controller_secret_test.go b/controllers/coherence_controller_secret_test.go new file mode 100644 index 000000000..ae060f3e4 --- /dev/null +++ b/controllers/coherence_controller_secret_test.go @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ + +package controllers + +import ( + "context" + "encoding/json" + "testing" + + coh "github.com/oracle/coherence-operator/api/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +type podListCountingClient struct { + client.Client + podListCalls int +} + +func (c *podListCountingClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if _, ok := list.(*corev1.PodList); ok { + c.podListCalls++ + } + return c.Client.List(ctx, list, opts...) +} + +func TestHealthSecretRequestsListsPodsOnce(t *testing.T) { + const ( + namespace = "test" + secretName = "health-secret" + ) + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := coh.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + + connection := func(name string) *coh.HealthMutatorConnection { + selector := corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: name}, + Key: "value", + } + return &coh.HealthMutatorConnection{ + BasicAuth: &coh.HTTPBasicAuth{Username: selector, Password: selector}, + } + } + resource := func(name string, desired *coh.HealthMutatorConnection) *coh.Coherence { + return &coh.Coherence{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: coh.CoherenceStatefulSetResourceSpec{HealthMutatorConnection: desired}, + } + } + pod := func(name string, owner *coh.Coherence, live *coh.HealthMutatorConnection) *corev1.Pod { + data, err := json.Marshal(live) + if err != nil { + t.Fatal(err) + } + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: owner.Namespace, + Labels: owner.Spec.CreatePodSelectorLabels(owner), + Annotations: map[string]string{coh.HealthMutatorAnnotation: string(data)}, + }, + } + } + + desired := resource("desired", connection(secretName)) + liveOnly := resource("live-only", nil) + unrelated := resource("unrelated", nil) + otherNamespace := resource("other-namespace", connection(secretName)) + otherNamespace.Namespace = "other" + + objects := []client.Object{ + desired, + liveOnly, + unrelated, + otherNamespace, + pod("desired-0", desired, connection(secretName)), + pod("live-only-0", liveOnly, connection(secretName)), + pod("unrelated-0", unrelated, connection("other-secret")), + } + kube := &podListCountingClient{Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build()} + requests := mapHealthSecretRequests(context.Background(), kube, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: secretName, Namespace: namespace}, + }) + + if kube.podListCalls != 1 { + t.Fatalf("expected one Pod list, got %d", kube.podListCalls) + } + if len(requests) != 2 { + t.Fatalf("expected two requests, got %v", requests) + } + requested := map[string]int{} + for _, request := range requests { + requested[request.Name]++ + } + if requested[desired.Name] != 1 { + t.Errorf("expected one request for desired reference, got %d", requested[desired.Name]) + } + if requested[liveOnly.Name] != 1 { + t.Errorf("expected one request for live Pod reference, got %d", requested[liveOnly.Name]) + } + if requested[unrelated.Name] != 0 || requested[otherNamespace.Name] != 0 { + t.Errorf("unexpected requests: %v", requested) + } +} diff --git a/controllers/coherence_controller_test.go b/controllers/coherence_controller_test.go new file mode 100644 index 000000000..580573103 --- /dev/null +++ b/controllers/coherence_controller_test.go @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ +package controllers_test + +import ( + "context" + "strings" + "testing" + + "github.com/go-logr/logr" + coh "github.com/oracle/coherence-operator/api/v1" + "github.com/oracle/coherence-operator/controllers" + "github.com/oracle/coherence-operator/pkg/clients" + "github.com/oracle/coherence-operator/pkg/fakes" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + kubescheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" +) + +func TestInvalidHealthMutatorConfigurationReportsFailure(t *testing.T) { + if err := coh.AddToScheme(kubescheme.Scheme); err != nil { + t.Fatal(err) + } + + selector := corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "health-secret"}, + Key: "value", + } + validConnection := &coh.HealthMutatorConnection{ + Secure: true, + BasicAuth: &coh.HTTPBasicAuth{Username: selector, Password: selector}, + TLS: &coh.HTTPClientTLS{CASecret: selector, ServerName: "health.example"}, + ServerTLS: &coh.SSLSpec{ + Enabled: ptr.To(true), + KeyStore: ptr.To("server.p12"), + KeyStorePasswordFile: ptr.To("store-password"), + KeyPasswordFile: ptr.To("key-password"), + }, + } + + tests := []struct { + name string + connection *coh.HealthMutatorConnection + probe *coh.Probe + application *coh.ApplicationSpec + message string + }{ + { + name: "incomplete secure connection", + connection: &coh.HealthMutatorConnection{Secure: true}, + message: "secure health mutators require credentials", + }, + { + name: "managed connection with custom suspend probe", + connection: validConnection, + probe: &coh.Probe{HTTP: &coh.HTTPAction{ + Method: "PUT", + HTTPEndpoint: coh.HTTPEndpoint{Path: "/suspend", Port: intstr.FromInt(8089)}, + }}, + message: "managed secure health mode excludes custom suspend probes", + }, + { + name: "managed connection without JVM argument delivery", + connection: validConnection, + application: &coh.ApplicationSpec{ + UseImageEntryPoint: ptr.To(true), + UseJdkJavaOptions: ptr.To(false), + }, + message: "requires application.useJdkJavaOptions=true", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resource := &coh.Coherence{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "test"}, + } + resource.Spec.HealthMutatorConnection = tt.connection + resource.Spec.SuspendProbe = tt.probe + resource.Spec.Application = tt.application + + kube := fakes.NewFakeClient(resource) + events := fakes.NewFakeEventRecorder(10) + manager := &fakes.FakeManager{Scheme: kubescheme.Scheme, Client: kube, Events: events} + reconciler := &controllers.CoherenceReconciler{Client: kube, Log: logr.Discard()} + reconciler.SetCommonReconciler("test.Coherence", manager, clients.ClientSet{}) + + result, err := reconciler.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: resource.Namespace, Name: resource.Name}}) + if err != nil { + t.Fatalf("reconcile returned an error: %v", err) + } + if result != (ctrl.Result{}) { + t.Fatalf("invalid permanent configuration was requeued: %+v", result) + } + + actual := &coh.Coherence{} + if err := kube.Get(context.Background(), types.NamespacedName{Namespace: resource.Namespace, Name: resource.Name}, actual); err != nil { + t.Fatal(err) + } + if actual.Status.Phase != coh.ConditionTypeFailed { + t.Fatalf("phase = %q, want %q", actual.Status.Phase, coh.ConditionTypeFailed) + } + condition := actual.Status.Conditions.GetCondition(coh.ConditionTypeFailed) + if condition == nil || condition.Status != corev1.ConditionTrue || condition.Reason != "InvalidHealthMutatorConfiguration" || !strings.Contains(condition.Message, tt.message) { + t.Fatalf("unexpected Failed condition: %+v", condition) + } + + event, found := manager.NextEvent() + if !found { + t.Fatal("missing invalid configuration event") + } + if event.Type != corev1.EventTypeWarning || event.Reason != "InvalidHealthMutatorConfiguration" || !strings.Contains(event.Message, tt.message) { + t.Fatalf("unexpected event: %+v", event) + } + if _, found := manager.NextEvent(); found { + t.Fatal("unexpected additional event") + } + + result, err = reconciler.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: resource.Namespace, Name: resource.Name}}) + if err != nil || result != (ctrl.Result{}) { + t.Fatalf("unchanged invalid configuration did not remain quiet: result=%+v err=%v", result, err) + } + if _, found := manager.NextEvent(); found { + t.Fatal("unchanged invalid configuration emitted another event") + } + }) + } +} diff --git a/controllers/coherence_controller_validation_test.go b/controllers/coherence_controller_validation_test.go new file mode 100644 index 000000000..b40415f53 --- /dev/null +++ b/controllers/coherence_controller_validation_test.go @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ + +package controllers + +import ( + "strings" + "testing" + + coh "github.com/oracle/coherence-operator/api/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +func TestValidateHealthMutatorLauncherConfiguration(t *testing.T) { + selector := corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "health-secret"}, + Key: "value", + } + secureConnection := &coh.HealthMutatorConnection{ + Secure: true, + BasicAuth: &coh.HTTPBasicAuth{Username: selector, Password: selector}, + TLS: &coh.HTTPClientTLS{CASecret: selector, ServerName: "health.example"}, + ServerTLS: &coh.SSLSpec{ + Enabled: ptr.To(true), + KeyStore: ptr.To("server.p12"), + KeyStorePasswordFile: ptr.To("store-password"), + KeyPasswordFile: ptr.To("key-password"), + }, + } + + tests := []struct { + name string + connection *coh.HealthMutatorConnection + application *coh.ApplicationSpec + wantError string + }{ + { + name: "secure normal operator launcher", + connection: secureConnection, + }, + { + name: "secure image entry point with default JDK options", + connection: secureConnection, + application: &coh.ApplicationSpec{ + UseImageEntryPoint: ptr.To(true), + }, + }, + { + name: "secure image entry point with explicit JDK options", + connection: secureConnection, + application: &coh.ApplicationSpec{ + UseImageEntryPoint: ptr.To(true), + UseJdkJavaOptions: ptr.To(true), + }, + }, + { + name: "secure image entry point without JVM argument delivery", + connection: secureConnection, + application: &coh.ApplicationSpec{ + UseImageEntryPoint: ptr.To(true), + UseJdkJavaOptions: ptr.To(false), + }, + wantError: "requires application.useJdkJavaOptions=true", + }, + { + name: "secure image entry point with empty alternate JVM options", + connection: secureConnection, + application: &coh.ApplicationSpec{ + UseImageEntryPoint: ptr.To(true), + UseJdkJavaOptions: ptr.To(false), + AlternateJdkJavaOptions: ptr.To(""), + }, + wantError: "non-empty application.alternateJdkJavaOptions", + }, + { + name: "secure image entry point with alternate JVM options", + connection: secureConnection, + application: &coh.ApplicationSpec{ + UseImageEntryPoint: ptr.To(true), + UseJdkJavaOptions: ptr.To(false), + AlternateJdkJavaOptions: ptr.To("ALT_JAVA_OPTS"), + }, + }, + { + name: "secure custom entry point without image entry point delivery", + connection: secureConnection, + application: &coh.ApplicationSpec{ + EntryPoint: []string{"/application/start"}, + }, + wantError: "application.entryPoint requires application.useImageEntryPoint=true", + }, + { + name: "secure custom entry point with default JDK options delivery", + connection: secureConnection, + application: &coh.ApplicationSpec{ + EntryPoint: []string{"/application/start"}, + UseImageEntryPoint: ptr.To(true), + }, + }, + { + name: "secure custom entry point with alternate JVM options delivery", + connection: secureConnection, + application: &coh.ApplicationSpec{ + EntryPoint: []string{"/application/start"}, + UseImageEntryPoint: ptr.To(true), + UseJdkJavaOptions: ptr.To(false), + AlternateJdkJavaOptions: ptr.To("ALT_JAVA_OPTS"), + }, + }, + { + name: "omitted managed health leaves image settings unchanged", + application: &coh.ApplicationSpec{ + UseImageEntryPoint: ptr.To(true), + UseJdkJavaOptions: ptr.To(false), + }, + }, + { + name: "insecure managed health leaves custom entry point unchanged", + connection: &coh.HealthMutatorConnection{Secure: false}, + application: &coh.ApplicationSpec{ + EntryPoint: []string{"/application/start"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + deployment := &coh.Coherence{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "test"}, + } + deployment.Spec.HealthMutatorConnection = tt.connection + deployment.Spec.Application = tt.application + + err := validateHealthMutatorConfiguration(deployment) + if tt.wantError == "" { + if err != nil { + t.Fatalf("validateHealthMutatorConfiguration() returned an error: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantError) { + t.Fatalf("validateHealthMutatorConfiguration() error = %v, want containing %q", err, tt.wantError) + } + }) + } +} diff --git a/controllers/errorhandling/error_handler.go b/controllers/errorhandling/error_handler.go index 370c30073..2408f2342 100644 --- a/controllers/errorhandling/error_handler.go +++ b/controllers/errorhandling/error_handler.go @@ -16,6 +16,7 @@ import ( "github.com/go-logr/logr" coh "github.com/oracle/coherence-operator/api/v1" + "github.com/oracle/coherence-operator/pkg/probe" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -107,6 +108,13 @@ func (eh *ErrorHandler) HandleError(ctx context.Context, err error, resource coh // categorizeError categorizes an error based on its type and content func (eh *ErrorHandler) categorizeError(err error) ErrorCategory { + // Preserve the existing service-suspension recovery policy for typed errors, + // without classifying sanitized authentication details as permanent failures. + var suspension *probe.SuspensionError + if errors.As(err, &suspension) { + return ErrorCategoryRecoverable + } + // Check for Kubernetes API errors if apierrors.IsNotFound(err) { return ErrorCategoryTransient // Resource might appear later @@ -422,7 +430,8 @@ func (eh *ErrorHandler) attemptRecovery(ctx context.Context, err error, resource errStr := err.Error() // Service suspension failure - if strings.Contains(errStr, "failed to suspend services") { + var suspension *probe.SuspensionError + if errors.As(err, &suspension) || strings.Contains(errStr, "failed to suspend services") { return eh.recoverFromServiceSuspensionFailure(ctx, resource) } @@ -468,53 +477,36 @@ func (eh *ErrorHandler) attemptRecovery(ctx context.Context, err error, resource // recoverFromServiceSuspensionFailure attempts to recover from a service suspension failure func (eh *ErrorHandler) recoverFromServiceSuspensionFailure(ctx context.Context, resource coh.CoherenceResource) (reconcile.Result, error) { - // 1. Log the recovery attempt eh.Log.Info("Attempting to recover from service suspension failure", - "resource", resource.GetName(), - "namespace", resource.GetNamespace()) - - // 2. Record an event + "resource", resource.GetName(), "namespace", resource.GetNamespace()) eh.EventRecorder.Eventf(resource, nil, corev1.EventTypeNormal, "RecoveryAttempt", "RecoverServiceSuspension", "Attempting to recover from service suspension failure") - // 3. Implement the recovery logic - // For service suspension failures, we'll try to force remove the finalizer - // This allows the resource to be deleted even if service suspension failed latest := resource.DeepCopyResource() if err := eh.Client.Get(ctx, types.NamespacedName{ - Namespace: resource.GetNamespace(), - Name: resource.GetName(), + Namespace: resource.GetNamespace(), Name: resource.GetName(), }, latest); err != nil { return reconcile.Result{RequeueAfter: time.Minute}, err } - // Check if this is a deletion and has the Coherence finalizer + // Preserve the existing escape hatch for deletion; normal scaling and updates + // must not acquire a deletion bypass merely because suspension failed. if latest.GetDeletionTimestamp() != nil { - // This is a deletion, so we'll try to remove the finalizer annotations := latest.GetAnnotations() if annotations == nil { annotations = make(map[string]string) } - - // Add an annotation to indicate we're bypassing the finalizer annotations["coherence.oracle.com/finalizer-bypass"] = "true" latest.SetAnnotations(annotations) - - // Update the resource with the annotation if err := eh.Client.Update(ctx, latest); err != nil { eh.Log.Error(err, "Failed to add finalizer bypass annotation") return reconcile.Result{RequeueAfter: time.Minute}, err } - eh.Log.Info("Added finalizer bypass annotation to resource", - "resource", resource.GetName(), - "namespace", resource.GetNamespace()) - + "resource", resource.GetName(), "namespace", resource.GetNamespace()) eh.EventRecorder.Eventf(resource, nil, corev1.EventTypeNormal, "RecoveryAction", "BypassFinalizer", "Added finalizer bypass annotation to allow deletion despite service suspension failure") } - - // 4. Return a result that requeues after a short delay to check if recovery was successful return reconcile.Result{RequeueAfter: 10 * time.Second}, nil } diff --git a/controllers/errorhandling/suspension_recovery_test.go b/controllers/errorhandling/suspension_recovery_test.go new file mode 100644 index 000000000..a4d91a2e8 --- /dev/null +++ b/controllers/errorhandling/suspension_recovery_test.go @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ +package errorhandling + +import ( + "context" + "fmt" + "testing" + "time" + + coh "github.com/oracle/coherence-operator/api/v1" + "github.com/oracle/coherence-operator/pkg/probe" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/events" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestSuspensionRecoveryPreservesDeletionPolicy(t *testing.T) { + failures := map[string]error{ + "legacy": fmt.Errorf("failed to suspend services"), + "unauthorized": &probe.SuspensionError{Cause: &probe.RequestError{StatusCode: 401}}, + "forbidden": &probe.SuspensionError{Cause: &probe.RequestError{StatusCode: 403}}, + "tls": &probe.SuspensionError{Cause: &probe.RequestError{Kind: "transport or TLS"}}, + "secret": &probe.SuspensionError{Cause: &probe.RequestError{Kind: "Secret unavailable"}}, + } + for name, failure := range failures { + for _, deleting := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/deleting=%t", name, deleting), func(t *testing.T) { + scheme := runtime.NewScheme() + if err := coh.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + c := &coh.Coherence{ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "test", Finalizers: []string{coh.CoherenceFinalizer}}} + if deleting { + now := metav1.Now() + c.DeletionTimestamp = &now + } + kube := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(c).WithObjects(c).Build() + handler := &ErrorHandler{Client: kube, EventRecorder: events.NewFakeRecorder(20)} + result, err := handler.HandleError(context.Background(), fmt.Errorf("wrapped: %w", failure), c, "suspension failed") + if err != nil || result.RequeueAfter != 10*time.Second { + t.Fatalf("expected existing recovery retry, got %+v, %v", result, err) + } + actual := &coh.Coherence{} + if err := kube.Get(context.Background(), client.ObjectKeyFromObject(c), actual); err != nil { + t.Fatal(err) + } + _, bypass := actual.Annotations["coherence.oracle.com/finalizer-bypass"] + if bypass != deleting { + t.Fatalf("deletion bypass=%t, deleting=%t", bypass, deleting) + } + if actual.Annotations[AnnotationErrorCount] != "1" { + t.Fatalf("existing error tracking was skipped: %v", actual.Annotations) + } + if len(actual.Finalizers) != 1 { + t.Fatal("recovery should annotate the resource for finalization, not remove its finalizer directly") + } + }) + } + } +} diff --git a/controllers/finalizer/finalizer.go b/controllers/finalizer/finalizer.go index 8342ee2e3..b65421e41 100644 --- a/controllers/finalizer/finalizer.go +++ b/controllers/finalizer/finalizer.go @@ -8,7 +8,6 @@ package finalizer import ( "context" - "fmt" "strconv" "github.com/go-logr/logr" @@ -70,16 +69,14 @@ func (fm *FinalizerManager) EnsureFinalizerRemoved(ctx context.Context, c *coh.C // FinalizeDeployment performs any required finalizer tasks for the Coherence resource func (fm *FinalizerManager) FinalizeDeployment(ctx context.Context, c *coh.Coherence, findStatefulSet func(ctx context.Context, namespace, name string) (*appsv1.StatefulSet, bool, error)) error { - // Check if the finalizer bypass annotation is present + // Preserve the existing deletion escape hatch, including presence-only semantics. annotations := c.GetAnnotations() - if annotations != nil { - if _, bypass := annotations["coherence.oracle.com/finalizer-bypass"]; bypass { - fm.Log.Info("Bypassing service suspension due to finalizer-bypass annotation", - "Namespace", c.Namespace, "Name", c.Name) - fm.EventRecorder.Eventf(c, nil, corev1.EventTypeNormal, "FinalizerBypassed", "Finalize", - "Service suspension bypassed due to finalizer-bypass annotation") - return nil - } + if _, bypass := annotations["coherence.oracle.com/finalizer-bypass"]; bypass { + fm.Log.Info("Bypassing service suspension due to finalizer-bypass annotation", + "Namespace", c.Namespace, "Name", c.Name) + fm.EventRecorder.Eventf(c, nil, corev1.EventTypeNormal, "FinalizerBypassed", "Finalize", + "Service suspension bypassed due to finalizer-bypass annotation") + return nil } // determine whether we can skip service suspension @@ -103,40 +100,36 @@ func (fm *FinalizerManager) FinalizeDeployment(ctx context.Context, c *coh.Coher // Get the StatefulSet sts, stsExists, err := findStatefulSet(ctx, c.Namespace, c.Name) if err != nil { - return errors.Wrapf(err, "getting StatefulSet %s/%s", c.Namespace, c.Name) + return fm.suspensionFailure(c, &probe.SuspensionCauseError{Kind: probe.SuspensionCauseWorkloadLookupFailed, Cause: err}) } if stsExists { if sts.Status.ReadyReplicas == 0 { fm.Log.Info("Skipping suspension of Coherence services in deployment " + c.Name + " - No Pods are ready") } else { - // Do service suspension... - p := probe.CoherenceProbe{ - Client: fm.Client, - EventRecorder: events.NewOwnedEventRecorder(c, fm.EventRecorder), - } - if p.SuspendServices(ctx, c, sts) == probe.ServiceSuspendFailed { - // Log the failure but don't return an error if we've already tried multiple times - // This prevents resources from being stuck in a deleting state indefinitely - errorCount := 1 - if annotations != nil { - if countStr, ok := annotations["coherence.oracle.com/error-count"]; ok { - if parsedCount, err := strconv.Atoi(countStr); err == nil { - errorCount = parsedCount - } - } - } - - if errorCount > 3 { + p := probe.CoherenceProbe{Client: fm.Client, EventRecorder: events.NewOwnedEventRecorder(c, fm.EventRecorder)} + _, err = p.SuspendServicesWithError(ctx, c, sts) + if err != nil { + // Retain the established recovery policy. This is a recorded general + // error count, not a count of suspension attempts made here. + errorCount, parseErr := strconv.Atoi(annotations["coherence.oracle.com/error-count"]) + if parseErr == nil && errorCount > 3 { fm.Log.Info("Service suspension failed multiple times, allowing deletion to proceed", "Namespace", c.Namespace, "Name", c.Name, "ErrorCount", errorCount) fm.EventRecorder.Eventf(c, nil, corev1.EventTypeWarning, "ServiceSuspensionFailed", "SuspendServices", "Service suspension failed multiple times, allowing deletion to proceed anyway") return nil } - - return fmt.Errorf("failed to suspend services") } + return err } } return nil } + +func (fm *FinalizerManager) suspensionFailure(c *coh.Coherence, cause error) error { + err := &probe.SuspensionError{Cause: cause} + if fm.EventRecorder != nil { + fm.EventRecorder.Eventf(c, nil, corev1.EventTypeWarning, "ServiceSuspendFailed", "SuspendServices", "%s", err.Error()) + } + return err +} diff --git a/controllers/finalizer/health_mutator_test.go b/controllers/finalizer/health_mutator_test.go new file mode 100644 index 000000000..7b7e0ca69 --- /dev/null +++ b/controllers/finalizer/health_mutator_test.go @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ +package finalizer + +import ( + "context" + "errors" + coh "github.com/oracle/coherence-operator/api/v1" + "github.com/oracle/coherence-operator/pkg/probe" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/events" + "k8s.io/utils/ptr" + "net" + "net/http" + "net/http/httptest" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "strconv" + "testing" +) + +func TestDesiredZeroPreservesHistoricalFinalizationSkip(t *testing.T) { + c := &coh.Coherence{ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "test"}} + c.Spec.Replicas = ptr.To(int32(0)) + manager := &FinalizerManager{EventRecorder: events.NewFakeRecorder(20)} + lookup := func(context.Context, string, string) (*appsv1.StatefulSet, bool, error) { + t.Fatal("desired-zero finalization unexpectedly inspected workload state") + return nil, false, nil + } + if err := manager.FinalizeDeployment(context.Background(), c, lookup); err != nil { + t.Fatalf("historical desired-zero skip failed: %v", err) + } +} + +func TestNoReadyPodsPreservesHistoricalFinalizationSkip(t *testing.T) { + c := &coh.Coherence{ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "test"}} + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "test"}, + Spec: appsv1.StatefulSetSpec{Replicas: ptr.To(int32(1))}, + Status: appsv1.StatefulSetStatus{Replicas: 1, ReadyReplicas: 0}, + } + manager := &FinalizerManager{EventRecorder: events.NewFakeRecorder(20)} + lookup := func(context.Context, string, string) (*appsv1.StatefulSet, bool, error) { return sts, true, nil } + if err := manager.FinalizeDeployment(context.Background(), c, lookup); err != nil { + t.Fatalf("historical no-ready-Pods skip failed: %v", err) + } +} + +func TestDeletionBypassAnnotationPreservesPresenceSemantics(t *testing.T) { + for _, value := range []string{"true", "false", ""} { + t.Run("value="+value, func(t *testing.T) { + c := &coh.Coherence{ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "test", Annotations: map[string]string{"coherence.oracle.com/finalizer-bypass": value}}} + manager := &FinalizerManager{EventRecorder: events.NewFakeRecorder(10)} + lookup := func(context.Context, string, string) (*appsv1.StatefulSet, bool, error) { + t.Fatal("existing bypass should skip workload lookup and suspension") + return nil, false, nil + } + if err := manager.FinalizeDeployment(context.Background(), c, lookup); err != nil { + t.Fatalf("existing bypass was rejected: %v", err) + } + }) + } +} + +func TestDeletionErrorCountPolicyAfterRejectedSuspension(t *testing.T) { + for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden} { + for _, count := range []string{"", "invalid", "999999999999999999999999", "3", "4", "99"} { + t.Run(strconv.Itoa(status)+"/count="+count, func(t *testing.T) { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if r.Method != http.MethodPut || r.URL.Path != "/suspend" { + t.Errorf("unexpected suspension request: %s %s", r.Method, r.URL.Path) + } + w.WriteHeader(status) + })) + defer server.Close() + host, portText, err := net.SplitHostPort(server.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + port, err := strconv.Atoi(portText) + if err != nil { + t.Fatal(err) + } + c := &coh.Coherence{ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "test", Annotations: map[string]string{"coherence.oracle.com/error-count": count}}} + c.Spec.Replicas = ptr.To(int32(1)) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "test-0", Namespace: "test", Labels: map[string]string{"app": "test"}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: coh.ContainerNameCoherence, Ports: []corev1.ContainerPort{{Name: coh.PortNameHealth, ContainerPort: int32(port)}}}}}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: host, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}}, + } + sts := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "test"}, Spec: appsv1.StatefulSetSpec{Replicas: ptr.To(int32(1)), Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "test"}}}, Status: appsv1.StatefulSetStatus{ReadyReplicas: 1, Replicas: 1}} + manager := &FinalizerManager{Client: fake.NewClientBuilder().WithObjects(pod).Build(), EventRecorder: events.NewFakeRecorder(20)} + lookup := func(context.Context, string, string) (*appsv1.StatefulSet, bool, error) { return sts, true, nil } + err = manager.FinalizeDeployment(context.Background(), c, lookup) + if count == "4" || count == "99" { + if err != nil { + t.Fatalf("existing error-count policy should permit deletion: %v", err) + } + } else { + var requestErr *probe.RequestError + if !errors.As(err, &requestErr) || requestErr.StatusCode != status { + t.Fatalf("expected preserved HTTP %d failure, got %v", status, err) + } + } + if requests != 1 { + t.Fatalf("expected one PUT and no fallback, got %d requests", requests) + } + }) + } + } +} diff --git a/controllers/job/job_controller.go b/controllers/job/job_controller.go index 08011c3db..252d5c3bb 100644 --- a/controllers/job/job_controller.go +++ b/controllers/job/job_controller.go @@ -15,6 +15,7 @@ import ( coh "github.com/oracle/coherence-operator/api/v1" "github.com/oracle/coherence-operator/controllers/reconciler" "github.com/oracle/coherence-operator/pkg/clients" + cohevents "github.com/oracle/coherence-operator/pkg/events" "github.com/oracle/coherence-operator/pkg/patching" "github.com/oracle/coherence-operator/pkg/probe" "github.com/oracle/coherence-operator/pkg/utils" @@ -414,8 +415,9 @@ func (in *ReconcileJob) maybeExecuteProbe(ctx context.Context, job *batchv1.Job, } p := probe.CoherenceProbe{ - Client: in.GetClient(), - Config: in.GetManager().GetConfig(), + Client: in.GetClient(), + Config: in.GetManager().GetConfig(), + EventRecorder: cohevents.NewOwnedEventRecorder(deployment, in.GetEventRecorder()), } status := deployment.GetStatus() @@ -424,10 +426,14 @@ func (in *ReconcileJob) maybeExecuteProbe(ctx context.Context, job *batchv1.Job, probeStatus := status.FindJobProbeStatus(name) podCondition := in.findPodReadyCondition(pod) if in.shouldExecuteProbe(probeStatus, podCondition) { - _, err := p.RunProbe(ctx, pod, deployment.GetWkaServiceName(), &action.Probe) + success, err := p.RunProbe(ctx, pod, deployment.GetWkaServiceName(), &action.Probe) + if err == nil && !success { + err = fmt.Errorf("probe returned an unsuccessful result") + } if err == nil { logger.Info(fmt.Sprintf("Executed probe using pod %s", name), "Error", "nil") probeStatus.Success = ptr.To(true) + probeStatus.Error = nil } else { logger.Info(fmt.Sprintf("Executed probe using pod %s", name), "Error", err) probeStatus.Success = ptr.To(false) @@ -456,7 +462,7 @@ func (in *ReconcileJob) shouldExecuteProbe(probeStatus coh.CoherenceJobProbeStat if podCondition == nil || podCondition.Status != corev1.ConditionTrue { return false } - if podCondition.LastTransitionTime.Before(probeStatus.LastReadyTime) { + if probeStatus.LastReadyTime != nil && !podCondition.LastTransitionTime.After(probeStatus.LastReadyTime.Time) { return false } return true diff --git a/controllers/job/job_controller_test.go b/controllers/job/job_controller_test.go new file mode 100644 index 000000000..8569acd56 --- /dev/null +++ b/controllers/job/job_controller_test.go @@ -0,0 +1,234 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ + +package job + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "testing" + "time" + + "github.com/go-logr/logr" + coh "github.com/oracle/coherence-operator/api/v1" + "github.com/oracle/coherence-operator/pkg/clients" + "github.com/oracle/coherence-operator/pkg/operator" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/rest" + k8sevents "k8s.io/client-go/tools/events" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/recorder" +) + +type probeTestManager struct { + manager.Manager + client client.Client + recorder recorder.EventRecorder +} + +func (m *probeTestManager) GetClient() client.Client { + return m.client +} + +func (m *probeTestManager) GetConfig() *rest.Config { + return &rest.Config{} +} + +func (m *probeTestManager) GetEventRecorder(string) recorder.EventRecorder { + return m.recorder +} + +func TestMaybeExecuteProbeRecordsHTTPResult(t *testing.T) { + for _, tc := range []struct { + name string + statusCode int + success bool + errorText string + }{ + {name: "success", statusCode: http.StatusOK, success: true}, + {name: "failure", statusCode: http.StatusServiceUnavailable, errorText: "HTTP probe rejected: status 503"}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(tc.statusCode) + })) + defer server.Close() + + host, port := serverAddress(t, server.URL) + reconciler, job, deployment, _ := readyActionTestObjects(t, host, coh.Probe{ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{Path: "/test", Port: intstr.FromInt(port)}, + }}) + + statuses, err := reconciler.maybeExecuteProbe(context.Background(), job, deployment, logr.Discard()) + if err != nil { + t.Fatal(err) + } + if len(statuses) != 1 || statuses[0].Success == nil || *statuses[0].Success != tc.success { + t.Fatalf("statuses=%+v", statuses) + } + if tc.errorText == "" { + if statuses[0].Error != nil { + t.Fatalf("unexpected error: %s", *statuses[0].Error) + } + } else if statuses[0].Error == nil || *statuses[0].Error != tc.errorText { + t.Fatalf("error=%v, want %q", statuses[0].Error, tc.errorText) + } + }) + } +} + +func TestMaybeExecuteProbeRecordsTCPFailure(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + host, value, err := net.SplitHostPort(listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + port, err := strconv.Atoi(value) + if err != nil { + t.Fatal(err) + } + if err = listener.Close(); err != nil { + t.Fatal(err) + } + + reconciler, job, deployment, _ := readyActionTestObjects(t, host, coh.Probe{ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromInt(port)}, + }}) + statuses, err := reconciler.maybeExecuteProbe(context.Background(), job, deployment, logr.Discard()) + if err != nil { + t.Fatal(err) + } + if len(statuses) != 1 || statuses[0].Success == nil || *statuses[0].Success { + t.Fatalf("statuses=%+v", statuses) + } + const expected = "probe returned an unsuccessful result" + if statuses[0].Error == nil || *statuses[0].Error != expected { + t.Fatalf("error=%v, want %q", statuses[0].Error, expected) + } +} + +func TestMaybeExecuteLegacyNoOpProbeWarns(t *testing.T) { + for _, tt := range []struct { + name string + probe coh.Probe + reason string + }{ + {name: "empty handler", probe: coh.Probe{}, reason: "the probe handler is empty"}, + {name: "gRPC handler", probe: coh.Probe{ProbeHandler: corev1.ProbeHandler{GRPC: &corev1.GRPCAction{Port: 1408}}}, reason: "gRPC probe actions are not supported"}, + } { + t.Run(tt.name, func(t *testing.T) { + reconciler, job, deployment, eventRecorder := readyActionTestObjects(t, "127.0.0.1", tt.probe) + + statuses, err := reconciler.maybeExecuteProbe(context.Background(), job, deployment, logr.Discard()) + if err != nil { + t.Fatal(err) + } + if len(statuses) != 1 || statuses[0].Success == nil || !*statuses[0].Success || statuses[0].Error != nil { + t.Fatalf("statuses=%+v", statuses) + } + + select { + case event := <-eventRecorder.Events: + if !strings.Contains(event, "Warning ProbeActionNoOp") || !strings.Contains(event, tt.reason) { + t.Fatalf("event=%q, want ProbeActionNoOp containing %q", event, tt.reason) + } + default: + t.Fatal("expected ProbeActionNoOp warning event") + } + }) + } +} + +func TestShouldExecuteProbeOncePerReadyTransition(t *testing.T) { + reconciler := &ReconcileJob{} + readyTime := metav1.NewTime(time.Now()) + condition := &corev1.PodCondition{Type: corev1.PodReady, Status: corev1.ConditionTrue, LastTransitionTime: readyTime} + + if !reconciler.shouldExecuteProbe(coh.CoherenceJobProbeStatus{}, condition) { + t.Fatal("first probe attempt was skipped") + } + if reconciler.shouldExecuteProbe(coh.CoherenceJobProbeStatus{LastReadyTime: &readyTime, Success: ptr.To(false)}, condition) { + t.Fatal("failed probe was repeated without a new ready transition") + } + if reconciler.shouldExecuteProbe(coh.CoherenceJobProbeStatus{LastReadyTime: &readyTime, Success: ptr.To(true)}, condition) { + t.Fatal("successful probe was repeated without a new ready transition") + } + + later := metav1.NewTime(readyTime.Add(time.Second)) + condition.LastTransitionTime = later + if !reconciler.shouldExecuteProbe(coh.CoherenceJobProbeStatus{LastReadyTime: &readyTime, Success: ptr.To(true)}, condition) { + t.Fatal("probe was not executed after a new ready transition") + } +} + +func readyActionTestObjects(t *testing.T, host string, action coh.Probe) (*ReconcileJob, *batchv1.Job, *coh.CoherenceJob, *k8sevents.FakeRecorder) { + t.Helper() + selector := map[string]string{"job": "ready-action"} + readyTime := metav1.NewTime(time.Now()) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ready-action-0", + Namespace: "test", + Labels: map[string]string{ + "job": selector["job"], + operator.LabelTestHostName: host, + }, + }, + Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{ + Type: corev1.PodReady, Status: corev1.ConditionTrue, LastTransitionTime: readyTime, + }}}, + } + eventRecorder := k8sevents.NewFakeRecorder(10) + manager := &probeTestManager{client: fake.NewClientBuilder().WithObjects(pod).Build(), recorder: eventRecorder} + reconciler := &ReconcileJob{} + reconciler.SetCommonReconciler("test.Job", manager, clients.ClientSet{}) + + deployment := &coh.CoherenceJob{ + ObjectMeta: metav1.ObjectMeta{Name: "ready-action", Namespace: "test"}, + Spec: coh.CoherenceJobResourceSpec{ + CoherenceResourceSpec: coh.CoherenceResourceSpec{Replicas: ptr.To(int32(1))}, + Cluster: "test", + ReadyAction: &coh.CoherenceJobProbe{Probe: action}, + }, + } + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: "ready-action", Namespace: "test"}, + Spec: batchv1.JobSpec{Selector: &metav1.LabelSelector{MatchLabels: selector}}, + Status: batchv1.JobStatus{Ready: ptr.To(int32(1))}, + } + return reconciler, job, deployment, eventRecorder +} + +func serverAddress(t *testing.T, rawURL string) (string, int) { + t.Helper() + u, err := url.Parse(rawURL) + if err != nil { + t.Fatal(err) + } + host, value, err := net.SplitHostPort(u.Host) + if err != nil { + t.Fatal(err) + } + port, err := strconv.Atoi(value) + if err != nil { + t.Fatal(err) + } + return host, port +} diff --git a/controllers/statefulset/statefulset_controller.go b/controllers/statefulset/statefulset_controller.go index 10e3cbbe1..7c6aec0c6 100644 --- a/controllers/statefulset/statefulset_controller.go +++ b/controllers/statefulset/statefulset_controller.go @@ -168,12 +168,12 @@ func (in *ReconcileStatefulSet) ReconcileAllResourceOfKind(ctx context.Context, in.GetEventRecorder().Eventf(deployment, nil, corev1.EventTypeNormal, reconciler.EventReasonScaling, "SuspendServices", "suspending Coherence services in statefuleset %s", request.Name) // we are scaling down to zero and suspend services flag is true, so suspend services - suspended := in.suspendServices(ctx, deployment, stsCurrent) + suspended, suspendErr := in.suspendServices(ctx, deployment, stsCurrent) switch suspended { case probe.ServiceSuspendFailed: in.GetEventRecorder().Eventf(deployment, nil, corev1.EventTypeWarning, reconciler.EventReasonScaling, "SuspendServices", - "failed suspending Coherence services in statefuleset %s", request.Name) - return reconcile.Result{RequeueAfter: time.Minute}, fmt.Errorf("failed to suspend services prior to scaling down to zero") + "failed suspending Coherence services in statefuleset %s: %s", request.Name, suspendErr.Error()) + return reconcile.Result{RequeueAfter: time.Minute}, suspendErr case probe.ServiceSuspendSkipped: logger.Info("skipping suspension of Coherence services prior to deletion of StatefulSet") in.GetEventRecorder().Eventf(deployment, nil, corev1.EventTypeNormal, reconciler.EventReasonScaling, "SuspendServices", @@ -223,8 +223,9 @@ func (in *ReconcileStatefulSet) execActions(ctx context.Context, sts *appsv1.Sta spec, found := deployment.GetStatefulSetSpec() if found { coherenceProbe := probe.CoherenceProbe{ - Client: in.GetClient(), - Config: in.GetManager().GetConfig(), + Client: in.GetClient(), + Config: in.GetManager().GetConfig(), + EventRecorder: events.NewOwnedEventRecorder(deployment, in.GetEventRecorder()), } for _, action := range spec.Actions { @@ -553,19 +554,6 @@ func (in *ReconcileStatefulSet) maybePatchStatefulSet(ctx context.Context, deplo logger.V(0).Info("WARNING - Updating StatefulSet without a StatusHA test, update was forced") } - // if there is only a single replica we need to do service suspension before update - if current.Status.ReadyReplicas == 1 { - suspended := in.suspendServices(ctx, deployment, current) - switch suspended { - case probe.ServiceSuspendFailed: - return reconcile.Result{RequeueAfter: time.Minute}, fmt.Errorf("failed to suspend services prior to updating single member deployment") - case probe.ServiceSuspendSkipped: - logger.Info("Skipping suspension of Coherence services in single member deployment " + deployment.GetName() + - " prior to update StatefulSet") - case probe.ServiceSuspendSuccessful: - } - } - // now apply the patch patched, err := in.ApplyThreeWayPatchWithCallback(ctx, current.GetName(), current, patch, data, callback) @@ -582,13 +570,13 @@ func (in *ReconcileStatefulSet) maybePatchStatefulSet(ctx context.Context, deplo } // suspendServices suspends Coherence services in the target deployment. -func (in *ReconcileStatefulSet) suspendServices(ctx context.Context, deployment coh.CoherenceResource, current *appsv1.StatefulSet) probe.ServiceSuspendStatus { +func (in *ReconcileStatefulSet) suspendServices(ctx context.Context, deployment coh.CoherenceResource, current *appsv1.StatefulSet) (probe.ServiceSuspendStatus, error) { p := probe.CoherenceProbe{ Client: in.GetClient(), Config: in.GetManager().GetConfig(), EventRecorder: events.NewOwnedEventRecorder(deployment, in.GetEventRecorder()), } - return p.SuspendServices(ctx, deployment, current) + return p.SuspendServicesWithError(ctx, deployment, current) } // Scale will scale a StatefulSet up or down diff --git a/controllers/statefulset/upgrade_strategy.go b/controllers/statefulset/upgrade_strategy.go index 6067e43ff..86447dc32 100644 --- a/controllers/statefulset/upgrade_strategy.go +++ b/controllers/statefulset/upgrade_strategy.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2025, Oracle and/or its affiliates. + * Copyright (c) 2020, 2026, Oracle and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl. */ @@ -40,14 +40,14 @@ func GetUpgradeStrategy(c coh.CoherenceResource, p probe.CoherenceProbe) Upgrade return ManualUpgradeStrategy{} } if name == coh.UpgradeByNode { - sp := spec.GetScalingProbe() + sp := probe.ResolveScalingProbe(spec) return ByNodeUpgradeStrategy{ cp: p, scalingProbe: sp, } } if name == coh.UpgradeByNodeLabel { - sp := spec.GetScalingProbe() + sp := probe.ResolveScalingProbe(spec) if spec.RollingUpdateLabel == nil { return ByNodeUpgradeStrategy{ cp: p, @@ -102,7 +102,7 @@ var _ UpgradeStrategy = ByNodeUpgradeStrategy{} type ByNodeUpgradeStrategy struct { cp probe.CoherenceProbe - scalingProbe *coh.Probe + scalingProbe probe.ResolvedScalingProbe } func (in ByNodeUpgradeStrategy) RollingUpgrade(ctx context.Context, sts *appsv1.StatefulSet, svc string, c kubernetes.Interface) (reconcile.Result, error) { @@ -119,7 +119,7 @@ var _ UpgradeStrategy = ByNodeLabelUpgradeStrategy{} type ByNodeLabelUpgradeStrategy struct { cp probe.CoherenceProbe - scalingProbe *coh.Probe + scalingProbe probe.ResolvedScalingProbe label string } @@ -177,7 +177,7 @@ func (p *PodNodeLabel) GetNodeId(ctx context.Context, c kubernetes.Interface, po // ----- helper methods ---------------------------------------------------------------------------- -func rollingUpgrade(cp probe.CoherenceProbe, scalingProbe *coh.Probe, fn PodNodeIdSupplier, idName string, ctx context.Context, sts *appsv1.StatefulSet, svc string, c kubernetes.Interface) (reconcile.Result, error) { +func rollingUpgrade(cp probe.CoherenceProbe, scalingProbe probe.ResolvedScalingProbe, fn PodNodeIdSupplier, idName string, ctx context.Context, sts *appsv1.StatefulSet, svc string, c kubernetes.Interface) (reconcile.Result, error) { var err error var replicas int32 @@ -270,7 +270,7 @@ func rollingUpgrade(cp probe.CoherenceProbe, scalingProbe *coh.Probe, fn PodNode // We have Pods to be upgraded nodeId, _ := fn.GetNodeId(ctx, c, pods.Items[0]) // Check Pods are "safe" - if cp.ExecuteProbeForSubSetOfPods(ctx, sts, svc, scalingProbe, pods, podsToUpdate) { + if cp.ExecuteScalingProbeForSubSetOfPods(ctx, sts, svc, scalingProbe, pods, podsToUpdate) { // delete the Pods log.Info("Upgrading all Pods for Node identifier", "Namespace", sts.Namespace, "Name", sts.Name, "NodeId", idName, "IdValue", nodeId, "Count", len(podsToUpdate.Items)) err = deletePods(ctx, podsToUpdate, c) diff --git a/controllers/statefulset/upgrade_strategy_provenance_test.go b/controllers/statefulset/upgrade_strategy_provenance_test.go new file mode 100644 index 000000000..331a49a7c --- /dev/null +++ b/controllers/statefulset/upgrade_strategy_provenance_test.go @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ + +package statefulset + +import ( + "testing" + + coh "github.com/oracle/coherence-operator/api/v1" + "github.com/oracle/coherence-operator/pkg/probe" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" +) + +func TestUpgradeStrategiesPreserveScalingProbeSource(t *testing.T) { + custom := &coh.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{ + Path: "/ha", + Port: intstr.FromString(coh.PortNameHealth), + Scheme: corev1.URISchemeHTTP, + }}} + + for _, tc := range []struct { + name string + strategy coh.RollingUpdateStrategyType + generated bool + custom *coh.Probe + }{ + {name: "by-node generated", strategy: coh.UpgradeByNode, generated: true}, + {name: "by-node custom", strategy: coh.UpgradeByNode, custom: custom}, + {name: "by-node-label generated", strategy: coh.UpgradeByNodeLabel, generated: true}, + {name: "by-node-label custom", strategy: coh.UpgradeByNodeLabel, custom: custom}, + } { + t.Run(tc.name, func(t *testing.T) { + spec := coh.CoherenceStatefulSetResourceSpec{ + RollingUpdateStrategy: ptr.To(tc.strategy), + RollingUpdateLabel: ptr.To("topology.kubernetes.io/zone"), + } + if tc.custom != nil { + spec.Scaling = &coh.ScalingSpec{Probe: tc.custom} + } + deployment := &coh.Coherence{Spec: spec} + strategy := GetUpgradeStrategy(deployment, probe.CoherenceProbe{}) + + var resolved probe.ResolvedScalingProbe + switch actual := strategy.(type) { + case ByNodeUpgradeStrategy: + resolved = actual.scalingProbe + case ByNodeLabelUpgradeStrategy: + resolved = actual.scalingProbe + default: + t.Fatalf("unexpected strategy type %T", strategy) + } + + if resolved.Generated != tc.generated { + t.Fatalf("generated = %t, want %t", resolved.Generated, tc.generated) + } + if tc.custom != nil && resolved.Probe != tc.custom { + t.Fatal("custom scaling probe identity was not preserved") + } + if resolved.Probe == nil { + t.Fatal("resolved scaling probe is nil") + } + }) + } +} diff --git a/docs/about/04_coherence_spec.adoc b/docs/about/04_coherence_spec.adoc index ea29277b7..8449564d2 100644 --- a/docs/about/04_coherence_spec.adoc +++ b/docs/about/04_coherence_spec.adoc @@ -72,6 +72,13 @@ m| status |   m| <> | fals * <> * <> * <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> * <> * <> * <> @@ -235,7 +242,7 @@ annotations: + foo.io/two: "value2" + + + see: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/[Kubernetes Annotations] m| map[string]string | false -m| readyAction | ReadyAction is a probe that will be executed when one or more Pods reach the ready state. The probe will be executed on every Pod that is ready. One the required number of ready Pods is reached the probe will also be executed on every Pod that becomes ready after that time. m| *<> | false +m| readyAction | ReadyAction is attempted for each Ready Pod after ReadyCount is reached. The Operator does not automatically make another attempt while a Pod remains Ready. If a Pod stops being Ready and subsequently becomes Ready again, another attempt is made. The most recently recorded result is available in status.jobProbes. m| *<> | false m| forceExit | ForceExit is a flag to indicate whether the Operator should call System.exit to forcefully exit the process when the configured main class completes execution. m| *bool | false m| envFrom | List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. m| []https://{k8s-doc-link}/#envfromsource-v1-core[corev1.EnvFromSource] | false m| global | Global contains attributes that will be applied to all resources managed by the Coherence Operator. m| *<> | false @@ -439,6 +446,7 @@ CoherenceStatefulSetResourceSpec defines the specification of a Coherence resour [cols="1,10,1,1"options="header"] |=== | Field | Description | Type | Required +m| healthMutatorConnection | HealthMutatorConnection configures the selected health server and suspension client. m| *<> | false m| cluster | The optional name of the Coherence cluster that this Coherence resource belongs to. If this value is set the Pods controlled by this Coherence resource will form a cluster with other Pods controlled by Coherence resources with the same cluster name. If not set the Coherence resource's name will be used as the cluster name. m| *string | false m| statefulSetAnnotations | StatefulSetAnnotations are free-form yaml that will be added to the Coherence cluster `StatefulSet` as annotations. Any annotations should be placed BELOW this "annotations:" key, for example: + + @@ -458,7 +466,7 @@ m| autoResumeServices | AutoResumeServices is a map of Coherence service names t m| suspendServiceTimeout | SuspendServiceTimeout sets the number of seconds to wait for the service suspend call to return (the default is 60 seconds) m| *int | false m| haBeforeUpdate | Whether to perform a StatusHA test on the cluster before performing an update or deletion. This field can be set to "false" to force through an update even when a Coherence deployment is in an unstable state. The default is true, to always check for StatusHA before updating a Coherence deployment. m| *bool | false m| allowUnsafeDelete | AllowUnsafeDelete controls whether the Operator will add a finalizer to the Coherence resource so that it can intercept deletion of the resource and initiate a controlled shutdown of the Coherence cluster. The default value is `false`. The primary use for setting this flag to `true` is in CI/CD environments so that cleanup jobs can delete a whole namespace without requiring the Operator to have removed finalizers from any Coherence resources deployed into that namespace. It is not recommended to set this flag to `true` in a production environment, especially when using Coherence persistence features. m| *bool | false -m| actions | Actions to execute once all the Pods are ready after an initial deployment m| []<> | false +m| actions | Actions to execute once all the Pods are ready after an initial deployment. A maximum of 128 actions may be configured. m| []<> | false m| envFrom | List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. m| []https://{k8s-doc-link}/#envfromsource-v1-core[corev1.EnvFromSource] | false m| global | Global contains attributes that will be applied to all resources managed by the Coherence Operator. m| *<> | false m| initResources | InitResources is the optional resource requests and limits for the init-container that the Operator adds to the Pod. + @@ -558,6 +566,107 @@ m| annotations | Annotations is an unstructured key value map stored with a reso <> +=== HTTPAction + +HTTPAction is an Operator HTTP action. Only HTTP 200 is successful; redirects are rejected. + +[cols="1,10,1,1"options="header"] +|=== +| Field | Description | Type | Required +m| path | Path is the URL path to request. It must be absolute and defaults to "/". m| string | false +m| port | Port is the numeric or named port on the target Pod. m| https://pkg.go.dev/k8s.io/apimachinery/pkg/util/intstr#IntOrString | true +m| host | Host is the optional connection host. It defaults to the target Pod's address. m| string | false +m| scheme | Scheme is the protocol used for the request and defaults to HTTP. m| https://{k8s-doc-link}/#urischeme-v1-core[corev1.URIScheme] | false +m| httpHeaders | HTTPHeaders contains non-secret custom request headers. m| []<> | false +m| method | Method defaults to GET. m| string | false +m| basicAuth | BasicAuth references credentials in the resource namespace. It can only be used with the target Pod address, so Host must be empty when BasicAuth is configured. m| *<> | false +m| tls | TLS configures verified server authentication. m| *<> | false +|=== + +<
> + +=== HTTPActionHeader + +HTTPActionHeader is a bounded non-secret HTTP request header. + +[cols="1,10,1,1"options="header"] +|=== +| Field | Description | Type | Required +m| name | Name is the HTTP header name. m| string | true +m| value | Value is the HTTP header value. m| string | true +|=== + +<
> + +=== HTTPBasicAuth + +HTTPBasicAuth contains namespace-local credential references. + +[cols="1,10,1,1"options="header"] +|=== +| Field | Description | Type | Required +m| username | Username references the Secret key containing the Basic authentication username. m| https://{k8s-doc-link}/#secretkeyselector-v1-core[corev1.SecretKeySelector] | true +m| password | Password references the Secret key containing the Basic authentication password. m| https://{k8s-doc-link}/#secretkeyselector-v1-core[corev1.SecretKeySelector] | true +|=== + +<
> + +=== HTTPClientTLS + +HTTPClientTLS supplies a PEM trust bundle and optional verified DNS name. + +[cols="1,10,1,1"options="header"] +|=== +| Field | Description | Type | Required +m| caSecret | CASecret references the Secret key containing the PEM-encoded CA certificate bundle. m| https://{k8s-doc-link}/#secretkeyselector-v1-core[corev1.SecretKeySelector] | true +m| serverName | ServerName is the optional DNS name used to verify the server certificate. m| string | false +|=== + +<
> + +=== HTTPEndpoint + +HTTPEndpoint identifies a target without Kubernetes GET-only semantics. + +[cols="1,10,1,1"options="header"] +|=== +| Field | Description | Type | Required +m| path | Path is the URL path to request. It must be absolute and defaults to "/". m| string | false +m| port | Port is the numeric or named port on the target Pod. m| https://pkg.go.dev/k8s.io/apimachinery/pkg/util/intstr#IntOrString | true +m| host | Host is the optional connection host. It defaults to the target Pod's address. m| string | false +m| scheme | Scheme is the protocol used for the request and defaults to HTTP. m| https://{k8s-doc-link}/#urischeme-v1-core[corev1.URIScheme] | false +|=== + +<
> + +=== HealthJAAS + +HealthJAAS selects the packaged validator or an application-owned JAAS configuration. + +[cols="1,10,1,1"options="header"] +|=== +| Field | Description | Type | Required +m| mode | Mode is "packaged" to generate the packaged JAAS configuration or "existing" to use the application's configuration. It defaults to "packaged". m| string | false +|=== + +<
> + +=== HealthMutatorConnection + +HealthMutatorConnection configures authenticated health operations for StatefulSets. + +[cols="1,10,1,1"options="header"] +|=== +| Field | Description | Type | Required +m| secure | Secure enables authenticated PUT over verified TLS. m| bool | true +m| basicAuth | BasicAuth supplies credentials for authenticated health requests. m| *<> | false +m| tls | TLS configures CA trust and server name verification for health requests. m| *<> | false +m| serverTLS | ServerTLS configures the member's one-way TLS identity. TrustStore fields are accepted for compatibility but are not used by HealthSSLProvider. m| *<> | false +m| jaas | JAAS selects how the member's CoherenceREST login context is configured. m| *<> | false +|=== + +<
> + === ImageSpec ImageSpec defines the settings for a Docker image @@ -840,11 +949,12 @@ m| ssl | SSL configures SSL settings for a Coherence component m| *<> | false m| timeoutSeconds | Number of seconds after which the handler times out (only applies to http and tcp handlers). Defaults to 1 second. Minimum value is 1. m| *int | false |=== diff --git a/docs/applications/080_entrypoint.adoc b/docs/applications/080_entrypoint.adoc index 2c3690b8e..2456769e8 100644 --- a/docs/applications/080_entrypoint.adoc +++ b/docs/applications/080_entrypoint.adoc @@ -1,6 +1,6 @@ /////////////////////////////////////////////////////////////////////////////// - Copyright (c) 2025, Oracle and/or its affiliates. + Copyright (c) 2025, 2026, Oracle and/or its affiliates. Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. @@ -165,12 +165,16 @@ spec: alternateJdkJavaOptions: "ALT_JAVA_OPTS" ---- -In the Coherence container the application code can then access the The `ALT_JAVA_OPTS` environment variable -to obtain the JVM options the Operator configured. +In the Coherence container the application code can then access the `ALT_JAVA_OPTS` environment variable +to obtain the JVM options the Operator configured. The image entry point is responsible for passing the value of this +environment variable, including the generated Java argument-file reference, to the actual Coherence member JVM. + +This is required when Operator-managed protected Health endpoints are enabled with +`spec.healthMutatorConnection.secure: true`. In that mode, `useJdkJavaOptions: false` is accepted only when a non-empty +`alternateJdkJavaOptions` is configured as the explicit JVM-argument delivery contract. ==== Use Java Argument Files Directly In use cases where the `JDK_JAVA_OPTIONS` environment variable has been disabled application code could also directly access the Java argument files the operator configured and use those to configure the Coherence JVM. - diff --git a/docs/health/010_protected_health_endpoints.adoc b/docs/health/010_protected_health_endpoints.adoc new file mode 100644 index 000000000..645808afa --- /dev/null +++ b/docs/health/010_protected_health_endpoints.adoc @@ -0,0 +1,390 @@ +/////////////////////////////////////////////////////////////////////////////// + + Copyright (c) 2026, Oracle and/or its affiliates. + Licensed under the Universal Permissive License v 1.0 as shown at + http://oss.oracle.com/licenses/upl. + +/////////////////////////////////////////////////////////////////////////////// + += Protected Coherence Health Endpoints +:description: Coherence Operator Documentation - Protected Coherence Health Endpoints +:keywords: oracle coherence, kubernetes, operator, health, basic authentication, TLS, scale to zero + +Recent Coherence releases protect the state-changing Health endpoints used to suspend and resume cache services. +Coherence Operator 3.6.0 is the first Operator release that can configure and call these protected endpoints during +safe shutdown and scale-to-zero. + +The recommended configuration uses `spec.healthMutatorConnection` to configure both sides of the connection: + +* the Coherence member exposes `PUT /suspend` using Basic authentication over TLS; and +* the Operator reads credentials and a CA certificate from Kubernetes Secrets and sends an authenticated, verified + HTTPS request. + +Authentication or TLS failure blocks the transition. The Operator does not retry anonymously, use `GET`, or disable +certificate verification. + +== Hardened Health mutator behavior + +The hardened Coherence Health server changes the mutator endpoints as follows: + +* `/suspend` and `/resume` accept `PUT` only. The old `GET` routes are not available. +* Mutators are disabled by default. A request returns `403` until + `coherence.health.http.mutators.enabled=true` is configured. +* An enabled mutator must use `basic`, `cert`, or `cert+basic` authentication. There is no unauthenticated mode. +* Read-only endpoints such as `/ready`, `/started`, `/live`, `/healthz`, `/safe`, and `/ha` remain available for + Kubernetes and Operator health checks without mutator credentials. + +An older Operator sends an unauthenticated `GET /suspend`, so it cannot safely scale a deployment using a hardened +Coherence Health server to zero. Operator 3.6.0 sends `PUT` and adds the managed and application-owned configuration +options described on this page. + +== Compatible versions + +The following are the first releases on each maintained Coherence line that contain the Health mutator hardening. Later +patches on the same line also contain it. + +[cols="2,2",options="header"] +|=== +|Coherence line |First hardened release + +|Oracle Coherence 14.1.1.2206 +|14.1.1.2206.18 + +|Oracle Coherence 14.1.2 +|14.1.2.0.8 + +|Oracle Coherence 15.1.1 +|15.1.1.0.4 + +|Coherence Community Edition 14.1.2 +|14.1.2-0-8 + +|Coherence Community Edition 15.1.1 +|15.1.1-0-4 + +|Coherence Community Edition 22.06 +|22.06.18 + +|Coherence Community Edition feature releases +|26.07 +|=== + +Use Coherence Operator 3.6.0 or later with a hardened Coherence release. The Operator image and CRD must be upgraded +together so that the `healthMutatorConnection` and method-aware `suspendProbe.http` fields are both available. + +== Choose a configuration model + +[cols="2,3,3",options="header"] +|=== +|Model |Use when |Configuration owner + +|Operator-managed Basic authentication and one-way TLS (recommended) +|The application can use the Coherence Health server and Basic authentication. +|The Operator configures the Coherence listener, TLS identity, JAAS, kubelet probes, and Operator client from +`healthMutatorConnection`. + +|Operator-managed security with an existing JAAS file +|The application already owns the JVM's global JAAS configuration. +|The application provides the `CoherenceREST` JAAS entry; the Operator configures the remaining server and client +settings. + +|Operator-managed security with application-mounted TLS files +|The application already mounts the member keystore and password files. +|The application owns the volume; the Operator uses explicit file paths and configures the remaining settings. + +|Application-owned server with an Operator HTTP action +|The application already configures a Basic-authenticated HTTPS Health server. +|The application configures Coherence, TLS, JAAS, and kubelet probes. The Operator configures only the authenticated +`PUT` request. + +|Application-owned server with an exec action +|The server uses client certificates, `cert+basic`, or another client flow that the Operator HTTP action cannot express. +|The application image supplies and owns the client command and all server configuration. + +|No managed security +|The deployment uses an older, unhardened Health endpoint or a completely application-owned endpoint. +|Omit `healthMutatorConnection`. The application is responsible for any endpoint configuration and protection. +|=== + +For complete manifests and application instructions for each model, see the +<>. + +== Operator-managed Basic authentication over one-way TLS + +Set `spec.healthMutatorConnection.secure` to `true` to enable managed security. The Operator then: + +* enables Coherence Health mutators; +* selects Basic authentication and the `CoherenceREST` JAAS context; +* configures the member's `HealthSSLProvider` and server keystore; +* reads the Basic username, password, and client CA from namespace-local Secrets; +* sends `PUT /suspend` over verified HTTPS; and +* changes generated readiness, liveness, and startup probes on the Health port to HTTPS. + +Managed mode uses one-way TLS. The member presents a certificate to the Operator and kubelet, but does not request a +client certificate. The member therefore needs a keystore, but no truststore. The separate `tls.caSecret` is the CA +used by the Operator to verify the member certificate. + +=== Required Secrets + +Create the following Secrets in the same namespace as the `Coherence` resource. Use your normal credential and +certificate management process; do not put credentials or private keys directly in the `Coherence` resource. + +[cols="2,2,3",options="header"] +|=== +|Secret |Required keys |Purpose + +|`health-auth-v1` +|`username`, `password` +|Credentials used by the Operator and the `CoherenceREST` JAAS login module. + +|`health-stores-v1` +|`server.jks`, `store-password` +|Member certificate and password files. The key and store passwords may use different keys when required. + +|`health-ca-v1` +|`ca.pem` +|PEM CA bundle used by the Operator to verify the member certificate. +|=== + +Secret key values are consumed as exact bytes. Avoid an unintended trailing newline in usernames and passwords. Prefer +immutable, versioned Secrets so that a resource update clearly selects a new credential or certificate generation. + +=== Managed configuration example + +The recommended example uses the Operator-provided JAAS login module and Secret-mounted JKS material. The +<> contains the +complete resource, Secret creation, rendering, application, and verification steps. + +The important fields are: + +[cols="2,4",options="header"] +|=== +|Field |Meaning + +|`secure: true` +|Enables the complete managed configuration. Security fields are rejected when this value is `false`. + +|`basicAuth` +|References non-optional username and password keys in namespace-local Secrets. + +|`tls.caSecret` +|References the PEM CA bundle used by the Operator client. It is not a member truststore. + +|`tls.serverName` +|DNS name verified against the member certificate's subject alternative names. + +|`serverTLS` +|Configures the member's TLS identity. `enabled` must be `true` and `requireClientCert` must be `false`. + +|`serverTLS.keyStoreProvider` +|Optionally selects the Java keystore provider propagated to the managed `HealthSSLProvider`. + +|`jaas.mode` +|Selects `packaged` (the default) or `existing` JAAS configuration. +|=== + +The managed `HealthSSLProvider` does not use the `serverTLS.trustStore`, `trustStorePasswordFile`, `trustStoreType`, +`trustStoreAlgorithm`, or `trustStoreProvider` fields. They remain accepted for compatibility with the shared TLS API, +but are not used for one-way TLS. + +=== Use an existing JAAS configuration + +Set `jaas.mode: existing` when the application already sets `java.security.auth.login.config`. The existing file must +contain a login context named exactly `CoherenceREST` and must validate the same Secret-backed credentials used by the +Operator client. + +Do not use the default `packaged` mode when another global JAAS file is already configured. Managed mode refuses to +overwrite it. The published example includes the required `ConfigMap`, volume mount, JVM argument, and +`healthMutatorConnection` fragment. + +=== Use application-mounted TLS files + +If the application already mounts the server keystore and password files, omit `serverTLS.secrets` and specify absolute +container paths instead. The published example includes the required volume and explicit-path configuration. + +The Operator still manages the Coherence Health properties, JAAS mode, Basic credentials, client CA, and authenticated +request. + +=== Supported application packaging + +Managed mode supports normal Java/classpath applications and Spring Boot 2 or 3 applications whose executable JAR is +configured with `spec.application.springBootFatJar`. It does not support Cloud Native Buildpack launchers or a Spring +Boot application without a configured fat JAR. + +Managed mode also requires an explicit path for the Operator's generated JVM arguments to reach the member JVM. The +supported launcher configurations are: + +* the default Operator-controlled launcher, with no custom `spec.application.entryPoint` and + `spec.application.useImageEntryPoint` omitted or `false`; +* `spec.application.useImageEntryPoint: true` with `useJdkJavaOptions` omitted or `true`, which supplies the generated + argument-file reference through `JDK_JAVA_OPTIONS`; or +* `spec.application.useImageEntryPoint: true` with `useJdkJavaOptions: false` and a non-empty + `alternateJdkJavaOptions` environment variable name. + +The last configuration is an explicit application contract: the image entry point must read the named environment +variable and pass its generated argument-file reference to the actual member JVM. The Operator cannot verify that the +application honors this contract. A custom `application.entryPoint` is supported only when +`useImageEntryPoint: true` also selects one of the two environment-variable delivery paths above. + +The Operator rejects an unsupported launcher combination before reconciling the workload. This validation applies only +to `healthMutatorConnection.secure: true`; omitted or insecure managed Health configuration retains the existing +launcher behavior. + +Managed mode requires the Coherence Health server. Do not set `COHERENCE_OPERATOR_HEALTH_CHECK=true`; that selects the +legacy Operator Health server and the Pod configuration fails closed. + +[[application_owned]] +== Application-owned Health security + +Use an application-owned configuration when the application already configures its Health listener, needs certificate +authentication, or cannot use the managed launcher arrangement. Omit `healthMutatorConnection`; do not combine managed +and application-owned server properties. + +The application must configure all of the following: + +* `coherence.health.http.mutators.enabled=true`; +* `coherence.health.http.auth=basic`, `cert`, or `cert+basic`; +* an HTTPS socket provider selected by `coherence.health.http.provider`; +* the socket provider's identity and, for client-certificate authentication, trust configuration; +* a JAAS `CoherenceREST` context when Basic authentication is used; and +* explicit readiness, liveness, and startup probes that can use the application's TLS contract, because the Operator + does not rewrite probes in application-owned mode. + +The read-only endpoints do not require mutator authentication. With one-way TLS, Kubernetes HTTP probes can call them +over HTTPS. If the listener requires a client certificate, use an exec or other probe capable of presenting it. + +=== Basic authentication with an Operator HTTP action + +For a Basic-authenticated endpoint, configure a method-aware `suspendProbe.http`. The Operator reads credentials and +the CA from Secrets and performs a verified HTTPS request. The published example includes both a complete +application-owned Basic-over-TLS resource and a reusable Operator-client fragment. + +The application must independently configure matching Basic credentials, TLS material, and Coherence Health system +properties. The Operator HTTP action supports `GET` and `PUT`, verified server TLS, Secret-backed Basic authentication, +and non-secret headers. Use `PUT` for hardened `/suspend`. + +=== Certificate authentication and other custom clients + +Coherence also supports `cert` and `cert+basic` mutator authentication. These modes require a TLS client certificate. +The Operator HTTP action does not present one, so configure an application-owned exec action or another +application-owned client that can satisfy the server's authentication contract. + +The exec fragment in the published example demonstrates the resource shape but uses Basic authentication. Replace its +command and mounted client material for the selected certificate contract. The application image must contain the +required client program. Exec actions are less portable than the managed HTTP client and provide fewer sanitized +diagnostics. + +An empty custom probe or a gRPC-only probe is retained as a successful no-op for compatibility with existing +resources. The Operator emits a `ProbeActionNoOp` Warning event when it encounters either form. No request or command +is sent, so the result does not prove StatusHA or that services were suspended. Configure an Operator-supported +`http`, `httpGet`, `exec`, or `tcpSocket` action instead. + +== Disable or bypass managed security + +Omitting `healthMutatorConnection`, or setting only `healthMutatorConnection.secure: false`, disables the Operator's +managed Health security configuration. The Operator then sends the default `PUT /suspend` over HTTP unless a custom +`suspendProbe` is configured. These two disabled forms generate equivalent Pod templates, so switching between them +does not by itself replace Pods. Switching from `secure: true` to either disabled form does change the template and +requires a rollout to remove the secure-generation annotation, credentials, mounts, JVM configuration, and HTTPS probe +wiring. Enabling managed security makes the corresponding template changes and also requires a rollout. + +This does not disable the security layer in a hardened Coherence release. Hardened Coherence keeps mutators disabled +until they are explicitly enabled and always requires one of its supported authentication modes. Consequently, +`secure: false` by itself cannot suspend a hardened member. + +Setting `spec.suspendServicesOnShutdown: false` skips service suspension entirely. This is an operational bypass, not +a security setting. It can make scale-to-zero or shutdown unsafe, particularly with persistence, and should not be used +to work around an authentication or TLS configuration error. + +== Upgrade an existing deployment + +Upgrade the Operator deployment and CRD to 3.6.0 or later before rolling a hardened Coherence image into an existing +deployment. Add the selected Health security configuration as part of the Coherence image rollout. Upgrading only the +Operator does not enable security inside existing member Pods, and upgrading only Coherence leaves an older Operator +unable to authenticate or use the required `PUT` method. + +[WARNING] +==== +The 3.6.0 CRD limits `spec.actions` to 128 entries. Installing the CRD does not remove or retroactively reject an +existing `Coherence` resource with more than 128 actions, but Kubernetes will reject subsequent updates until the +submitted resource contains no more than 128 actions. Check existing resources and reduce longer action lists before +the upgrade or as the first change after upgrading the CRD. +==== + +For Operator-managed mode, confirm before the rollout that: + +* all referenced Secrets and keys exist in the `Coherence` resource namespace; +* the member certificate is valid for `tls.serverName`; +* the member keystore and password files are readable by the Coherence container; +* an existing JAAS file contains the exact `CoherenceREST` context when `jaas.mode: existing` is selected; and +* custom `coherence.health.*` JVM properties are removed because they conflict with managed mode. + +Keep the previous image, resource definition, and Secret generation available until the new Pods are ready and safe +shutdown has been tested. Rotate credentials and certificates by creating a new immutable Secret generation and +updating all related references together. + +== Verify the configuration + +After the rollout, verify these observable results: + +[cols="2,3",options="header"] +|=== +|Check |Expected result + +|Generated kubelet probes in managed mode +|Readiness, liveness, and startup probes that target the Health port use `HTTPS`. + +|Unauthenticated `GET /ready` +|`200` when the member is ready. Read-only probes do not require mutator credentials. + +|`PUT /suspend` while mutators are disabled +|`403`. + +|Unauthenticated or invalid-credential `PUT /suspend` +|`401` when Basic authentication is configured. + +|Authenticated `PUT /suspend` +|`200`. Send `PUT /resume` after a direct manual test. + +|Invalid CA, certificate name, credential, or Secret reference during scale-to-zero +|The transition remains blocked and existing members remain running. +|=== + +Use application-level write and read checks when validating persistence recovery. Ready Pods and unchanged PVC names do +not by themselves prove that application data was recovered. + +== Troubleshooting + +[cols="2,3",options="header"] +|=== +|Observation |Action + +|`403` from `PUT /suspend` +|Mutators are disabled. In managed mode, confirm that the member Pod contains the current +`healthMutatorConnection` generation and started successfully. In application-owned mode, set +`coherence.health.http.mutators.enabled=true`. + +|`401` from `PUT /suspend` +|Confirm that the Operator client and `CoherenceREST` JAAS context use the same Secret keys and exact credential bytes. + +|TLS name or trust failure +|Confirm `tls.serverName`, the certificate subject alternative names, and the PEM bundle in `tls.caSecret`. Do not +disable certificate verification. A member-side truststore does not repair Operator-client trust. + +|Pod configuration rejects the selected Health server +|Remove `COHERENCE_OPERATOR_HEALTH_CHECK=true`. Managed mode requires the Coherence Health server. + +|Pod configuration reports a direct-property conflict +|Remove application-supplied `coherence.health.*` and `coherence.health.security.*` JVM properties, or omit +`healthMutatorConnection` and use the application-owned model. + +|Scale-to-zero remains blocked +|Treat the block as fail-closed. Correct the authentication, TLS, Secret, or runtime configuration while members remain +running, or restore `spec.replicas` to a non-zero value. Do not delete the `Coherence` resource or its namespace while +the transition is blocked: desired replicas can be zero while members are still running, and deletion can terminate +those members without the expected suspension. Wait until scale-to-zero completes or the restored deployment is healthy +before deleting it. +|=== + +Do not add an anonymous or `GET` fallback, combine a custom `suspendProbe` with +`healthMutatorConnection.secure: true`, or disable safe suspension to hide a security configuration failure. diff --git a/docs/health/health-mutators.md b/docs/health/health-mutators.md new file mode 100644 index 000000000..82a0b5fb3 --- /dev/null +++ b/docs/health/health-mutators.md @@ -0,0 +1,46 @@ + + +# Authenticated Coherence health mutators + +The generated client documentation publishes the complete first-run guide at +[`010_protected_health_endpoints.adoc`](010_protected_health_endpoints.adoc). +Reusable customer manifests and procedures are in the +[`096_protected_health_endpoints` example](../../examples/096_protected_health_endpoints/README.adoc). +The notes below summarize implementation boundaries; they are not a substitute +for that client workflow. + +`spec.healthMutatorConnection` is an opt-in StatefulSet setting for Coherence runtimes that expose the hardened Coherence health handler. It does not switch health-server implementations. If `COHERENCE_OPERATOR_HEALTH_CHECK=true` selects the legacy Operator health server, managed secure mode fails during Pod configuration. + +The supported managed path sends `PUT /suspend` with Secret-backed Basic credentials over verified HTTPS. It enables `coherence.health.http.mutators.enabled=true`, selects `coherence.health.http.auth=basic`, configures `HealthSSLProvider`, and uses the `CoherenceREST` JAAS context. Read-only health probes stay unauthenticated. Explicit custom suspend and kubelet probes remain customer-owned and are not rewritten. + +One-way member TLS requires a server keystore and the keystore/key password files. The member-side provider does not load a truststore because it does not request client certificates. The separate `tls.caSecret` remains required for the Operator to verify each member's certificate. Older `serverTLS.trustStore*` fields are still accepted but are ignored by this health provider. + +The initial supported packaging paths are normal Java/classpath and Spring Boot +2/3 with a configured fat JAR. The Cloud Native Buildpack launcher is rejected +explicitly. Startup fails closed if the selected application runtime cannot load +the hardened handler contract. Qualify the exact released Operator image, +Coherence artifact, Java version, application image, and Kubernetes version used +for a deployment. + +## Protected endpoint configuration examples + +See the [published customer example](../../examples/096_protected_health_endpoints/README.adoc) +for supported managed and application-owned configurations. + +## Repository acceptance fixtures + +The repository-only managed fixture and manual application-owned qualification +workflow now live beside their scripts and manifests in +[`test/health-mutator/README.md`](../../test/health-mutator/README.md). They use +test applications, generated credentials, disposable namespaces, failure +injection, and destructive cleanup; they are not production deployment +instructions. + +Do not add GET or anonymous fallback, disable TLS verification, select the +Operator health server as a workaround, or remove old credential and certificate +material while an old Pod generation may still use it. Mixed-generation +credential or certificate rotation and no-downtime rotation remain unqualified. diff --git a/docs/scaling/010_overview.adoc b/docs/scaling/010_overview.adoc index 496b96351..381ac8ec3 100644 --- a/docs/scaling/010_overview.adoc +++ b/docs/scaling/010_overview.adoc @@ -1,6 +1,6 @@ /////////////////////////////////////////////////////////////////////////////// - Copyright (c) 2020, 2025, Oracle and/or its affiliates. + Copyright (c) 2020, 2026, Oracle and/or its affiliates. Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. @@ -37,6 +37,11 @@ What these steps ensure is that the deployment will not be resized unless the cl When scaling down only a single member will be removed at a time, ensuring that the cluster is in a safe state before removing the next member. +For a Coherence runtime that protects the `PUT /suspend` health mutator, configure the Operator to authenticate over +verified TLS before relying on safe scale-down or scale-to-zero. See +<> for the complete managed +configuration, transition checks, and fail-closed recovery procedure. + NOTE: The Operator will only apply safe scaling functionality to deployments that are storage enabled. If a deployment is storage disabled then it can be scaled up or down by the required number of members in one step as there is no fear of data loss in a storage disabled member. @@ -312,4 +317,3 @@ spec: - safe.sh ---- <1> This deployment will check the status of the services by running the `sh safe.sh` command in the `Pod`. - diff --git a/docs/sitegen.yaml b/docs/sitegen.yaml index b5aabbfe3..c6148f35c 100644 --- a/docs/sitegen.yaml +++ b/docs/sitegen.yaml @@ -1,5 +1,5 @@ # -# Copyright (c) 2020, 2024, Oracle and/or its affiliates. +# Copyright (c) 2020, 2026, Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at # http://oss.oracle.com/licenses/upl. # @@ -111,6 +111,14 @@ backend: items: - includes: - "docs/scaling/*.adoc" + - title: "Protected Health Endpoints" + pathprefix: "/health" + glyph: + type: "icon" + value: "security" + items: + - includes: + - "docs/health/*.adoc" - title: "Start-up Order" pathprefix: "/ordering" glyph: diff --git a/examples/096_protected_health_endpoints/README.adoc b/examples/096_protected_health_endpoints/README.adoc new file mode 100644 index 000000000..a03e9d6bf --- /dev/null +++ b/examples/096_protected_health_endpoints/README.adoc @@ -0,0 +1,284 @@ +/////////////////////////////////////////////////////////////////////////////// + + Copyright (c) 2026, Oracle and/or its affiliates. + Licensed under the Universal Permissive License v 1.0 as shown at + http://oss.oracle.com/licenses/upl. + +/////////////////////////////////////////////////////////////////////////////// + += Protected Coherence Health Endpoints Example +:description: Coherence Operator Documentation - Protected Coherence Health Endpoints Example +:keywords: oracle coherence, kubernetes, operator, health, authentication, tls, scale to zero, example + +This example shows how to configure the Coherence Operator to call a protected Coherence Health mutator during safe +shutdown and scale-to-zero. It covers the recommended Operator-managed configuration and the application-owned +alternatives. + +See the <> documentation for +compatible versions, configuration ownership, upgrade guidance, security constraints, and troubleshooting. + +[TIP] +==== +image:GitHub-Mark-32px[] The complete source for this example is in the +https://{examples-source}096_protected_health_endpoints[Coherence Operator GitHub] repository. +==== + +== Configuration choices + +[cols="2,3,3",options="header"] +|=== +|Configuration |Use when |Example + +|Managed security with packaged JAAS (recommended) +|The application can use the Coherence Health server and Basic authentication. The Operator owns the member and client +configuration. +|`manifests/managed-packaged-jaas.yaml` + +|Managed security with an existing JAAS file +|The application already owns the JVM's global JAAS configuration and supplies the required `CoherenceREST` context. +|`manifests/managed-existing-jaas.yaml` + +|Managed security with application-mounted TLS files +|The application already mounts the member keystore and its password files. +|`manifests/managed-mounted-store-paths.yaml` + +|Application-owned Basic-over-TLS server +|The application owns the Coherence Health system properties, TLS, JAAS, and kubelet probes. The Operator owns only +the authenticated suspension request. +|`manifests/application-owned-basic.yaml` + +|Application-owned server with an HTTP action +|The application already owns a Basic-authenticated HTTPS server and needs only the Operator-client fragment. +|`manifests/application-owned-http-action.yaml` + +|Application-owned server with a custom action +|The application needs a client flow that the Operator's HTTP action cannot express. +|`manifests/application-owned-exec-action.yaml` +|=== + +Use the managed configuration unless the application already owns the protected Health server. Do not combine +`healthMutatorConnection.secure: true` with a custom `suspendProbe` or application-supplied `coherence.health.*` +properties. + +== Prerequisites + +Before using this example: + +* install Coherence Operator 3.6.0 or later and its matching CRD; +* select a Coherence release that supports protected `PUT` Health mutators; +* make the application image available to every Kubernetes node, preferably by immutable digest; +* obtain a server keystore containing the member private key and certificate; +* obtain the keystore and key password files; and +* obtain the PEM CA bundle that the Operator will use to verify the member certificate. + +The member certificate must contain `health.example`, or the value selected for `tls.serverName`, in its subject +alternative names. The managed provider uses one-way TLS and does not require a member truststore. + +Run the commands below from the root of the Operator repository. Select the context and namespace explicitly: + +[source,bash] +---- +export CONTEXT='YOUR_KUBERNETES_CONTEXT' +export NAMESPACE='YOUR_NAMESPACE' +export COHERENCE_IMAGE='registry.example/application@sha256:REPLACE_WITH_DIGEST' + +kubectl --context "$CONTEXT" create namespace "$NAMESPACE" \ + --dry-run=client -o yaml | kubectl --context "$CONTEXT" apply -f - +---- + +== Create the Secrets + +Create immutable, versioned Secrets in the same namespace as the `Coherence` resource. Do not put credentials or +private keys directly in the resource. + +The following example creates a high-entropy password without a trailing newline and copies externally issued TLS +material into a private temporary directory. Replace the paths under `/secure/material` with the locations supplied by +the approved certificate and secret-management process. + +[source,bash] +---- +umask 077 +export MATERIAL="$(mktemp -d "${TMPDIR:-/tmp}/coherence-health.XXXXXX")" +trap 'rm -f "$MATERIAL/username" "$MATERIAL/password" "$MATERIAL/server.jks" "$MATERIAL/store-password" "$MATERIAL/ca.pem"; rmdir "$MATERIAL" 2>/dev/null || true' EXIT + +printf '%s' 'health-operator' > "$MATERIAL/username" +openssl rand -hex 32 | tr -d '\n' > "$MATERIAL/password" +install -m 600 /secure/material/server.jks "$MATERIAL/server.jks" +install -m 600 /secure/material/store-password "$MATERIAL/store-password" +install -m 600 /secure/material/ca.pem "$MATERIAL/ca.pem" + +kubectl --context "$CONTEXT" -n "$NAMESPACE" create secret generic health-auth-v1 \ + --from-file=username="$MATERIAL/username" \ + --from-file=password="$MATERIAL/password" + +kubectl --context "$CONTEXT" -n "$NAMESPACE" create secret generic health-stores-v1 \ + --from-file=server.jks="$MATERIAL/server.jks" \ + --from-file=store-password="$MATERIAL/store-password" + +kubectl --context "$CONTEXT" -n "$NAMESPACE" create secret generic health-ca-v1 \ + --from-file=ca.pem="$MATERIAL/ca.pem" + +for secret in health-auth-v1 health-stores-v1 health-ca-v1; do + kubectl --context "$CONTEXT" -n "$NAMESPACE" patch secret "$secret" \ + --type=merge -p '{"immutable":true}' +done +---- + +Credential and password files are consumed as exact bytes. An unintended trailing newline changes the value. Versioned +Secret names make the selected generation explicit; they do not by themselves prove that a mixed-generation live +rotation is safe. + +== Use managed security + +The recommended configuration uses `healthMutatorConnection.secure: true`. The Operator configures the member's +`HealthSSLProvider`, authenticated Coherence mutators, packaged `CoherenceREST` JAAS entry, Secret mounts, HTTPS kubelet +probes, and authenticated `PUT /suspend` client. + +[source,yaml] +.manifests/managed-packaged-jaas.yaml +---- +include::manifests/managed-packaged-jaas.yaml[] +---- + +Render and apply the manifest: + +[source,bash] +---- +envsubst '${NAMESPACE} ${COHERENCE_IMAGE}' \ + < examples/096_protected_health_endpoints/manifests/managed-packaged-jaas.yaml \ + > "$MATERIAL/protected-health.yaml" + +kubectl --context "$CONTEXT" apply -f "$MATERIAL/protected-health.yaml" +kubectl --context "$CONTEXT" -n "$NAMESPACE" rollout status \ + statefulset/protected-health --timeout=10m +---- + +The example enables active persistence and therefore requires a default `ReadWriteOnce` StorageClass. Replace the image, +application, cache configuration, persistence settings, and resource sizing with the deployment's actual values. + +=== Use an existing JAAS file + +When the application already sets `java.security.auth.login.config`, use `jaas.mode: existing`. The existing file must +contain a login context named exactly `CoherenceREST` and must validate the same credentials used by the Operator client. + +[source,yaml] +.manifests/managed-existing-jaas.yaml +---- +include::manifests/managed-existing-jaas.yaml[] +---- + +The file is a `ConfigMap` plus a `spec` fragment. Merge that fragment into the StatefulSet-backed `Coherence` resource. +Managed startup fails instead of overwriting an existing global JAAS selection. + +=== Use application-mounted keystore files + +When the application already mounts the server keystore and password files, omit `serverTLS.secrets` and supply absolute +container paths: + +[source,yaml] +.manifests/managed-mounted-store-paths.yaml +---- +include::manifests/managed-mounted-store-paths.yaml[] +---- + +This choice changes only ownership of the member TLS mount. The Operator still configures the Health server, Basic +credentials, client CA, JAAS mode, and authenticated suspension request. + +== Use application-owned Health security + +Use application-owned mode when the application must configure the Health listener itself or cannot use the managed +launcher arrangement. Omit `healthMutatorConnection`; the application owns: + +* `coherence.health.http.mutators.enabled=true`; +* the `basic`, `cert`, or `cert+basic` authentication selection; +* the HTTPS socket provider and server identity; +* the `CoherenceREST` JAAS context when Basic authentication is used; +* the required Secret volumes and JVM properties; and +* HTTPS readiness, liveness, and any startup probes. + +The following complete Basic-over-TLS example uses the Operator support library's `SecretLoginModule`. The selected +application launcher must place both Coherence and that LoginModule on the member JVM classpath. + +[source,yaml] +.manifests/application-owned-basic.yaml +---- +include::manifests/application-owned-basic.yaml[] +---- + +Render and apply it in the same way as the managed example: + +[source,bash] +---- +envsubst '${NAMESPACE} ${COHERENCE_IMAGE}' \ + < examples/096_protected_health_endpoints/manifests/application-owned-basic.yaml \ + > "$MATERIAL/application-owned-health.yaml" + +kubectl --context "$CONTEXT" apply -f "$MATERIAL/application-owned-health.yaml" +kubectl --context "$CONTEXT" -n "$NAMESPACE" rollout status \ + statefulset/application-owned-health --timeout=10m +---- + +The `suspendProbe.http` block configures only the Operator client. Unlike managed mode, it does not configure the member +JVM or rewrite kubelet probes to HTTPS. + +=== Use only the custom HTTP-action fragment + +If the application already supplies the protected server configuration, merge the following Operator-client and probe +fragment into its `Coherence` resource: + +[source,yaml] +.manifests/application-owned-http-action.yaml +---- +include::manifests/application-owned-http-action.yaml[] +---- + +The HTTP action supports verified server TLS and Secret-backed Basic credentials, but it does not present a TLS client +certificate. + +=== Use a custom in-Pod client + +For certificate authentication, `cert+basic`, or another client flow that the HTTP action cannot express, use an +application-owned exec action. The application image must contain the required client program and must mount its client +material. + +[source,yaml] +.manifests/application-owned-exec-action.yaml +---- +include::manifests/application-owned-exec-action.yaml[] +---- + +The supplied fragment demonstrates the resource shape with `curl` and Basic authentication. Replace the command and +mounted material for the selected certificate contract. An exec action is less portable and provides fewer sanitized +diagnostics than the managed HTTP client. + +== Verify the deployment + +For managed mode, confirm that generated probes targeting the Health port use HTTPS: + +[source,bash] +---- +kubectl --context "$CONTEXT" -n "$NAMESPACE" get statefulset protected-health \ + -o jsonpath='{range .spec.template.spec.containers[*]}{.name}{" ready="}{.readinessProbe.httpGet.scheme}{":"}{.readinessProbe.httpGet.port}{" live="}{.livenessProbe.httpGet.scheme}{":"}{.livenessProbe.httpGet.port}{"\n"}{end}' +---- + +Then verify the deployment's application data before and after scale-to-zero. A failed credential, CA, or certificate +name must block the transition while the existing members remain running. Ready Pods and unchanged PVC names do not by +themselves prove that application data was recovered. + +Do not add an anonymous or `GET` fallback, disable certificate verification, or set +`spec.suspendServicesOnShutdown: false` to hide a security configuration failure. + +WARNING: If scale-to-zero is blocked, correct the configuration or restore `spec.replicas` to a non-zero value before +deleting the `Coherence` resource or its namespace. Desired replicas can be zero while members are still running, and +deletion in that state can terminate those members without the expected suspension. + +== Clean up an example namespace + +Delete only a namespace that was created specifically for this example: + +[source,bash] +---- +kubectl --context "$CONTEXT" delete namespace "$NAMESPACE" +---- + +Namespace deletion can also delete dynamically provisioned volumes according to the StorageClass reclaim policy. diff --git a/examples/096_protected_health_endpoints/manifests/application-owned-basic.yaml b/examples/096_protected_health_endpoints/manifests/application-owned-basic.yaml new file mode 100644 index 000000000..93e56bd2b --- /dev/null +++ b/examples/096_protected_health_endpoints/manifests/application-owned-basic.yaml @@ -0,0 +1,97 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at +# http://oss.oracle.com/licenses/upl. +# +# Complete application-owned Basic-over-TLS configuration. Render NAMESPACE +# and COHERENCE_IMAGE with envsubst before applying. The application image must +# place Coherence and the selected JAAS LoginModule on the member classpath. +apiVersion: v1 +kind: ConfigMap +metadata: + name: application-health-jaas-v1 + namespace: ${NAMESPACE} +data: + jaas.conf: | + CoherenceREST { + com.oracle.coherence.k8s.SecretLoginModule required + usernameFile="/application/health-auth/username" + passwordFile="/application/health-auth/password"; + }; +--- +apiVersion: coherence.oracle.com/v1 +kind: Coherence +metadata: + name: application-owned-health + namespace: ${NAMESPACE} +spec: + replicas: 3 + image: ${COHERENCE_IMAGE} + imagePullPolicy: IfNotPresent + healthPort: 8089 + + # The application owns every Coherence Health security property. There is + # deliberately no healthMutatorConnection block. + jvm: + args: + - -Dcoherence.health.http.mutators.enabled=true + - -Dcoherence.health.http.auth=basic + - -Dcoherence.health.http.provider=HealthSSLProvider + - -Dcoherence.health.security.keystore=file:/application/health-tls/server.jks + - -Dcoherence.health.security.keystore.password=/application/health-tls/store-password + - -Dcoherence.health.security.key.password=/application/health-tls/store-password + - -Dcoherence.health.security.keystore.type=JKS + - -Djava.security.auth.login.config=/application/health-jaas/jaas.conf + + volumes: + - name: application-health-auth + secret: + secretName: health-auth-v1 + - name: application-health-tls + secret: + secretName: health-stores-v1 + - name: application-health-jaas + configMap: + name: application-health-jaas-v1 + volumeMounts: + - name: application-health-auth + mountPath: /application/health-auth + readOnly: true + - name: application-health-tls + mountPath: /application/health-tls + readOnly: true + - name: application-health-jaas + mountPath: /application/health-jaas + readOnly: true + + # This block configures only the Operator client. The JVM arguments, JAAS + # file, and volumes above configure the member server. + suspendProbe: + timeoutSeconds: 60 + http: + method: PUT + scheme: HTTPS + path: /suspend + port: 8089 + basicAuth: + username: {name: health-auth-v1, key: username} + password: {name: health-auth-v1, key: password} + tls: + caSecret: {name: health-ca-v1, key: ca.pem} + serverName: health.example + + # Application-owned mode does not rewrite these probes to HTTPS. + readinessProbe: + httpGet: + scheme: HTTPS + path: /ready + port: 8089 + livenessProbe: + httpGet: + scheme: HTTPS + path: /live + port: 8089 + startupProbe: + httpGet: + scheme: HTTPS + path: /started + port: 8089 diff --git a/examples/096_protected_health_endpoints/manifests/application-owned-exec-action.yaml b/examples/096_protected_health_endpoints/manifests/application-owned-exec-action.yaml new file mode 100644 index 000000000..4cf4a2c77 --- /dev/null +++ b/examples/096_protected_health_endpoints/manifests/application-owned-exec-action.yaml @@ -0,0 +1,52 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at +# http://oss.oracle.com/licenses/upl. +# +# Last-resort application-owned action. The image must contain /bin/sh and curl. +# The application independently configures its protected health server. +spec: + healthPort: 8089 + volumes: + - name: health-client-material + projected: + sources: + - secret: + name: health-auth-v1 + items: + - {key: username, path: username} + - {key: password, path: password} + - secret: + name: health-ca-v1 + items: + - {key: ca.pem, path: ca.pem} + volumeMounts: + - name: health-client-material + mountPath: /application/health-client + readOnly: true + suspendProbe: + timeoutSeconds: 60 + exec: + command: + - /bin/sh + - -ec + - >- + curl --silent --show-error --fail + --cacert /application/health-client/ca.pem + --resolve health.example:8089:127.0.0.1 + --user "$(cat /application/health-client/username):$(cat /application/health-client/password)" + --request PUT https://health.example:8089/suspend + readinessProbe: + httpGet: + scheme: HTTPS + path: /ready + port: 8089 + livenessProbe: + httpGet: + scheme: HTTPS + path: /live + port: 8089 + startupProbe: + httpGet: + scheme: HTTPS + path: /started + port: 8089 diff --git a/examples/096_protected_health_endpoints/manifests/application-owned-http-action.yaml b/examples/096_protected_health_endpoints/manifests/application-owned-http-action.yaml new file mode 100644 index 000000000..5fec2396d --- /dev/null +++ b/examples/096_protected_health_endpoints/manifests/application-owned-http-action.yaml @@ -0,0 +1,40 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at +# http://oss.oracle.com/licenses/upl. +# +# Operator-client fragment for an application-owned protected Health server. +# Use only when the application independently configures the protected +# Coherence health server. Do not add healthMutatorConnection with this probe. +spec: + healthPort: 8089 + suspendProbe: + timeoutSeconds: 60 + http: + method: PUT + scheme: HTTPS + path: /suspend + port: 8089 + basicAuth: + username: {name: health-auth-v1, key: username} + password: {name: health-auth-v1, key: password} + tls: + caSecret: {name: health-ca-v1, key: ca.pem} + serverName: health.example + + # Custom probes are required because unmanaged mode does not rewrite the + # generated probes to HTTPS. Read-only endpoints must remain unauthenticated. + readinessProbe: + httpGet: + scheme: HTTPS + path: /ready + port: 8089 + livenessProbe: + httpGet: + scheme: HTTPS + path: /live + port: 8089 + startupProbe: + httpGet: + scheme: HTTPS + path: /started + port: 8089 diff --git a/examples/096_protected_health_endpoints/manifests/managed-existing-jaas.yaml b/examples/096_protected_health_endpoints/manifests/managed-existing-jaas.yaml new file mode 100644 index 000000000..1802f1a00 --- /dev/null +++ b/examples/096_protected_health_endpoints/manifests/managed-existing-jaas.yaml @@ -0,0 +1,51 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at +# http://oss.oracle.com/licenses/upl. +# +# Managed Health security with an application-owned JAAS file. The file must +# retain the exact CoherenceREST context. +apiVersion: v1 +kind: ConfigMap +metadata: + name: health-jaas-v1 + namespace: ${NAMESPACE} +data: + jaas.conf: | + CoherenceREST { + com.oracle.coherence.k8s.SecretLoginModule required + usernameFile="/coherence/health-auth/username/value" + passwordFile="/coherence/health-auth/password/value"; + }; +--- +# Merge this spec fragment into a StatefulSet-backed Coherence resource. +spec: + healthPort: 8089 + jvm: + args: + - -Djava.security.auth.login.config=/application/security/jaas.conf + volumes: + - name: health-jaas + configMap: + name: health-jaas-v1 + volumeMounts: + - name: health-jaas + mountPath: /application/security + readOnly: true + healthMutatorConnection: + secure: true + basicAuth: + username: {name: health-auth-v1, key: username} + password: {name: health-auth-v1, key: password} + tls: + caSecret: {name: health-ca-v1, key: ca.pem} + serverName: health.example + serverTLS: + enabled: true + requireClientCert: false + secrets: health-stores-v1 + keyStore: server.jks + keyStorePasswordFile: store-password + keyPasswordFile: store-password + keyStoreType: JKS + jaas: + mode: existing diff --git a/examples/096_protected_health_endpoints/manifests/managed-mounted-store-paths.yaml b/examples/096_protected_health_endpoints/manifests/managed-mounted-store-paths.yaml new file mode 100644 index 000000000..d2ea38b87 --- /dev/null +++ b/examples/096_protected_health_endpoints/manifests/managed-mounted-store-paths.yaml @@ -0,0 +1,34 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at +# http://oss.oracle.com/licenses/upl. +# +# Use explicit paths when the application owns the TLS volume mount. The +# credentials and client CA are still namespace-local Secret references. +spec: + healthPort: 8089 + volumes: + - name: application-health-tls + secret: + secretName: health-stores-v1 + volumeMounts: + - name: application-health-tls + mountPath: /application/health-tls + readOnly: true + healthMutatorConnection: + secure: true + basicAuth: + username: {name: health-auth-v1, key: username} + password: {name: health-auth-v1, key: password} + tls: + caSecret: {name: health-ca-v1, key: ca.pem} + serverName: health.example + serverTLS: + enabled: true + requireClientCert: false + # No 'secrets' field: every value below is an absolute container path. + keyStore: /application/health-tls/server.jks + keyStorePasswordFile: /application/health-tls/store-password + keyPasswordFile: /application/health-tls/store-password + keyStoreType: JKS + jaas: + mode: packaged diff --git a/examples/096_protected_health_endpoints/manifests/managed-packaged-jaas.yaml b/examples/096_protected_health_endpoints/manifests/managed-packaged-jaas.yaml new file mode 100644 index 000000000..2a260faf5 --- /dev/null +++ b/examples/096_protected_health_endpoints/manifests/managed-packaged-jaas.yaml @@ -0,0 +1,50 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at +# http://oss.oracle.com/licenses/upl. +# +# Complete recommended managed configuration. Render NAMESPACE and +# COHERENCE_IMAGE with envsubst before applying. The image must contain the +# application and a Coherence runtime that supports protected PUT mutators. +apiVersion: coherence.oracle.com/v1 +kind: Coherence +metadata: + name: protected-health + namespace: ${NAMESPACE} +spec: + replicas: 3 + image: ${COHERENCE_IMAGE} + imagePullPolicy: IfNotPresent + healthPort: 8089 + healthMutatorConnection: + secure: true + basicAuth: + username: + name: health-auth-v1 + key: username + password: + name: health-auth-v1 + key: password + tls: + caSecret: + name: health-ca-v1 + key: ca.pem + serverName: health.example + serverTLS: + enabled: true + requireClientCert: false + secrets: health-stores-v1 + keyStore: server.jks + keyStorePasswordFile: store-password + keyPasswordFile: store-password + keyStoreType: JKS + jaas: + mode: packaged + coherence: + persistence: + mode: active + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/examples/README.adoc b/examples/README.adoc index 93708646c..0ef3d9910 100644 --- a/examples/README.adoc +++ b/examples/README.adoc @@ -1,6 +1,6 @@ /////////////////////////////////////////////////////////////////////////////// - Copyright (c) 2021, 2025 Oracle and/or its affiliates. + Copyright (c) 2021, 2026 Oracle and/or its affiliates. Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. @@ -70,6 +70,13 @@ Securing Coherence clusters using TLS. An example covering the use of Kubernetes `NetworkPolicy` rules with the Operator and Coherence clusters. -- +[CARD] +.Protected Health Endpoints +[link=examples/096_protected_health_endpoints/README.adoc] +-- +Configure authenticated, TLS-protected Coherence Health mutators for safe shutdown and scale-to-zero. +-- + [CARD] .Federation [link=examples/100_federation/README.adoc] diff --git a/java/coherence-operator/pom.xml b/java/coherence-operator/pom.xml index 9b0302429..b326ae638 100644 --- a/java/coherence-operator/pom.xml +++ b/java/coherence-operator/pom.xml @@ -1,7 +1,7 @@ @@ -213,6 +213,35 @@ + + legacy-coherence-toolchain + + + [24,) + + + + + org.apache.maven.plugins + maven-toolchains-plugin + + + + select-jdk-toolchain + + + ${coherence.legacy.java.version} + + + + + + + + release diff --git a/java/coherence-operator/src/main/java/com/oracle/coherence/k8s/HealthSecuritySupport.java b/java/coherence-operator/src/main/java/com/oracle/coherence/k8s/HealthSecuritySupport.java new file mode 100644 index 000000000..62116ab90 --- /dev/null +++ b/java/coherence-operator/src/main/java/com/oracle/coherence/k8s/HealthSecuritySupport.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ +package com.oracle.coherence.k8s; + +/** Checks Coherence health security capability without calling an administrative endpoint. */ +public final class HealthSecuritySupport { + private HealthSecuritySupport() { + } + + /** + * Verify the selected implementation exposes the supported security contract. + * + * @param args ignored + * @throws Exception if capability cannot be established + */ + public static void main(String[] args) throws Exception { + verify(Thread.currentThread().getContextClassLoader()); + } + + /** + * Verify the hardened Coherence handler visible to the supplied application class loader. + * + * @param loader the application class loader + * @throws Exception if the required contract is absent + */ + static void verify(ClassLoader loader) throws Exception { + Class handler = Class.forName("com.tangosol.internal.health.HealthHttpHandler", false, loader); + if (!"coherence.health.http.mutators.enabled".equals(handler.getField("PROP_MUTATORS_ENABLED").get(null))) { + throw new IllegalStateException("Unsupported health server"); + } + if (!"coherence.health.http.auth".equals(handler.getField("PROP_AUTH").get(null))) { + throw new IllegalStateException("Unsupported health authentication"); + } + handler.getDeclaredMethod("ensureMutatorAccess", + Class.forName("com.tangosol.internal.http.HttpRequest", false, loader)); + } +} diff --git a/java/coherence-operator/src/main/java/com/oracle/coherence/k8s/OperatorRestServer.java b/java/coherence-operator/src/main/java/com/oracle/coherence/k8s/OperatorRestServer.java index a2e59e8e9..0cf6df630 100644 --- a/java/coherence-operator/src/main/java/com/oracle/coherence/k8s/OperatorRestServer.java +++ b/java/coherence-operator/src/main/java/com/oracle/coherence/k8s/OperatorRestServer.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2025, Oracle and/or its affiliates. + * Copyright (c) 2019, 2026, Oracle and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl. */ diff --git a/java/coherence-operator/src/main/java/com/oracle/coherence/k8s/SecretLoginModule.java b/java/coherence-operator/src/main/java/com/oracle/coherence/k8s/SecretLoginModule.java new file mode 100644 index 000000000..817deeff0 --- /dev/null +++ b/java/coherence-operator/src/main/java/com/oracle/coherence/k8s/SecretLoginModule.java @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ +package com.oracle.coherence.k8s; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.MessageDigest; +import java.util.Arrays; +import java.util.Map; + +import javax.security.auth.Subject; +import javax.security.auth.callback.Callback; +import javax.security.auth.callback.CallbackHandler; +import javax.security.auth.callback.NameCallback; +import javax.security.auth.callback.PasswordCallback; +import javax.security.auth.login.FailedLoginException; +import javax.security.auth.login.LoginException; +import javax.security.auth.spi.LoginModule; + +/** + * Validates Basic credentials against customer-managed, mounted Secret files. + * Files are read for each login; supported rotation uses immutable Secret names. + */ +public class SecretLoginModule implements LoginModule { + private CallbackHandler handler; + private Map options; + private boolean authenticated; + + @Override + public void initialize(Subject subject, CallbackHandler callbacks, + Map sharedState, Map configuration) { + handler = callbacks; + options = configuration; + } + + @Override + public boolean login() throws LoginException { + NameCallback name = new NameCallback("username"); + PasswordCallback password = new PasswordCallback("password", false); + byte[] expectedUser = null; + byte[] expectedPassword = null; + byte[] suppliedPassword = null; + char[] chars = null; + try { + handler.handle(new Callback[] {name, password}); + expectedUser = read("usernameFile"); + expectedPassword = read("passwordFile"); + chars = password.getPassword(); + if (name.getName() == null || chars == null) { + throw new FailedLoginException("Invalid credentials"); + } + suppliedPassword = new String(chars).getBytes(StandardCharsets.UTF_8); + boolean userMatches = MessageDigest.isEqual(expectedUser, name.getName().getBytes(StandardCharsets.UTF_8)); + boolean passwordMatches = MessageDigest.isEqual(expectedPassword, suppliedPassword); + authenticated = userMatches & passwordMatches; + if (!authenticated) { + throw new FailedLoginException("Invalid credentials"); + } + return true; + } + catch (Exception failure) { + authenticated = false; + throw new FailedLoginException("Invalid credentials"); + } + finally { + password.clearPassword(); + if (chars != null) { + Arrays.fill(chars, '\0'); + } + for (byte[] value : new byte[][] {expectedUser, expectedPassword, suppliedPassword}) { + if (value != null) { + Arrays.fill(value, (byte) 0); + } + } + } + } + + private byte[] read(String option) throws Exception { + Object path = options.get(option); + if (!(path instanceof String)) { + throw new LoginException("Credential file is required"); + } + if (Files.size(Paths.get((String) path)) > 8192) { + throw new LoginException("Invalid credential file"); + } + byte[] value = Files.readAllBytes(Paths.get((String) path)); + if (value.length == 0) { + throw new LoginException("Invalid credential file"); + } + return value; + } + + @Override + public boolean commit() { + return authenticated; + } + + @Override + public boolean abort() { + authenticated = false; + return true; + } + + @Override + public boolean logout() { + authenticated = false; + return true; + } +} diff --git a/java/coherence-operator/src/main/resources/k8s-coherence-override.xml b/java/coherence-operator/src/main/resources/k8s-coherence-override.xml index d1f04f661..64ae9e339 100644 --- a/java/coherence-operator/src/main/resources/k8s-coherence-override.xml +++ b/java/coherence-operator/src/main/resources/k8s-coherence-override.xml @@ -1,6 +1,6 @@ @@ -115,6 +115,38 @@ + + + + + + + file:/coherence/health-tls/server.jks + + com.oracle.coherence.k8s.FileBasedPasswordProvider + + + String + /coherence/health-tls/storepass.txt + + + + + + + com.oracle.coherence.k8s.FileBasedPasswordProvider + + + String + /coherence/health-tls/keypass.txt + + + + + + + diff --git a/java/coherence-operator/src/test/java/com/oracle/coherence/k8s/HealthSSLProviderConfigTest.java b/java/coherence-operator/src/test/java/com/oracle/coherence/k8s/HealthSSLProviderConfigTest.java new file mode 100644 index 000000000..96f493596 --- /dev/null +++ b/java/coherence-operator/src/test/java/com/oracle/coherence/k8s/HealthSSLProviderConfigTest.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ +package com.oracle.coherence.k8s; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class HealthSSLProviderConfigTest { + @Test + void mapsManagedHealthStoreProperties() throws Exception { + try (InputStream input = getClass().getClassLoader().getResourceAsStream("k8s-coherence-override.xml")) { + assertNotNull(input); + String xml = new String(readAll(input), StandardCharsets.UTF_8); + int start = xml.indexOf(""); + assertTrue(start >= 0, "HealthSSLProvider is missing"); + String provider = xml.substring(start, xml.indexOf("", start)); + for (String expected : new String[] { + "", + "coherence.health.security.keystore", + "coherence.health.security.keystore.password", + "coherence.health.security.keystore.type", + "coherence.health.security.keystore.algorithm", + "coherence.health.security.keystore.provider", + "coherence.health.security.key.password"}) { + assertTrue(provider.contains(expected), "missing " + expected); + } + assertFalse(provider.contains("")); + assertFalse(provider.contains("coherence.health.security.truststore")); + assertFalse(provider.contains("")); + } + } + + private byte[] readAll(InputStream input) throws java.io.IOException { + java.io.ByteArrayOutputStream output = new java.io.ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int count; + while ((count = input.read(buffer)) >= 0) { + output.write(buffer, 0, count); + } + return output.toByteArray(); + } +} diff --git a/java/coherence-operator/src/test/java/com/oracle/coherence/k8s/HealthSecuritySupportTest.java b/java/coherence-operator/src/test/java/com/oracle/coherence/k8s/HealthSecuritySupportTest.java new file mode 100644 index 000000000..4daa7cada --- /dev/null +++ b/java/coherence-operator/src/test/java/com/oracle/coherence/k8s/HealthSecuritySupportTest.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ +package com.oracle.coherence.k8s; + +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; + +class HealthSecuritySupportTest { + @Test + void recognizesPinnedHardenedCoherenceHandler() throws Exception { + String configured = System.getProperty("health.mutator.coherence.jar", ""); + Assumptions.assumeTrue(!configured.isEmpty(), "pinned hardened Coherence jar is required by the dedicated CI lane"); + Path jar = Paths.get(configured); + Assumptions.assumeTrue(Files.isRegularFile(jar), "configured hardened Coherence jar does not exist"); + ClassLoader platformLoader = ClassLoader.getSystemClassLoader().getParent(); + try (URLClassLoader loader = new URLClassLoader(new java.net.URL[] {jar.toUri().toURL()}, platformLoader)) { + Class handler = Class.forName("com.tangosol.internal.health.HealthHttpHandler", false, loader); + assertEquals(jar.toUri(), handler.getProtectionDomain().getCodeSource().getLocation().toURI()); + assertDoesNotThrow(() -> HealthSecuritySupport.verify(loader)); + } + } +} diff --git a/java/coherence-operator/src/test/java/com/oracle/coherence/k8s/OperatorRestServerIT.java b/java/coherence-operator/src/test/java/com/oracle/coherence/k8s/OperatorRestServerIT.java index ec87d4ee4..73d20063b 100644 --- a/java/coherence-operator/src/test/java/com/oracle/coherence/k8s/OperatorRestServerIT.java +++ b/java/coherence-operator/src/test/java/com/oracle/coherence/k8s/OperatorRestServerIT.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2025, Oracle and/or its affiliates. + * Copyright (c) 2019, 2026, Oracle and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl. */ @@ -493,7 +493,7 @@ public void shouldSuspendAllServicesSingleMember() { // wait for ready Eventually.assertDeferred(() -> httpRequest(httpPort, OperatorRestServer.PATH_READY), is(200)); // suspend services - Eventually.assertDeferred(() -> this.httpRequest(httpPort, OperatorRestServer.PATH_SUSPEND), is(200)); + Eventually.assertDeferred(() -> this.httpRequest(httpPort, OperatorRestServer.PATH_SUSPEND, "PUT"), is(200)); Eventually.assertDeferred(() -> this.isServiceOneSuspended(app), is(true)); Eventually.assertDeferred(() -> this.isServiceTwoSuspended(app), is(true)); @@ -1034,10 +1034,15 @@ public void shouldResumeSpecificSuspendedServiceOnStartup() { // Must be public - used in Eventually.assertThat public int httpRequest(Capture httpPort, String path) { + return httpRequest(httpPort, path, "GET"); + } + + // Must be public - used in Eventually.assertThat + public int httpRequest(Capture httpPort, String path, String method) { try { URI uri = URI.create("http://127.0.0.1:" + httpPort.get() + path); HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection(); - connection.setRequestMethod("GET"); + connection.setRequestMethod(method); connection.connect(); return connection.getResponseCode(); } diff --git a/java/coherence-operator/src/test/java/com/oracle/coherence/k8s/SecretLoginModuleTest.java b/java/coherence-operator/src/test/java/com/oracle/coherence/k8s/SecretLoginModuleTest.java new file mode 100644 index 000000000..5ad69ccd5 --- /dev/null +++ b/java/coherence-operator/src/test/java/com/oracle/coherence/k8s/SecretLoginModuleTest.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ +package com.oracle.coherence.k8s; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +import javax.security.auth.Subject; +import javax.security.auth.callback.NameCallback; +import javax.security.auth.callback.PasswordCallback; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class SecretLoginModuleTest { + @Test + void validatesMountedCredentials() throws Exception { + Path directory = Files.createTempDirectory("health-login-"); + Path username = directory.resolve("username"); + Path password = directory.resolve("password"); + Files.write(username, "health-operator".getBytes(StandardCharsets.UTF_8)); + Files.write(password, "secret".getBytes(StandardCharsets.UTF_8)); + + SecretLoginModule module = module(username, password, "health-operator", "secret"); + assertTrue(module.login()); + assertTrue(module.commit()); + assertTrue(module.logout()); + + SecretLoginModule rejected = module(username, password, "health-operator", "wrong"); + assertThrows(javax.security.auth.login.FailedLoginException.class, rejected::login); + } + + private SecretLoginModule module(Path usernameFile, Path passwordFile, String username, String password) { + Map options = new HashMap<>(); + options.put("usernameFile", usernameFile.toString()); + options.put("passwordFile", passwordFile.toString()); + SecretLoginModule module = new SecretLoginModule(); + module.initialize(new Subject(), callbacks -> { + for (javax.security.auth.callback.Callback callback : callbacks) { + if (callback instanceof NameCallback) { + ((NameCallback) callback).setName(username); + } + else if (callback instanceof PasswordCallback) { + ((PasswordCallback) callback).setPassword(password.toCharArray()); + } + } + }, new HashMap<>(), options); + return module; + } +} diff --git a/java/pom.xml b/java/pom.xml index b20124e23..c77f5af7d 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -43,6 +43,7 @@ com.oracle.coherence.ce 21.12.5 + [17,18) 14.1.2-0-0 24.09.2 @@ -100,6 +101,7 @@ 3.1.0 2.2.1 3.0.0-M5 + 3.3.0 1.6.7 @@ -250,6 +252,12 @@ + + org.apache.maven.plugins + maven-toolchains-plugin + ${version.plugin.maven.toolchains} + + org.commonjava.maven.plugins directory-maven-plugin diff --git a/pkg/probe/http.go b/pkg/probe/http.go index c4f5e3eb7..dfe9c517f 100644 --- a/pkg/probe/http.go +++ b/pkg/probe/http.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2025, Oracle and/or its affiliates. + * Copyright (c) 2020, 2026, Oracle and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl. */ @@ -55,8 +55,7 @@ type GetHTTPInterface interface { func DoHTTPProbe(url *url.URL, headers http.Header, client GetHTTPInterface) (Result, string, error) { req, err := http.NewRequest("GET", url.String(), nil) if err != nil { - // Convert errors into failures to catch timeouts. - return Failure, err.Error(), nil + return Failure, "", &RequestError{Kind: "request configuration", Cause: err} } if _, ok := headers["User-Agent"]; !ok { if headers == nil { @@ -69,22 +68,25 @@ func DoHTTPProbe(url *url.URL, headers http.Header, client GetHTTPInterface) (Re if headers.Get("Host") != "" { req.Host = headers.Get("Host") } - log.Info("Executing HTTP Probe", "URL", url.String(), "Headers", headers) + log.Info("Executing HTTP Probe", "Method", "GET") res, err := client.Do(req) if err != nil { - // Convert errors into failures to catch timeouts. - return Failure, err.Error(), nil + kind := "transport" + if timeout, ok := err.(interface{ Timeout() bool }); ok && timeout.Timeout() { + kind = "transport timeout" + } + return Failure, "", &RequestError{Kind: kind, Cause: err} } defer closeBody(res) - b, err := io.ReadAll(res.Body) + b, err := io.ReadAll(io.LimitReader(res.Body, 64*1024)) if err != nil { - return Failure, "", err + return Failure, "", &RequestError{Kind: "uncertain response", Cause: err} } body := string(b) if res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusBadRequest { return Success, body, nil } - return Failure, fmt.Sprintf("HTTP probe failed with statuscode: %d", res.StatusCode), nil + return Failure, "", &RequestError{Kind: "rejected response", StatusCode: res.StatusCode} } func closeBody(res *http.Response) { diff --git a/pkg/probe/http_action.go b/pkg/probe/http_action.go new file mode 100644 index 000000000..3eb3e466e --- /dev/null +++ b/pkg/probe/http_action.go @@ -0,0 +1,254 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ +package probe + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io" + "net" + "net/http" + "strings" + "syscall" + + coh "github.com/oracle/coherence-operator/api/v1" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// RequestError preserves failure classification without exposing credentials or response bodies. +// Failures are terminal within a probe attempt, including ambiguous transport failures. +type RequestError struct { + Kind string + StatusCode int + Cause error +} + +func (e *RequestError) Error() string { + if e.StatusCode != 0 { + return fmt.Sprintf("HTTP probe rejected: status %d", e.StatusCode) + } + return "HTTP probe failed: " + e.diagnostic() +} +func (e *RequestError) Unwrap() error { return e.Cause } + +// diagnostic returns only controlled text. The wrapped cause remains available +// through Unwrap, but may contain request, credential, Secret or response data. +func (e *RequestError) diagnostic() string { + switch e.Kind { + case "transport timeout": + return "transport timeout" + case "transport", "transport or TLS": + return transportDiagnostic(e.Cause) + case "configuration": + return "configuration validation failure" + case "Secret client unavailable", "Secret unavailable", "Secret key missing or empty", + "port configuration", "endpoint configuration", "request configuration", + "invalid CA bundle", "invalid Basic username", "rejected response", "uncertain response", + "unknown active Pod connection", "invalid active Pod connection", + "managed security requires the generated suspend action", "missing active Pod connection", + "unsupported action": + return e.Kind + default: + return "request failure" + } +} + +func transportDiagnostic(err error) string { + if errors.Is(err, context.DeadlineExceeded) { + return "transport timeout" + } + var timeout interface{ Timeout() bool } + if errors.As(err, &timeout) && timeout.Timeout() { + return "transport timeout" + } + var unknownAuthority x509.UnknownAuthorityError + if errors.As(err, &unknownAuthority) { + return "TLS certificate signed by an unknown or untrusted CA" + } + var hostnameError x509.HostnameError + if errors.As(err, &hostnameError) { + return "TLS certificate hostname mismatch" + } + if errors.Is(err, syscall.ECONNREFUSED) { + return "connection refused" + } + var netError *net.OpError + if errors.As(err, &netError) { + return "transport failure" + } + return "transport or TLS failure" +} + +// SuspensionCauseKind identifies safe suspension diagnostics that are not HTTP +// request failures. +type SuspensionCauseKind string + +const ( + SuspensionCausePodNotReady SuspensionCauseKind = "PodNotReady" + SuspensionCausePodCountMismatch SuspensionCauseKind = "PodCountMismatch" + SuspensionCauseProbeUnavailable SuspensionCauseKind = "ProbeUnavailable" + SuspensionCauseWorkloadLookupFailed SuspensionCauseKind = "WorkloadLookupFailed" +) + +// SuspensionCauseError carries structured, sanitized context while preserving +// an underlying error for errors.Is/errors.As callers. +type SuspensionCauseError struct { + Kind SuspensionCauseKind + PodName string + PodPhase string + ActualPods int32 + ExpectedPods int32 + Cause error +} + +func (e *SuspensionCauseError) Error() string { + switch e.Kind { + case SuspensionCausePodNotReady: + return fmt.Sprintf("probe precondition failed: Pod %s is not ready (%s)", e.PodName, safePodPhase(e.PodPhase)) + case SuspensionCausePodCountMismatch: + return fmt.Sprintf("probe precondition failed: found %d Pods, expected %d", e.ActualPods, e.ExpectedPods) + case SuspensionCauseProbeUnavailable: + return "probe precondition failed: no running Pod could execute the probe" + case SuspensionCauseWorkloadLookupFailed: + return "workload state lookup failed" + default: + return "suspension failure" + } +} + +func (e *SuspensionCauseError) Unwrap() error { return e.Cause } + +func safePodPhase(phase string) string { + switch phase { + case "Pending", "Running", "Succeeded", "Failed", "Unknown", "Terminating": + return phase + default: + return "unknown state" + } +} + +func (in *CoherenceProbe) secretValue(ctx context.Context, namespace string, ref corev1.SecretKeySelector) ([]byte, error) { + if in.Client == nil { + return nil, &RequestError{Kind: "Secret client unavailable"} + } + secret := &corev1.Secret{} + if err := in.Client.Get(ctx, client.ObjectKey{Namespace: namespace, Name: ref.Name}, secret); err != nil { + return nil, &RequestError{Kind: "Secret unavailable", Cause: err} + } + value, ok := secret.Data[ref.Key] + if !ok || len(value) == 0 { + return nil, &RequestError{Kind: "Secret key missing or empty"} + } + return value, nil +} + +// ProbeUsingHTTPAction executes a verified, context-bound HTTP action without redirects. +func (in *CoherenceProbe) ProbeUsingHTTPAction(ctx context.Context, pod corev1.Pod, svc string, handler *coh.Probe) (bool, error) { + if err := handler.ValidateAction(); err != nil { + return false, &RequestError{Kind: "configuration", Cause: err} + } + ctx, cancel := context.WithTimeout(ctx, handler.GetTimeout()) + defer cancel() + a := handler.HTTP + scheme := a.Scheme + if scheme == "" { + scheme = corev1.URISchemeHTTP + } + host := a.Host + if host == "" { + host = in.GetPodIpOrHostName(pod) + } + port, err := in.findPort(pod, a.Port) + if err != nil { + return false, &RequestError{Kind: "port configuration", Cause: err} + } + u, err := buildHTTPProbeURL(scheme, host, port, a.Path) + if err != nil { + return false, &RequestError{Kind: "endpoint configuration", Cause: err} + } + method := a.Method + if method == "" { + method = http.MethodGet + } + req, err := http.NewRequestWithContext(ctx, method, u.String(), nil) + if err != nil { + return false, &RequestError{Kind: "request configuration", Cause: err} + } + req.Host = in.GetPodHostName(pod, svc) + for _, h := range a.HTTPHeaders { + if strings.EqualFold(h.Name, "Host") { + req.Host = h.Value + } else { + req.Header.Add(h.Name, h.Value) + } + } + config := &tls.Config{MinVersion: tls.VersionTLS12} + if a.TLS != nil { + pem, err := in.secretValue(ctx, pod.Namespace, a.TLS.CASecret) + if err != nil { + return false, err + } + config.RootCAs = x509.NewCertPool() + if !config.RootCAs.AppendCertsFromPEM(pem) { + return false, &RequestError{Kind: "invalid CA bundle"} + } + config.ServerName = a.TLS.ServerName + } + if a.BasicAuth != nil { + username, err := in.secretValue(ctx, pod.Namespace, a.BasicAuth.Username) + if err != nil { + return false, err + } + password, err := in.secretValue(ctx, pod.Namespace, a.BasicAuth.Password) + if err != nil { + return false, err + } + if strings.ContainsAny(string(username), ":\r\n") { + return false, &RequestError{Kind: "invalid Basic username"} + } + req.SetBasicAuth(string(username), string(password)) + } + transport := &http.Transport{TLSClientConfig: config, DisableKeepAlives: true} + defer transport.CloseIdleConnections() + httpClient := &http.Client{Transport: transport, CheckRedirect: func(_ *http.Request, _ []*http.Request) error { return http.ErrUseLastResponse }} + res, err := httpClient.Do(req) + if err != nil { + return false, &RequestError{Kind: "transport or TLS", Cause: err} + } + defer func() { + _ = res.Body.Close() + }() + if res.StatusCode != http.StatusOK { + return false, &RequestError{Kind: "rejected response", StatusCode: res.StatusCode} + } + if _, err = io.Copy(io.Discard, io.LimitReader(res.Body, 64*1024)); err != nil { + return false, &RequestError{Kind: "uncertain response", Cause: err} + } + return true, nil +} + +// SuspensionError preserves the cause for diagnostics and the existing lifecycle recovery policy. +type SuspensionError struct{ Cause error } + +func (e *SuspensionError) Error() string { + if e.Cause == nil { + return "required service suspension failed" + } + var r *RequestError + if errors.As(e.Cause, &r) { + return "required service suspension failed: " + r.Error() + } + var diagnostic *SuspensionCauseError + if errors.As(e.Cause, &diagnostic) { + return "required service suspension failed: " + diagnostic.Error() + } + return "required service suspension failed: suspension failure" +} +func (e *SuspensionError) Unwrap() error { return e.Cause } diff --git a/pkg/probe/http_action_test.go b/pkg/probe/http_action_test.go new file mode 100644 index 000000000..6c5aad658 --- /dev/null +++ b/pkg/probe/http_action_test.go @@ -0,0 +1,301 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ +package probe + +import ( + "context" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "syscall" + "testing" + + coh "github.com/oracle/coherence-operator/api/v1" + cohevents "github.com/oracle/coherence-operator/pkg/events" + "github.com/oracle/coherence-operator/pkg/operator" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/tools/events" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestSanitizedRequestAndSuspensionErrors(t *testing.T) { + sensitive := errors.New("Authorization: Basic dXNlcjpwYXNzd29yZA==; username=operator; password=private; Secret=auth; response body=private") + tests := []struct { + name string + err *RequestError + want string + }{ + {name: "rejected status", err: &RequestError{StatusCode: http.StatusForbidden, Cause: sensitive}, want: "HTTP probe rejected: status 403"}, + {name: "timeout", err: &RequestError{Kind: "transport or TLS", Cause: context.DeadlineExceeded}, want: "HTTP probe failed: transport timeout"}, + {name: "connection refused", err: &RequestError{Kind: "transport or TLS", Cause: &net.OpError{Op: "dial", Err: syscall.ECONNREFUSED}}, want: "HTTP probe failed: connection refused"}, + {name: "unknown CA", err: &RequestError{Kind: "transport or TLS", Cause: x509.UnknownAuthorityError{}}, want: "HTTP probe failed: TLS certificate signed by an unknown or untrusted CA"}, + {name: "hostname mismatch", err: &RequestError{Kind: "transport or TLS", Cause: x509.HostnameError{Certificate: &x509.Certificate{}, Host: "private.example"}}, want: "HTTP probe failed: TLS certificate hostname mismatch"}, + {name: "general transport", err: &RequestError{Kind: "transport or TLS", Cause: sensitive}, want: "HTTP probe failed: transport or TLS failure"}, + {name: "Secret unavailable", err: &RequestError{Kind: "Secret unavailable", Cause: sensitive}, want: "HTTP probe failed: Secret unavailable"}, + {name: "validation", err: &RequestError{Kind: "configuration", Cause: sensitive}, want: "HTTP probe failed: configuration validation failure"}, + {name: "unknown kind", err: &RequestError{Kind: sensitive.Error(), Cause: sensitive}, want: "HTTP probe failed: request failure"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.err.Error(); got != tt.want { + t.Fatalf("RequestError.Error() = %q, want %q", got, tt.want) + } + wrapped := &SuspensionError{Cause: fmt.Errorf("wrapped: %w", tt.err)} + if got := wrapped.Error(); got != "required service suspension failed: "+tt.want { + t.Fatalf("SuspensionError.Error() = %q", got) + } + if strings.Contains(wrapped.Error(), "private") || strings.Contains(wrapped.Error(), "Authorization") || strings.Contains(wrapped.Error(), "dXNlcj") { + t.Fatalf("sensitive data escaped sanitizer: %q", wrapped.Error()) + } + if !errors.Is(wrapped, sensitive) && tt.err.Cause == sensitive { + t.Fatalf("errors.Is() lost the wrapped cause: %v", wrapped) + } + var requestError *RequestError + if !errors.As(wrapped, &requestError) || requestError != tt.err { + t.Fatalf("errors.As() lost RequestError: %v", wrapped) + } + }) + } +} + +func TestSanitizedSuspensionCauseErrors(t *testing.T) { + sensitive := errors.New("Secret auth contained password private") + tests := []struct { + name string + cause *SuspensionCauseError + want string + }{ + { + name: "Pod not ready", + cause: &SuspensionCauseError{Kind: SuspensionCausePodNotReady, PodName: "storage-0", PodPhase: "Running"}, + want: "probe precondition failed: Pod storage-0 is not ready (Running)", + }, + { + name: "Pod phase is sanitized", + cause: &SuspensionCauseError{Kind: SuspensionCausePodNotReady, PodName: "storage-0", PodPhase: sensitive.Error()}, + want: "probe precondition failed: Pod storage-0 is not ready (unknown state)", + }, + { + name: "Pod count mismatch", + cause: &SuspensionCauseError{Kind: SuspensionCausePodCountMismatch, ActualPods: 1, ExpectedPods: 3}, + want: "probe precondition failed: found 1 Pods, expected 3", + }, + { + name: "workload lookup", + cause: &SuspensionCauseError{Kind: SuspensionCauseWorkloadLookupFailed, Cause: sensitive}, + want: "workload state lookup failed", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + wrapped := &SuspensionError{Cause: tt.cause} + want := "required service suspension failed: " + tt.want + if got := wrapped.Error(); got != want { + t.Fatalf("SuspensionError.Error() = %q, want %q", got, want) + } + if strings.Contains(wrapped.Error(), "private") || strings.Contains(wrapped.Error(), "password") { + t.Fatalf("sensitive data escaped sanitizer: %q", wrapped.Error()) + } + if tt.cause.Cause == sensitive && !errors.Is(wrapped, sensitive) { + t.Fatalf("errors.Is() lost the wrapped cause: %v", wrapped) + } + var causeError *SuspensionCauseError + if !errors.As(wrapped, &causeError) || causeError != tt.cause { + t.Fatalf("errors.As() lost SuspensionCauseError: %v", wrapped) + } + }) + } +} + +func actionForServer(t *testing.T, address string) *coh.Probe { + t.Helper() + host, port, err := net.SplitHostPort(strings.TrimPrefix(strings.TrimPrefix(address, "https://"), "http://")) + if err != nil { + t.Fatal(err) + } + n, _ := strconv.Atoi(port) + return &coh.Probe{HTTP: &coh.HTTPAction{Method: "PUT", HTTPEndpoint: coh.HTTPEndpoint{Host: host, Port: intstr.FromInt(n), Path: "/suspend"}}} +} + +func TestHTTPActionDispatchAndRejections(t *testing.T) { + for _, code := range []int{200, 201, 302, 401, 403, 404, 405, 500} { + t.Run(strconv.Itoa(code), func(t *testing.T) { + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if r.Method != "PUT" || r.URL.Path != "/suspend" { + t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) + } + w.Header().Set("Location", "/suspend") + w.WriteHeader(code) + })) + defer server.Close() + p := &CoherenceProbe{} + ok, err := p.RunProbe(context.Background(), corev1.Pod{}, "test", actionForServer(t, server.URL)) + if ok != (code == 200) || (err == nil) != (code == 200) || calls != 1 { + t.Fatalf("ok=%v err=%v calls=%d", ok, err, calls) + } + if err != nil { + var failure *RequestError + if !errors.As(err, &failure) || failure.StatusCode != code { + t.Fatalf("lost status: %v", err) + } + } + }) + } +} + +func TestHTTPActionVerifiedTLSAndCredentials(t *testing.T) { + calls := 0 + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + user, password, ok := r.BasicAuth() + if !ok || user != "operator" || password != "private:password" { + w.WriteHeader(401) + return + } + w.WriteHeader(200) + })) + defer server.Close() + selector := func(key string) corev1.SecretKeySelector { + return corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "auth"}, Key: key} + } + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "auth", Namespace: "test"}, Data: map[string][]byte{ + "user": []byte("operator"), "password": []byte("private:password"), "ca": pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}), + }} + p := &CoherenceProbe{Client: fake.NewClientBuilder().WithObjects(secret).Build()} + pod := corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "test"}} + action := actionForServer(t, server.URL) + action.HTTP.Scheme = corev1.URISchemeHTTPS + action.HTTP.BasicAuth = &coh.HTTPBasicAuth{Username: selector("user"), Password: selector("password")} + action.HTTP.TLS = &coh.HTTPClientTLS{CASecret: selector("ca")} + if ok, err := p.RunProbe(context.Background(), pod, "test", action); ok || err == nil || calls != 0 { + t.Fatalf("authenticated custom host was not rejected before dispatch: ok=%v err=%v calls=%d", ok, err, calls) + } + pod.Status.PodIP = action.HTTP.Host + action.HTTP.Host = "" + if ok, err := p.RunProbe(context.Background(), pod, "test", action); !ok || err != nil { + t.Fatalf("valid request failed: %v", err) + } + action.HTTP.TLS.ServerName = "wrong.example" + if ok, err := p.RunProbe(context.Background(), pod, "test", action); ok || err == nil || err.Error() != "HTTP probe failed: TLS certificate hostname mismatch" { + t.Fatalf("wrong TLS name: ok=%v err=%v", ok, err) + } + action.HTTP.TLS.ServerName = "" + action.HTTP.BasicAuth.Password.Key = "missing" + if ok, err := p.RunProbe(context.Background(), pod, "test", action); ok || err == nil || err.Error() != "HTTP probe failed: Secret key missing or empty" || strings.Contains(err.Error(), "private:password") { + t.Fatalf("missing key: %v %v", ok, err) + } + action.HTTP.BasicAuth.Password.Name = "missing-auth" + action.HTTP.BasicAuth.Password.Key = "password" + if ok, err := p.RunProbe(context.Background(), pod, "test", action); ok || err == nil || err.Error() != "HTTP probe failed: Secret unavailable" { + t.Fatalf("missing Secret: %v %v", ok, err) + } + action.HTTP.Scheme = corev1.URISchemeHTTP + if ok, err := p.RunProbe(context.Background(), pod, "test", action); ok || err == nil { + t.Fatal("accepted plaintext credentials") + } + if calls != 1 { + t.Fatalf("invalid configurations reached server: %d", calls) + } +} + +func TestLegacyEmptyAndGRPCProbesRemainSuccessfulNoOps(t *testing.T) { + p := &CoherenceProbe{} + for _, action := range []*coh.Probe{{}, {ProbeHandler: corev1.ProbeHandler{GRPC: &corev1.GRPCAction{Port: 1234}}}} { + if ok, err := p.RunProbe(context.Background(), corev1.Pod{}, "test", action); !ok || err != nil { + t.Fatalf("legacy no-op action failed: %v %v", ok, err) + } + } + if ok, err := p.RunProbe(context.Background(), corev1.Pod{}, "test", nil); ok || err == nil { + t.Fatalf("nil action succeeded: %v %v", ok, err) + } +} + +func TestSupportedLegacyProbeTakesPrecedenceOverGRPC(t *testing.T) { + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + calls++ + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + host, portText, err := net.SplitHostPort(strings.TrimPrefix(server.URL, "http://")) + if err != nil { + t.Fatal(err) + } + port, err := strconv.Atoi(portText) + if err != nil { + t.Fatal(err) + } + owner := &coh.Coherence{} + eventRecorder := events.NewFakeRecorder(10) + p := &CoherenceProbe{EventRecorder: cohevents.NewOwnedEventRecorder(owner, eventRecorder)} + action := &coh.Probe{ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{Host: host, Port: intstr.FromInt(port), Path: "/"}, + GRPC: &corev1.GRPCAction{Port: 1408}, + }} + + if ok, err := p.RunProbe(context.Background(), corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "storage-0"}}, "test", action); !ok || err != nil { + t.Fatalf("supported HTTP GET action did not take precedence: ok=%t err=%v", ok, err) + } + if calls != 1 { + t.Fatalf("HTTP GET action calls=%d, want 1", calls) + } + for len(eventRecorder.Events) > 0 { + if event := <-eventRecorder.Events; strings.Contains(event, "ProbeActionNoOp") { + t.Fatalf("supported mixed probe emitted no-op warning: %q", event) + } + } +} + +func TestNewHTTPProbeCannotBeCombinedWithLegacyAction(t *testing.T) { + p := &CoherenceProbe{} + action := &coh.Probe{ + HTTP: &coh.HTTPAction{}, + ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"true"}}}, + } + if ok, err := p.RunProbe(context.Background(), corev1.Pod{}, "test", action); ok || err == nil { + t.Fatalf("mixed new and legacy actions succeeded: %v %v", ok, err) + } +} + +func TestRejectedSuspensionDoesNotContactSecondPod(t *testing.T) { + for _, code := range []int{401, 403, 404, 405, 302} { + t.Run(strconv.Itoa(code), func(t *testing.T) { + calls := 0 + first := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(code) })) + defer first.Close() + second := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calls++; w.WriteHeader(200) })) + defer second.Close() + pods := corev1.PodList{} + for i, address := range []string{first.URL, second.URL} { + action := actionForServer(t, address) + pods.Items = append(pods.Items, corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: strconv.Itoa(i), Labels: map[string]string{operator.LabelTestHostName: action.HTTP.Host, operator.LabelTestHealthPort: action.HTTP.Port.String()}}, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}}}) + } + sts := &appsv1.StatefulSet{Spec: appsv1.StatefulSetSpec{Replicas: ptr.To(int32(2))}} + deployment := &coh.Coherence{} + p := &CoherenceProbe{EventRecorder: cohevents.NewOwnedEventRecorder(deployment, events.NewFakeRecorder(20))} + ok, err := p.executeProbeForPods(context.Background(), sts, "test", deployment.Spec.GetDefaultSuspendProbe(), pods, pods, probeExecutionOptions{suspension: true}) + var rejected *RequestError + if ok || !errors.As(err, &rejected) || rejected.StatusCode != code || calls != 0 { + t.Fatalf("ok=%v err=%v second calls=%d", ok, err, calls) + } + }) + } +} diff --git a/pkg/probe/http_test.go b/pkg/probe/http_test.go new file mode 100644 index 000000000..df9c1e0c6 --- /dev/null +++ b/pkg/probe/http_test.go @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ + +package probe + +import ( + "errors" + "io" + "net/http" + "net/url" + "strings" + "testing" +) + +type fakeHTTPClient struct { + response *http.Response + err error +} + +func (f fakeHTTPClient) Do(*http.Request) (*http.Response, error) { + return f.response, f.err +} + +type timeoutError struct{} + +func (timeoutError) Error() string { return "timed out" } +func (timeoutError) Timeout() bool { return true } + +func TestDoHTTPProbePreservesFailureClassification(t *testing.T) { + target, err := url.Parse("http://127.0.0.1/test") + if err != nil { + t.Fatal(err) + } + + t.Run("rejected response", func(t *testing.T) { + client := fakeHTTPClient{response: &http.Response{ + StatusCode: http.StatusServiceUnavailable, + Body: io.NopCloser(strings.NewReader("unavailable")), + }} + result, _, err := DoHTTPProbe(target, nil, client) + failure := &RequestError{} + if result != Failure || !errors.As(err, &failure) || failure.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("result=%q err=%v", result, err) + } + }) + + t.Run("transport timeout", func(t *testing.T) { + result, _, err := DoHTTPProbe(target, nil, fakeHTTPClient{err: timeoutError{}}) + failure := &RequestError{} + if result != Failure || !errors.As(err, &failure) || failure.Kind != "transport timeout" { + t.Fatalf("result=%q err=%v", result, err) + } + }) +} diff --git a/pkg/probe/probe.go b/pkg/probe/probe.go index 9762337f5..9a34fbce0 100644 --- a/pkg/probe/probe.go +++ b/pkg/probe/probe.go @@ -8,6 +8,7 @@ package probe import ( "context" + "encoding/json" "fmt" coh "github.com/oracle/coherence-operator/api/v1" "github.com/oracle/coherence-operator/pkg/events" @@ -49,6 +50,21 @@ type CoherenceProbe struct { translatePort func(name string, port int) int } +// ResolvedScalingProbe retains whether a scaling probe was generated by the Operator or explicitly configured. +type ResolvedScalingProbe struct { + Probe *coh.Probe + Generated bool +} + +// ResolveScalingProbe returns the scaling probe and its origin as a single runtime value. +func ResolveScalingProbe(spec *coh.CoherenceStatefulSetResourceSpec) ResolvedScalingProbe { + if spec == nil { + spec = &coh.CoherenceStatefulSetResourceSpec{} + } + generated := spec.Scaling == nil || spec.Scaling.Probe == nil + return ResolvedScalingProbe{Probe: spec.GetScalingProbe(), Generated: generated} +} + func (in *CoherenceProbe) SetGetPodHostName(fn func(pod corev1.Pod) string) { if in == nil { return @@ -105,8 +121,12 @@ func (in *CoherenceProbe) IsStatusHA(ctx context.Context, deployment coh.Coheren spec, found := deployment.GetStatefulSetSpec() if found { - p := spec.GetScalingProbe() - return in.ExecuteProbe(ctx, sts, deployment.GetWkaServiceName(), p) + resolved := ResolveScalingProbe(spec) + p := resolved.Probe + if p.HTTP != nil && p.HTTP.Method != "" && p.HTTP.Method != "GET" { + return false + } + return in.ExecuteScalingProbe(ctx, sts, deployment.GetWkaServiceName(), resolved) } return true } @@ -125,11 +145,17 @@ const ( // iota is reset to 0 // ALl Pods must be in the ready state // All Pods must pass the StatusHA check func (in *CoherenceProbe) SuspendServices(ctx context.Context, deployment coh.CoherenceResource, sts *appsv1.StatefulSet) ServiceSuspendStatus { + status, _ := in.SuspendServicesWithError(ctx, deployment, sts) + return status +} + +// SuspendServicesWithError preserves the cause for reconciliation policy. +func (in *CoherenceProbe) SuspendServicesWithError(ctx context.Context, deployment coh.CoherenceResource, sts *appsv1.StatefulSet) (ServiceSuspendStatus, error) { ns := deployment.GetNamespace() name := deployment.GetName() if deployment.GetType() != coh.CoherenceTypeStatefulSet { - return ServiceSuspendSkipped + return ServiceSuspendSkipped, nil } c := deployment.(*coh.Coherence) @@ -140,23 +166,46 @@ func (in *CoherenceProbe) SuspendServices(ctx context.Context, deployment coh.Co sts.Name, operator.FlagSkipServiceSuspend) log.Info(msg, "Namespace", ns, "Name", name) in.EventRecorder.Warn("ServiceSuspendSkipped", msg) - return ServiceSuspendSkipped + return ServiceSuspendSkipped, nil } if !stsSpec.IsSuspendServicesOnShutdown() { msg := fmt.Sprintf("Skipping suspension of Coherence services in StatefulSet %s, spec.SuspendServicesOnShutdown is set to false", sts.Name) log.Info(msg, "Namespace", ns, "Name", name) in.EventRecorder.Warn("ServiceSuspendSkipped", msg) - return ServiceSuspendSkipped + return ServiceSuspendSkipped, nil } + if err := stsSpec.HealthMutatorConnection.Validate(); err != nil { + return in.suspensionFailure(sts, &RequestError{Kind: "configuration", Cause: err}) + } + if stsSpec.HealthMutatorConnection != nil && stsSpec.HealthMutatorConnection.Secure && stsSpec.SuspendProbe != nil { + return in.suspensionFailure(sts, &RequestError{Kind: "managed security requires the generated suspend action"}) + } log.Info("Suspending Coherence services in StatefulSet "+sts.Name, "Namespace", ns, "Name", name) - if in.ExecuteProbe(ctx, sts, deployment.GetWkaServiceName(), stsSpec.GetSuspendProbe()) { + pods, err := in.GetPodsForStatefulSet(ctx, sts) + if err != nil { + return in.suspensionFailure(sts, &SuspensionCauseError{Kind: SuspensionCauseWorkloadLookupFailed, Cause: err}) + } + suspendProbe := stsSpec.GetSuspendProbe() + ok, err := in.executeProbeForPods(ctx, sts, deployment.GetWkaServiceName(), suspendProbe, pods, pods, probeExecutionOptions{ + suspension: true, + purpose: "suspend probe", + }) + if ok && err == nil { + if _, noOp := legacyNoOpReason(suspendProbe); noOp { + return ServiceSuspendSkipped, nil + } in.EventRecorder.Infof("ServiceSuspended", "suspended Coherence services in StatefulSet %s", sts.Name) - return ServiceSuspendSuccessful + return ServiceSuspendSuccessful, nil } - in.EventRecorder.Warnf("ServiceSuspendFailed", "failed to suspend Coherence services in StatefulSet %s", sts.Name) - return ServiceSuspendFailed + return in.suspensionFailure(sts, err) +} + +func (in *CoherenceProbe) suspensionFailure(sts *appsv1.StatefulSet, cause error) (ServiceSuspendStatus, error) { + err := &SuspensionError{Cause: cause} + in.EventRecorder.Warnf("ServiceSuspendFailed", "failed to suspend Coherence services in StatefulSet %s: %s", sts.Name, err.Error()) + return ServiceSuspendFailed, err } func (in *CoherenceProbe) GetPodsForStatefulSet(ctx context.Context, sts *appsv1.StatefulSet) (corev1.PodList, error) { @@ -170,16 +219,54 @@ func (in *CoherenceProbe) GetPodsForStatefulSet(ctx context.Context, sts *appsv1 } func (in *CoherenceProbe) ExecuteProbe(ctx context.Context, sts *appsv1.StatefulSet, svc string, probe *coh.Probe) bool { + return in.executeProbe(ctx, sts, svc, probe, probeExecutionOptions{purpose: "probe action"}) +} + +// ExecuteScalingProbe executes a resolved scaling probe while preserving whether the Operator generated it. +func (in *CoherenceProbe) ExecuteScalingProbe(ctx context.Context, sts *appsv1.StatefulSet, svc string, resolved ResolvedScalingProbe) bool { + return in.executeProbe(ctx, sts, svc, resolved.Probe, probeExecutionOptions{ + generatedHealthProbe: resolved.Generated, + purpose: "StatusHA probe", + }) +} + +func (in *CoherenceProbe) executeProbe(ctx context.Context, sts *appsv1.StatefulSet, svc string, probe *coh.Probe, options probeExecutionOptions) bool { pods, err := in.GetPodsForStatefulSet(ctx, sts) if err != nil { log.Error(err, "Error getting list of Pods for StatefulSet "+sts.Name) in.EventRecorder.Infof("CheckStatusHA", "Failed to get pods for StatefulSet %s: %s", sts.Name, err.Error()) return false } - return in.ExecuteProbeForSubSetOfPods(ctx, sts, svc, probe, pods, pods) + ok, _ := in.executeProbeForPods(ctx, sts, svc, probe, pods, pods, options) + return ok } func (in *CoherenceProbe) ExecuteProbeForSubSetOfPods(ctx context.Context, sts *appsv1.StatefulSet, svc string, probe *coh.Probe, stsPods, pods corev1.PodList) bool { + ok, _ := in.executeProbeForPods(ctx, sts, svc, probe, stsPods, pods, probeExecutionOptions{purpose: "probe action"}) + return ok +} + +// ExecuteScalingProbeForSubSetOfPods executes a resolved scaling probe against a subset of Pods. +func (in *CoherenceProbe) ExecuteScalingProbeForSubSetOfPods(ctx context.Context, sts *appsv1.StatefulSet, svc string, resolved ResolvedScalingProbe, stsPods, pods corev1.PodList) bool { + ok, _ := in.executeProbeForPods(ctx, sts, svc, resolved.Probe, stsPods, pods, probeExecutionOptions{ + generatedHealthProbe: resolved.Generated, + purpose: "StatusHA probe", + }) + return ok +} + +type probeExecutionOptions struct { + suspension bool + generatedHealthProbe bool + purpose string +} + +func (in *CoherenceProbe) executeProbeForPods(ctx context.Context, sts *appsv1.StatefulSet, svc string, probe *coh.Probe, stsPods, pods corev1.PodList, options probeExecutionOptions) (bool, error) { + if options.suspension { + if err := probe.ValidateAction(); err != nil { + return false, &RequestError{Kind: "configuration", Cause: err} + } + } logger := log.WithValues("Namespace", sts.GetNamespace(), "Name", sts.GetName()) // All Pods must be in the Running Phase @@ -188,7 +275,7 @@ func (in *CoherenceProbe) ExecuteProbeForSubSetOfPods(ctx context.Context, sts * msg := fmt.Sprintf("Cannot execute probe, one or more Pods is not in a ready state - %s (%v) ", pod.Name, phase) logger.Info(msg) in.EventRecorder.Warn("CheckStatusHA", msg) - return false + return false, &SuspensionCauseError{Kind: SuspensionCausePodNotReady, PodName: pod.Name, PodPhase: phase} } } @@ -198,17 +285,17 @@ func (in *CoherenceProbe) ExecuteProbeForSubSetOfPods(ctx context.Context, sts * msg := fmt.Sprintf("Skipping StatusHA check, no Pods found in StatefulSet %s", sts.Name) logger.Info(msg) in.EventRecorder.Info("CheckStatusHA", msg) - return true + return true, nil case sts.Spec.Replicas == nil && count != 1: msg := fmt.Sprintf("Pod count of %d does not yet match StatefulSet replica count: 1", count) logger.Info(msg) in.EventRecorder.Info("CheckStatusHA", msg) - return false + return false, &SuspensionCauseError{Kind: SuspensionCausePodCountMismatch, ActualPods: count, ExpectedPods: 1} case sts.Spec.Replicas != nil && count != *sts.Spec.Replicas: msg := fmt.Sprintf("Pod count of %d does not yet match StatefulSet replica count: %d", count, *sts.Spec.Replicas) in.EventRecorder.Info("CheckStatusHA", msg) logger.Info(msg) - return false + return false, &SuspensionCauseError{Kind: SuspensionCausePodCountMismatch, ActualPods: count, ExpectedPods: *sts.Spec.Replicas} } for _, pod := range pods.Items { @@ -217,12 +304,50 @@ func (in *CoherenceProbe) ExecuteProbeForSubSetOfPods(ctx context.Context, sts * log.Info("Using pod " + pod.Name + " to execute probe") } - ha, err := in.RunProbe(ctx, pod, svc, probe) + active := probe + if raw := pod.Annotations[coh.HealthMutatorAnnotation]; raw != "" { + connection := &coh.HealthMutatorConnection{} + if err := json.Unmarshal([]byte(raw), connection); err != nil { + return false, &RequestError{Kind: "unknown active Pod connection"} + } + if err := connection.Validate(); err != nil { + return false, &RequestError{Kind: "invalid active Pod connection", Cause: err} + } + if connection.Secure { + active = probe.DeepCopy() + if options.suspension { + if active.HTTP == nil || active.HTTP.Path != "/suspend" { + return false, &RequestError{Kind: "managed security requires the generated suspend action"} + } + active.HTTP.Scheme = corev1.URISchemeHTTPS + active.HTTP.BasicAuth = connection.BasicAuth + active.HTTP.TLS = connection.TLS + } else if options.generatedHealthProbe && active.HTTPGet != nil { + active.HTTPGet.Scheme = corev1.URISchemeHTTPS + } + } + } else { + for _, container := range pod.Spec.Containers { + for _, env := range container.Env { + if env.Name == coh.HealthMutatorEnv { + return false, &RequestError{Kind: "missing active Pod connection"} + } + } + } + } + ha, err := in.runProbe(ctx, pod, svc, active, options.purpose) if err == nil { msg := fmt.Sprintf("Executed probe using pod %s result=%t", pod.Name, ha) log.Info(msg) in.EventRecorder.Info("CheckStatusHA", msg) - return ha + return ha, nil + } + // HTTP probe failures are authoritative. In particular, a transport + // failure from one Pod must not be replaced by a later Pod's success at + // a StatusHA safety gate. Preserve fallback only for non-HTTP execution + // errors, matching the behavior before HTTP failures became typed. + if options.suspension || probe.HTTP != nil || probe.HTTPGet != nil { + return false, err } msg := fmt.Sprintf("Execute probe using pod %s (%t) error %s", pod.Name, ha, err.Error()) in.EventRecorder.Warn("CheckStatusHA", msg) @@ -234,7 +359,7 @@ func (in *CoherenceProbe) ExecuteProbeForSubSetOfPods(ctx context.Context, sts * } } - return false + return false, &SuspensionCauseError{Kind: SuspensionCauseProbeUnavailable} } // IsPodReady determines whether the specified Pods are in the Ready state. @@ -252,7 +377,25 @@ func (in *CoherenceProbe) IsPodReady(pod corev1.Pod) (bool, string) { } func (in *CoherenceProbe) RunProbe(ctx context.Context, pod corev1.Pod, svc string, handler *coh.Probe) (bool, error) { + return in.runProbe(ctx, pod, svc, handler, "probe action") +} + +func (in *CoherenceProbe) runProbe(ctx context.Context, pod corev1.Pod, svc string, handler *coh.Probe, purpose string) (bool, error) { + if err := handler.ValidateAction(); err != nil { + return false, &RequestError{Kind: "configuration", Cause: err} + } + if reason, noOp := legacyNoOpReason(handler); noOp { + if purpose == "" { + purpose = "probe action" + } + msg := fmt.Sprintf("Configured %s for Pod %s has no executable Operator action (%s); no request or command was sent, and the probe is treated as successful for backward compatibility", purpose, pod.Name, reason) + log.Info(msg) + in.EventRecorder.Warn("ProbeActionNoOp", msg) + return true, nil + } switch { + case handler.HTTP != nil: + return in.ProbeUsingHTTPAction(ctx, pod, svc, handler) case handler.Exec != nil: return in.ProbeUsingExec(ctx, pod, handler) case handler.HTTPGet != nil: @@ -264,6 +407,18 @@ func (in *CoherenceProbe) RunProbe(ctx context.Context, pod corev1.Pod, svc stri } } +// legacyNoOpReason identifies legacy probe definitions that the Operator has +// historically accepted as successful even though it cannot execute an action. +func legacyNoOpReason(handler *coh.Probe) (string, bool) { + if handler == nil || handler.HTTP != nil || handler.Exec != nil || handler.HTTPGet != nil || handler.TCPSocket != nil { + return "", false + } + if handler.GRPC != nil { + return "gRPC probe actions are not supported", true + } + return "the probe handler is empty", true +} + func (in *CoherenceProbe) ProbeUsingExec(ctx context.Context, pod corev1.Pod, handler *coh.Probe) (bool, error) { req := &mgmt.ExecRequest{ Pod: pod.Name, @@ -331,9 +486,9 @@ func (in *CoherenceProbe) ProbeUsingHTTP(pod corev1.Pod, svc string, handler *co } p := NewHTTPProbe() - result, s, err := p.Probe(u, header, handler.GetTimeout()) + result, _, err := p.Probe(u, header, handler.GetTimeout()) - log.Info("Executed HTTP Probe", "URL", u, "Result", fmt.Sprintf("%v", result), "Msg", s, "Error", err) + log.Info("Executed HTTP Probe", "Result", fmt.Sprintf("%v", result), "Error", err) return result == Success, err } diff --git a/pkg/probe/probe_test.go b/pkg/probe/probe_test.go index 6df2d6038..5de8ddf55 100644 --- a/pkg/probe/probe_test.go +++ b/pkg/probe/probe_test.go @@ -8,9 +8,12 @@ package probe import ( "context" + "encoding/json" + "errors" "net" "net/http" "net/http/httptest" + "strconv" "strings" "testing" @@ -21,6 +24,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/tools/events" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -94,6 +98,7 @@ func TestSuspendServicesReportsResult(t *testing.T) { responseCode int wantStatus ServiceSuspendStatus wantEvent string + wantError string rejectEvent string }{ { @@ -107,15 +112,21 @@ func TestSuspendServicesReportsResult(t *testing.T) { name: "failed suspend", responseCode: http.StatusInternalServerError, wantStatus: ServiceSuspendFailed, - wantEvent: "Warning ServiceSuspendFailed failed to suspend Coherence services in StatefulSet test-deployment", + wantEvent: "Warning ServiceSuspendFailed failed to suspend Coherence services in StatefulSet test-deployment: required service suspension failed: HTTP probe rejected: status 500", + wantError: "required service suspension failed: HTTP probe rejected: status 500", rejectEvent: "ServiceSuspended", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + const responseBody = "private response body containing Authorization: Basic dXNlcjpwYXNzd29yZA==" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPut { + t.Errorf("generated suspension used %s", r.Method) + } w.WriteHeader(tt.responseCode) + _, _ = w.Write([]byte(responseBody)) })) defer server.Close() @@ -170,8 +181,16 @@ func TestSuspendServicesReportsResult(t *testing.T) { EventRecorder: cohevents.NewOwnedEventRecorder(deployment, recorder), } - if got := coherenceProbe.SuspendServices(context.Background(), deployment, sts); got != tt.wantStatus { - t.Fatalf("SuspendServices() = %v, want %v", got, tt.wantStatus) + got, suspendErr := coherenceProbe.SuspendServicesWithError(context.Background(), deployment, sts) + if got != tt.wantStatus { + t.Fatalf("SuspendServicesWithError() = %v, want %v", got, tt.wantStatus) + } + if tt.wantError == "" { + if suspendErr != nil { + t.Fatalf("SuspendServicesWithError() returned %v", suspendErr) + } + } else if suspendErr == nil || suspendErr.Error() != tt.wantError { + t.Fatalf("SuspendServicesWithError() error = %v, want %q", suspendErr, tt.wantError) } var recordedEvents []string @@ -184,10 +203,453 @@ func TestSuspendServicesReportsResult(t *testing.T) { if containsEventWithReason(recordedEvents, tt.rejectEvent) { t.Errorf("events %q unexpectedly contain reason %q", recordedEvents, tt.rejectEvent) } + if containsEventWithReason(recordedEvents, "ProbeActionNoOp") { + t.Errorf("events %q unexpectedly report a no-op", recordedEvents) + } + if suspendErr != nil && strings.Contains(suspendErr.Error(), responseBody) { + t.Fatalf("response body escaped in error: %v", suspendErr) + } + for _, event := range recordedEvents { + if strings.Contains(event, responseBody) || strings.Contains(event, "dXNlcj") { + t.Fatalf("response body or credential escaped in event: %q", event) + } + } }) } } +func TestSuspendServicesReportsLegacyNoOpAsSkipped(t *testing.T) { + originalSkipServiceSuspend := viper.GetBool(operator.FlagSkipServiceSuspend) + viper.Set(operator.FlagSkipServiceSuspend, false) + t.Cleanup(func() { + viper.Set(operator.FlagSkipServiceSuspend, originalSkipServiceSuspend) + }) + + for _, tt := range []struct { + name string + probe *coh.Probe + reason string + }{ + {name: "empty handler", probe: &coh.Probe{}, reason: "the probe handler is empty"}, + {name: "gRPC handler", probe: &coh.Probe{ProbeHandler: corev1.ProbeHandler{GRPC: &corev1.GRPCAction{Port: 1408}}}, reason: "gRPC probe actions are not supported"}, + } { + t.Run(tt.name, func(t *testing.T) { + labels := map[string]string{"app": "test-deployment"} + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "test-deployment-0", Namespace: "test-namespace", Labels: labels}, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}, + }, + } + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "test-deployment", Namespace: "test-namespace"}, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(int32(1)), + Selector: &metav1.LabelSelector{MatchLabels: labels}, + }, + } + deployment := &coh.Coherence{ + ObjectMeta: metav1.ObjectMeta{Name: "test-deployment", Namespace: "test-namespace"}, + Spec: coh.CoherenceStatefulSetResourceSpec{SuspendProbe: tt.probe}, + } + eventRecorder := events.NewFakeRecorder(10) + coherenceProbe := CoherenceProbe{ + Client: fake.NewClientBuilder().WithObjects(pod).Build(), + EventRecorder: cohevents.NewOwnedEventRecorder(deployment, eventRecorder), + } + + status, err := coherenceProbe.SuspendServicesWithError(context.Background(), deployment, sts) + if status != ServiceSuspendSkipped || err != nil { + t.Fatalf("status=%v error=%v, want skipped without error", status, err) + } + + var recordedEvents []string + for len(eventRecorder.Events) > 0 { + recordedEvents = append(recordedEvents, <-eventRecorder.Events) + } + if !containsEventWithReason(recordedEvents, "ProbeActionNoOp") { + t.Fatalf("events %q do not contain ProbeActionNoOp", recordedEvents) + } + if containsEventWithReason(recordedEvents, "ServiceSuspended") { + t.Fatalf("events %q incorrectly report successful suspension", recordedEvents) + } + if !containsEventText(recordedEvents, "Configured suspend probe", tt.reason, "no request or command was sent") { + t.Fatalf("events %q do not explain the no-op", recordedEvents) + } + }) + } +} + +func TestSuspendServicesReportsPreconditionCause(t *testing.T) { + originalSkipServiceSuspend := viper.GetBool(operator.FlagSkipServiceSuspend) + viper.Set(operator.FlagSkipServiceSuspend, false) + t.Cleanup(func() { + viper.Set(operator.FlagSkipServiceSuspend, originalSkipServiceSuspend) + }) + + tests := []struct { + name string + podReady bool + replicas int32 + wantCause string + }{ + { + name: "Pod not ready", + podReady: false, + replicas: 1, + wantCause: "probe precondition failed: Pod test-deployment-0 is not ready (Running)", + }, + { + name: "Pod count mismatch", + podReady: true, + replicas: 2, + wantCause: "probe precondition failed: found 1 Pods, expected 2", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + conditions := []corev1.PodCondition{} + if tt.podReady { + conditions = append(conditions, corev1.PodCondition{Type: corev1.PodReady, Status: corev1.ConditionTrue}) + } + labels := map[string]string{"app": "test-deployment"} + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "test-deployment-0", Namespace: "test-namespace", Labels: labels}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: conditions}, + } + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "test-deployment", Namespace: "test-namespace"}, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(tt.replicas), + Selector: &metav1.LabelSelector{MatchLabels: labels}, + }, + } + deployment := &coh.Coherence{ObjectMeta: metav1.ObjectMeta{Name: "test-deployment", Namespace: "test-namespace"}} + recorder := events.NewFakeRecorder(10) + coherenceProbe := CoherenceProbe{ + Client: fake.NewClientBuilder().WithObjects(pod).Build(), + EventRecorder: cohevents.NewOwnedEventRecorder(deployment, recorder), + } + + status, err := coherenceProbe.SuspendServicesWithError(context.Background(), deployment, sts) + wantError := "required service suspension failed: " + tt.wantCause + if status != ServiceSuspendFailed || err == nil || err.Error() != wantError { + t.Fatalf("status=%v error=%v, want %v and %q", status, err, ServiceSuspendFailed, wantError) + } + wantEvent := "Warning ServiceSuspendFailed failed to suspend Coherence services in StatefulSet test-deployment: " + wantError + var recordedEvents []string + for len(recorder.Events) > 0 { + recordedEvents = append(recordedEvents, <-recorder.Events) + } + if !containsEvent(recordedEvents, wantEvent) { + t.Fatalf("events %q do not contain %q", recordedEvents, wantEvent) + } + }) + } +} + +func TestStatusHARejectedResponseDoesNotFallThrough(t *testing.T) { + for _, status := range []int{http.StatusBadRequest, http.StatusServiceUnavailable} { + t.Run(strconv.Itoa(status), func(t *testing.T) { + secondCalls := 0 + first := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/ha" { + t.Errorf("probe path = %q, want /ha", r.URL.Path) + } + w.WriteHeader(status) + })) + defer first.Close() + second := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + secondCalls++ + w.WriteHeader(http.StatusOK) + })) + defer second.Close() + + pods := corev1.PodList{Items: []corev1.Pod{ + readyPodForProbeServer(t, "storage-0", first.URL), + readyPodForProbeServer(t, "storage-1", second.URL), + }} + sts := &appsv1.StatefulSet{Spec: appsv1.StatefulSetSpec{Replicas: ptr.To(int32(len(pods.Items)))}} + deployment := &coh.Coherence{} + coherenceProbe := CoherenceProbe{ + EventRecorder: cohevents.NewOwnedEventRecorder(deployment, events.NewFakeRecorder(10)), + } + + ok, err := coherenceProbe.executeProbeForPods( + context.Background(), sts, "test", deployment.Spec.GetDefaultScalingProbe(), pods, pods, probeExecutionOptions{}) + var requestError *RequestError + if ok || !errors.As(err, &requestError) || requestError.StatusCode != status { + t.Fatalf("ok=%v err=%v, want rejected status %d", ok, err, status) + } + if secondCalls != 0 { + t.Fatalf("second Pod was probed %d times after status %d", secondCalls, status) + } + }) + } +} + +func TestStatusHATransportFailureDoesNotFallThrough(t *testing.T) { + first := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + firstURL := first.URL + first.Close() + + secondCalls := 0 + second := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + secondCalls++ + w.WriteHeader(http.StatusOK) + })) + defer second.Close() + + pods := corev1.PodList{Items: []corev1.Pod{ + readyPodForProbeServer(t, "storage-0", firstURL), + readyPodForProbeServer(t, "storage-1", second.URL), + }} + sts := &appsv1.StatefulSet{Spec: appsv1.StatefulSetSpec{Replicas: ptr.To(int32(len(pods.Items)))}} + deployment := &coh.Coherence{} + coherenceProbe := CoherenceProbe{ + EventRecorder: cohevents.NewOwnedEventRecorder(deployment, events.NewFakeRecorder(10)), + } + + ok, err := coherenceProbe.executeProbeForPods( + context.Background(), sts, "test", deployment.Spec.GetDefaultScalingProbe(), pods, pods, probeExecutionOptions{}) + var requestError *RequestError + if ok || !errors.As(err, &requestError) || requestError.Kind != "transport" { + t.Fatalf("ok=%v err=%v, want authoritative transport failure", ok, err) + } + if secondCalls != 0 { + t.Fatalf("second Pod was probed %d times after transport failure", secondCalls) + } +} + +func TestResolveScalingProbePreservesSource(t *testing.T) { + for _, spec := range []*coh.CoherenceStatefulSetResourceSpec{ + nil, + {}, + {Scaling: &coh.ScalingSpec{}}, + } { + resolved := ResolveScalingProbe(spec) + if !resolved.Generated || resolved.Probe == nil || resolved.Probe.HTTPGet == nil || resolved.Probe.HTTPGet.Path != "/ha" { + t.Fatalf("default scaling probe was not identified as generated: %#v", resolved) + } + } + + custom := &coh.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{ + Path: "/ha", + Port: intstr.FromString(coh.PortNameHealth), + Scheme: corev1.URISchemeHTTP, + }}} + resolved := ResolveScalingProbe(&coh.CoherenceStatefulSetResourceSpec{Scaling: &coh.ScalingSpec{Probe: custom}}) + if resolved.Generated || resolved.Probe != custom { + t.Fatalf("explicit scaling probe lost its source or identity: %#v", resolved) + } +} + +func TestStatusHALegacyNoOpWarnsAndSucceeds(t *testing.T) { + for _, tt := range []struct { + name string + probe *coh.Probe + reason string + }{ + {name: "empty handler", probe: &coh.Probe{}, reason: "the probe handler is empty"}, + {name: "gRPC handler", probe: &coh.Probe{ProbeHandler: corev1.ProbeHandler{GRPC: &corev1.GRPCAction{Port: 1408}}}, reason: "gRPC probe actions are not supported"}, + } { + t.Run(tt.name, func(t *testing.T) { + labels := map[string]string{"app": "storage"} + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "storage-0", Namespace: "test", Labels: labels}, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}, + }, + } + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "storage", Namespace: "test"}, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(int32(1)), + Selector: &metav1.LabelSelector{MatchLabels: labels}, + }, + } + deployment := &coh.Coherence{ObjectMeta: metav1.ObjectMeta{Name: "storage", Namespace: "test"}} + eventRecorder := events.NewFakeRecorder(10) + coherenceProbe := CoherenceProbe{ + Client: fake.NewClientBuilder().WithObjects(pod).Build(), + EventRecorder: cohevents.NewOwnedEventRecorder(deployment, eventRecorder), + } + + if !coherenceProbe.ExecuteScalingProbe(context.Background(), sts, "storage-wka", ResolvedScalingProbe{Probe: tt.probe}) { + t.Fatal("legacy no-op StatusHA probe did not retain successful compatibility behavior") + } + + var recordedEvents []string + for len(eventRecorder.Events) > 0 { + recordedEvents = append(recordedEvents, <-eventRecorder.Events) + } + if !containsEventWithReason(recordedEvents, "ProbeActionNoOp") || + !containsEventText(recordedEvents, "Configured StatusHA probe", tt.reason, "no request or command was sent") { + t.Fatalf("events %q do not explain the StatusHA no-op", recordedEvents) + } + }) + } +} + +func TestLegacyNoOpClassification(t *testing.T) { + for _, tt := range []struct { + name string + probe *coh.Probe + wantNoOp bool + }{ + {name: "nil", probe: nil}, + {name: "empty", probe: &coh.Probe{}, wantNoOp: true}, + {name: "gRPC", probe: &coh.Probe{ProbeHandler: corev1.ProbeHandler{GRPC: &corev1.GRPCAction{Port: 1408}}}, wantNoOp: true}, + {name: "HTTP", probe: &coh.Probe{HTTP: &coh.HTTPAction{}}}, + {name: "HTTP GET", probe: &coh.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{}}}}, + {name: "exec", probe: &coh.Probe{ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{}}}}, + {name: "TCP", probe: &coh.Probe{ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{}}}}, + {name: "exec with gRPC", probe: &coh.Probe{ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{}, GRPC: &corev1.GRPCAction{Port: 1408}}}}, + } { + t.Run(tt.name, func(t *testing.T) { + _, got := legacyNoOpReason(tt.probe) + if got != tt.wantNoOp { + t.Fatalf("legacyNoOpReason() no-op=%t, want %t", got, tt.wantNoOp) + } + }) + } +} + +func TestStatusHAUsesHTTPSOnlyForGeneratedProbe(t *testing.T) { + t.Run("generated secure probe uses HTTPS", func(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/ha" { + t.Errorf("probe path = %q, want /ha", r.URL.Path) + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + assertStatusHAForServer(t, server.URL, nil, true) + }) + + t.Run("generated non-secure probe keeps HTTP", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/ha" { + t.Errorf("probe path = %q, want /ha", r.URL.Path) + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + assertStatusHAForServer(t, server.URL, nil, false) + }) + + for _, port := range []string{"named health port", "numeric application port"} { + t.Run("explicit HTTP /ha on "+port+" remains HTTP", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/ha" { + t.Errorf("probe path = %q, want /ha", r.URL.Path) + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + _, portText, err := net.SplitHostPort(strings.TrimPrefix(server.URL, "http://")) + if err != nil { + t.Fatal(err) + } + probePort := intstr.FromString(coh.PortNameHealth) + if port == "numeric application port" { + portNumber, err := strconv.Atoi(portText) + if err != nil { + t.Fatal(err) + } + probePort = intstr.FromInt(portNumber) + } + custom := &coh.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{ + Path: "/ha", + Port: probePort, + Scheme: corev1.URISchemeHTTP, + }}} + assertStatusHAForServer(t, server.URL, custom, true) + }) + } +} + +func assertStatusHAForServer(t *testing.T, address string, custom *coh.Probe, secure bool) { + t.Helper() + pod := readyPodForProbeServer(t, "storage-0", address) + pod.Namespace = "test" + pod.Labels["app"] = "storage" + if secure { + pod.Annotations = map[string]string{coh.HealthMutatorAnnotation: secureHealthMutatorAnnotation(t)} + } + sts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: "storage", Namespace: "test"}, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(int32(1)), + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "storage"}}, + }, + } + deployment := &coh.Coherence{ObjectMeta: metav1.ObjectMeta{Name: "storage", Namespace: "test"}} + if custom != nil { + deployment.Spec.Scaling = &coh.ScalingSpec{Probe: custom} + } + coherenceProbe := CoherenceProbe{ + Client: fake.NewClientBuilder().WithObjects(&pod).Build(), + EventRecorder: cohevents.NewOwnedEventRecorder(deployment, events.NewFakeRecorder(10)), + } + if !coherenceProbe.IsStatusHA(context.Background(), deployment, sts) { + t.Fatalf("StatusHA failed for secure=%t custom=%#v address=%s", secure, custom, address) + } +} + +func secureHealthMutatorAnnotation(t *testing.T) string { + t.Helper() + selector := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "health"}, Key: "value"} + connection := &coh.HealthMutatorConnection{ + Secure: true, + BasicAuth: &coh.HTTPBasicAuth{Username: selector, Password: selector}, + TLS: &coh.HTTPClientTLS{CASecret: selector, ServerName: "health.example"}, + ServerTLS: &coh.SSLSpec{ + Enabled: ptr.To(true), + KeyStore: ptr.To("server.jks"), + KeyStorePasswordFile: ptr.To("store-password"), + KeyPasswordFile: ptr.To("key-password"), + }, + } + if err := connection.Validate(); err != nil { + t.Fatalf("invalid secure Health test connection: %v", err) + } + data, err := json.Marshal(connection) + if err != nil { + t.Fatal(err) + } + return string(data) +} + +func readyPodForProbeServer(t *testing.T, name, address string) corev1.Pod { + t.Helper() + host, port, err := net.SplitHostPort(strings.TrimPrefix(strings.TrimPrefix(address, "https://"), "http://")) + if err != nil { + t.Fatalf("failed to split test server address: %v", err) + } + return corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + operator.LabelTestHostName: host, + operator.LabelTestHealthPort: port, + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{{ + Type: corev1.PodReady, Status: corev1.ConditionTrue, + }}, + }, + } +} + func containsEvent(recordedEvents []string, expected string) bool { for _, event := range recordedEvents { if event == expected { @@ -206,3 +668,19 @@ func containsEventWithReason(recordedEvents []string, reason string) bool { } return false } + +func containsEventText(recordedEvents []string, fragments ...string) bool { + for _, event := range recordedEvents { + matches := true + for _, fragment := range fragments { + if !strings.Contains(event, fragment) { + matches = false + break + } + } + if matches { + return true + } + } + return false +} diff --git a/pkg/runner/health_mutator.go b/pkg/runner/health_mutator.go new file mode 100644 index 000000000..46d69a3ea --- /dev/null +++ b/pkg/runner/health_mutator.go @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ +package runner + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + v1 "github.com/oracle/coherence-operator/api/v1" + "github.com/oracle/coherence-operator/pkg/runner/run_details" +) + +const healthSecuritySupportClass = "com.oracle.coherence.k8s.HealthSecuritySupport" + +var ( + healthAuthDirectory = "/coherence/health-auth" + healthTLSDirectory = "/coherence/health-tls" +) + +func configureHealthMutators(details *run_details.RunDetails) error { + raw := details.Getenv(v1.HealthMutatorEnv) + if raw == "" { + return nil + } + connection := &v1.HealthMutatorConnection{} + if err := json.Unmarshal([]byte(raw), connection); err != nil { + return fmt.Errorf("invalid managed health configuration") + } + if err := connection.Validate(); err != nil { + return err + } + if !connection.Secure { + return nil + } + if details.ResolvedOperatorHealth { + return fmt.Errorf("managed secure health mutators require the Coherence health server") + } + contextName := "CoherenceREST" + mode := "packaged" + if connection.JAAS != nil { + if connection.JAAS.Mode != "" { + mode = connection.JAAS.Mode + } + } + // Probe capability without invoking a mutator. Unknown classpaths fail closed. + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cmd, err := createHealthSecurityCheckCommand(ctx, details) + if err != nil { + return err + } + if err := cmd.Run(); err != nil { + return fmt.Errorf("coherence health server security capability could not be established") + } + userArgs := details.Getenv(v1.EnvVarJvmArgs) + " " + details.Getenv("JAVA_TOOL_OPTIONS") + " " + details.Getenv("JDK_JAVA_OPTIONS") + " " + details.Getenv("_JAVA_OPTIONS") + for _, property := range []string{"coherence.health.", "coherence.health.security."} { + if strings.Contains(userArgs, "-D"+property) { + return fmt.Errorf("direct health properties conflict with managed security") + } + } + if mode == "packaged" { + if strings.Contains(userArgs, "java.security.auth.login.config") { + return fmt.Errorf("existing JAAS configuration requires jaas.mode=existing") + } + for _, name := range []string{"username", "password"} { + content, err := os.ReadFile(filepath.Join(healthAuthDirectory, name, "value")) + if err != nil || len(content) == 0 { + return fmt.Errorf("health credential file unavailable") + } + } + config := contextName + " { com.oracle.coherence.k8s.SecretLoginModule required usernameFile=\"" + filepath.Join(healthAuthDirectory, "username", "value") + "\" passwordFile=\"" + filepath.Join(healthAuthDirectory, "password", "value") + "\"; };\n" + name := filepath.Join(details.UtilsDir, "health-jaas.conf") + if err := os.WriteFile(name, []byte(config), 0600); err != nil { + return fmt.Errorf("cannot write health JAAS configuration") + } + details.AddSystemPropertyArg("java.security.auth.login.config", name) + } else if !strings.Contains(userArgs, "java.security.auth.login.config") { + return fmt.Errorf("existing mode requires an explicit JAAS login configuration") + } + ssl := connection.ServerTLS + base := "" + if ssl.Secrets != nil && *ssl.Secrets != "" { + base = healthTLSDirectory + } + file := func(value *string) (string, error) { + path := *value + if base != "" { + if filepath.Base(path) != path || path == "." || path == ".." { + return "", fmt.Errorf("invalid health store Secret key") + } + path = filepath.Join(base, path) + } + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() { + return "", fmt.Errorf("health TLS file unavailable") + } + return path, nil + } + mappings := []struct { + value *string + name string + url bool + }{ + {ssl.KeyStore, "keystore", true}, + {ssl.KeyStorePasswordFile, "keystore.password", false}, + {ssl.KeyPasswordFile, "key.password", false}, + } + for _, mapping := range mappings { + value, err := file(mapping.value) + if err != nil { + return err + } + if mapping.url { + value = "file:" + value + } + details.AddSystemPropertyArg("coherence.health.security."+mapping.name, value) + } + for _, mapping := range []struct { + value *string + name string + }{ + {ssl.KeyStoreType, "keystore.type"}, + {ssl.KeyStoreAlgorithm, "keystore.algorithm"}, + {ssl.KeyStoreProvider, "keystore.provider"}, + } { + if mapping.value != nil { + details.AddSystemPropertyArg("coherence.health.security."+mapping.name, *mapping.value) + } + } + details.AddSystemPropertyArg("coherence.health.http.mutators.enabled", "true") + details.AddSystemPropertyArg("coherence.health.http.auth", "basic") + details.AddSystemPropertyArg("coherence.health.http.provider", "HealthSSLProvider") + return nil +} + +func createHealthSecurityCheckCommand(ctx context.Context, details *run_details.RunDetails) (*exec.Cmd, error) { + if details.IsBuildPacks() { + return nil, fmt.Errorf("managed secure health mutators do not support the Cloud Native Buildpack launcher") + } + + var args []string + if details.IsSpringBoot() { + jar, _ := details.LookupEnv(v1.EnvVarSpringBootFatJar) + if jar == "" { + return nil, fmt.Errorf("managed secure health mutators require a configured Spring Boot fat JAR") + } + classpath := strings.ReplaceAll(details.GetClasspath(), ":", ",") + args = append(args, + "-Dloader.path="+classpath, + "-Dcoherence.operator.springboot.listener=false", + "-Dloader.main="+healthSecuritySupportClass, + v1.JvmOptClassPath, jar) + if details.AppType == v1.AppTypeSpring2 { + args = append(args, v1.SpringBootMain2) + } else { + args = append(args, v1.SpringBootMain3) + } + } else { + args = append(args, v1.JvmOptClassPath, details.GetClasspath(), healthSecuritySupportClass) + } + cmd := exec.CommandContext(ctx, details.GetJavaExecutable(), args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd, nil +} diff --git a/pkg/runner/health_mutator_test.go b/pkg/runner/health_mutator_test.go new file mode 100644 index 000000000..44ac1432c --- /dev/null +++ b/pkg/runner/health_mutator_test.go @@ -0,0 +1,277 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ +package runner + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + coh "github.com/oracle/coherence-operator/api/v1" + "github.com/oracle/coherence-operator/pkg/runner/run_details" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" +) + +func TestManagedHealthCoherenceTLSMappingWithExistingJAAS(t *testing.T) { + directory := t.TempDir() + material := writeManagedHealthMaterial(t, directory) + connection := managedHealthConnection(material, "existing") + connection.ServerTLS.KeyStoreProvider = ptr.To("SunJSSE") + connection.ServerTLS.TrustStore = ptr.To("legacy-truststore-not-mounted.jks") + connection.ServerTLS.TrustStorePasswordFile = ptr.To("legacy-truststore-password-not-mounted") + connection.ServerTLS.TrustStoreType = ptr.To("JKS") + connection.ServerTLS.TrustStoreAlgorithm = ptr.To("SunX509") + connection.ServerTLS.TrustStoreProvider = ptr.To("LegacyProvider") + details := newManagedHealthDetails(t, directory, connection, map[string]string{ + coh.EnvVarJvmArgs: "-Djava.security.auth.login.config=/application/jaas.conf", + }) + + if err := configureHealthMutators(details); err != nil { + t.Fatal(err) + } + arguments := details.GetArguments() + args := strings.Join(arguments, " ") + for _, required := range []string{ + "-Dcoherence.health.http.mutators.enabled=true", + "-Dcoherence.health.http.auth=basic", + "-Dcoherence.health.http.provider=HealthSSLProvider", + "-Dcoherence.health.security.keystore=file:" + material, + "-Dcoherence.health.security.keystore.password=" + material, + "-Dcoherence.health.security.key.password=" + material, + "-Dcoherence.health.security.keystore.type=JKS", + "-Dcoherence.health.security.keystore.algorithm=SunX509", + "-Dcoherence.health.security.keystore.provider=SunJSSE", + } { + if count := argumentCount(arguments, required); count != 1 { + t.Errorf("expected exactly one %s argument, found %d in %v", required, count, arguments) + } + } + if fileQualified := "-Dcoherence.health.security.keystore.provider=file:SunJSSE"; argumentCount(arguments, fileQualified) != 0 { + t.Fatalf("keystore provider was emitted as a file URL: %v", arguments) + } + if strings.Contains(args, "coherence.operator.health") { + t.Fatal("managed Coherence security configured the legacy Operator server") + } + if strings.Contains(args, "coherence.health.security.truststore") { + t.Fatal("one-way health TLS configured a server truststore from legacy fields") + } +} + +func TestManagedHealthOmitsKeyStoreProviderWhenUnset(t *testing.T) { + directory := t.TempDir() + connection := managedHealthConnection(writeManagedHealthMaterial(t, directory), "existing") + details := newManagedHealthDetails(t, directory, connection, map[string]string{ + coh.EnvVarJvmArgs: "-Djava.security.auth.login.config=/application/jaas.conf", + }) + + if err := configureHealthMutators(details); err != nil { + t.Fatal(err) + } + for _, arg := range details.GetArguments() { + if strings.HasPrefix(arg, "-Dcoherence.health.security.keystore.provider=") { + t.Fatalf("unset keystore provider emitted argument %q", arg) + } + } +} + +func TestManagedHealthPackagedJAAS(t *testing.T) { + directory := t.TempDir() + authDirectory := filepath.Join(directory, "auth") + tlsDirectory := filepath.Join(directory, "tls") + previousAuth, previousTLS := healthAuthDirectory, healthTLSDirectory + healthAuthDirectory, healthTLSDirectory = authDirectory, tlsDirectory + t.Cleanup(func() { + healthAuthDirectory, healthTLSDirectory = previousAuth, previousTLS + }) + for name, value := range map[string]string{"username": "health-operator", "password": "secret"} { + path := filepath.Join(authDirectory, name) + if err := os.MkdirAll(path, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "value"), []byte(value), 0600); err != nil { + t.Fatal(err) + } + } + if err := os.MkdirAll(tlsDirectory, 0700); err != nil { + t.Fatal(err) + } + for _, name := range []string{"server.jks", "store-password"} { + if err := os.WriteFile(filepath.Join(tlsDirectory, name), []byte("fixture"), 0600); err != nil { + t.Fatal(err) + } + } + connection := managedHealthConnection("server.jks", "packaged") + connection.ServerTLS.Secrets = ptr.To("health-stores-v1") + connection.ServerTLS.KeyStorePasswordFile = ptr.To("store-password") + connection.ServerTLS.KeyPasswordFile = ptr.To("store-password") + details := newManagedHealthDetails(t, directory, connection, nil) + + if err := configureHealthMutators(details); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(details.UtilsDir, "health-jaas.conf")) + if err != nil { + t.Fatal(err) + } + config := string(data) + if !strings.Contains(config, "CoherenceREST {") || !strings.Contains(config, "SecretLoginModule") || !strings.Contains(config, filepath.Join(authDirectory, "username", "value")) { + t.Fatalf("invalid packaged JAAS configuration: %s", config) + } + if !strings.Contains(strings.Join(details.GetArguments(), " "), "-Djava.security.auth.login.config="+filepath.Join(details.UtilsDir, "health-jaas.conf")) { + t.Fatal("packaged JAAS file was not added to the member JVM") + } +} + +func TestManagedHealthRejectsOperatorServer(t *testing.T) { + directory := t.TempDir() + connection := managedHealthConnection(writeManagedHealthMaterial(t, directory), "existing") + details := newManagedHealthDetails(t, directory, connection, map[string]string{ + coh.EnvVarJvmArgs: "-Djava.security.auth.login.config=/application/jaas.conf", + }) + details.ResolvedOperatorHealth = true + if err := configureHealthMutators(details); err == nil || !strings.Contains(err.Error(), "require the Coherence health server") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestManagedHealthRejectsIncompleteOrConflictingConfiguration(t *testing.T) { + for _, raw := range []string{`{"secure":true}`, `{"secure":false,"jaas":{"mode":"packaged"}}`, `invalid`} { + if err := configureHealthMutators(newSSLRunDetails(map[string]string{coh.HealthMutatorEnv: raw})); err == nil { + t.Fatalf("accepted incomplete configuration %s", raw) + } + } + + directory := t.TempDir() + connection := managedHealthConnection(writeManagedHealthMaterial(t, directory), "existing") + for _, property := range []string{ + "-Dcoherence.health.http.auth=none", + "-Dcoherence.health.security.keystore=/application/server.jks", + } { + details := newManagedHealthDetails(t, directory, connection, map[string]string{ + coh.EnvVarJvmArgs: property + " -Djava.security.auth.login.config=/application/jaas.conf", + }) + if err := configureHealthMutators(details); err == nil || !strings.Contains(err.Error(), "conflict") { + t.Fatalf("accepted conflicting property %s: %v", property, err) + } + } +} + +func TestHealthSecurityCheckUsesApplicationLauncher(t *testing.T) { + directory := t.TempDir() + details := newManagedHealthDetails(t, directory, nil, nil) + details.AddClasspath("/application/classes") + command, err := createHealthSecurityCheckCommand(context.Background(), details) + if err != nil { + t.Fatal(err) + } + joined := strings.Join(command.Args, " ") + if !strings.Contains(joined, "-cp") || !strings.Contains(joined, "/application/classes") || !strings.HasSuffix(joined, healthSecuritySupportClass) { + t.Fatalf("invalid Java capability command: %s", joined) + } + + spring := newManagedHealthDetails(t, directory, nil, map[string]string{ + coh.EnvVarAppType: coh.AppTypeSpring3, + coh.EnvVarSpringBootFatJar: "/application/app.jar", + }) + spring.AppType = coh.AppTypeSpring3 + command, err = createHealthSecurityCheckCommand(context.Background(), spring) + if err != nil { + t.Fatal(err) + } + joined = strings.Join(command.Args, " ") + for _, value := range []string{"-Dloader.main=" + healthSecuritySupportClass, "-cp /application/app.jar", coh.SpringBootMain3} { + if !strings.Contains(joined, value) { + t.Fatalf("Spring Boot capability command %q is missing %q", joined, value) + } + } + + unpackagedSpring := newManagedHealthDetails(t, directory, nil, map[string]string{coh.EnvVarAppType: coh.AppTypeSpring3}) + unpackagedSpring.AppType = coh.AppTypeSpring3 + if _, err := createHealthSecurityCheckCommand(context.Background(), unpackagedSpring); err == nil || !strings.Contains(err.Error(), "configured Spring Boot fat JAR") { + t.Fatalf("Spring Boot without a configured fat JAR was not rejected: %v", err) + } + + buildpack := newManagedHealthDetails(t, directory, nil, map[string]string{ + coh.EnvVarCnbpEnabled: "true", + coh.EnvVarAppType: coh.AppTypeSpring3, + }) + buildpack.AppType = coh.AppTypeSpring3 + if _, err := createHealthSecurityCheckCommand(context.Background(), buildpack); err == nil || !strings.Contains(err.Error(), "do not support the Cloud Native Buildpack launcher") { + t.Fatalf("buildpack packaging was not rejected: %v", err) + } +} + +func managedHealthConnection(material, mode string) *coh.HealthMutatorConnection { + selector := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "auth-v1"}, Key: "value"} + return &coh.HealthMutatorConnection{ + Secure: true, + BasicAuth: &coh.HTTPBasicAuth{Username: selector, Password: selector}, + TLS: &coh.HTTPClientTLS{CASecret: selector, ServerName: "health.example"}, + JAAS: &coh.HealthJAAS{Mode: mode}, + ServerTLS: &coh.SSLSpec{ + Enabled: ptr.To(true), + KeyStore: ptr.To(material), + KeyStorePasswordFile: ptr.To(material), + KeyPasswordFile: ptr.To(material), + KeyStoreType: ptr.To("JKS"), + KeyStoreAlgorithm: ptr.To("SunX509"), + RequireClientCert: ptr.To(false), + }, + } +} + +func newManagedHealthDetails(t *testing.T, directory string, connection *coh.HealthMutatorConnection, env map[string]string) *run_details.RunDetails { + t.Helper() + bin := filepath.Join(directory, "bin") + if err := os.MkdirAll(bin, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(bin, "java"), []byte("#!/bin/sh\nexit 0\n"), 0700); err != nil { + t.Fatal(err) + } + utils := filepath.Join(directory, "utils") + if err := os.MkdirAll(utils, 0700); err != nil { + t.Fatal(err) + } + values := map[string]string{coh.EnvVarCohUtilDir: utils} + for name, value := range env { + values[name] = value + } + if connection != nil { + data, err := json.Marshal(connection) + if err != nil { + t.Fatal(err) + } + values[coh.HealthMutatorEnv] = string(data) + } + details := newSSLRunDetails(values) + details.JavaHome = directory + details.Classpath = filepath.Join(directory, "operator.jar") + return details +} + +func writeManagedHealthMaterial(t *testing.T, directory string) string { + t.Helper() + material := filepath.Join(directory, "store") + if err := os.WriteFile(material, []byte("fixture"), 0600); err != nil { + t.Fatal(err) + } + return material +} + +func argumentCount(args []string, expected string) int { + count := 0 + for _, arg := range args { + if arg == expected { + count++ + } + } + return count +} diff --git a/pkg/runner/run_details/run_details.go b/pkg/runner/run_details/run_details.go index c08618220..a3a2873fb 100644 --- a/pkg/runner/run_details/run_details.go +++ b/pkg/runner/run_details/run_details.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2025, Oracle and/or its affiliates. + * Copyright (c) 2020, 2026, Oracle and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl. */ @@ -53,28 +53,29 @@ func NewRunDetails(v *viper.Viper, log logr.Logger) *RunDetails { // RunDetails contains the information to run an application. type RunDetails struct { - Command string - CoherenceHome string - JavaHome string - UtilsDir string - Dir string - GetSite bool - UseOperatorHealth bool - AppType string - Classpath string - MainClass string - InnerMainClass string - MainArgs []string - BuildPacks *bool - ExtraEnv []string - ClassPathFile string - JvmArgsFile string - args []string - vmOptions []string - memoryArgs []string - diagnosticArgs []string - env *viper.Viper - log logr.Logger + Command string + CoherenceHome string + JavaHome string + UtilsDir string + Dir string + GetSite bool + UseOperatorHealth bool + ResolvedOperatorHealth bool + AppType string + Classpath string + MainClass string + InnerMainClass string + MainArgs []string + BuildPacks *bool + ExtraEnv []string + ClassPathFile string + JvmArgsFile string + args []string + vmOptions []string + memoryArgs []string + diagnosticArgs []string + env *viper.Viper + log logr.Logger } func (in *RunDetails) GetAllArgs() []string { diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index b737d29be..1ea045540 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2025, Oracle and/or its affiliates. + * Copyright (c) 2020, 2026, Oracle and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl. */ @@ -380,6 +380,7 @@ func configureCommand(details *run_details.RunDetails) error { details.AddSystemPropertyArg(v1.SysPropCoherenceOverride, "k8s-coherence-override.xml") details.AddSystemPropertyFromEnvVar(v1.EnvVarCohOverride, v1.SysPropOperatorOverride) + details.ResolvedOperatorHealth = true post2206 := checkCoherenceVersion("14.1.1.2206.0", details) if post2206 { // at least CE 22.06 @@ -605,6 +606,10 @@ func configureCommand(details *run_details.RunDetails) error { details.AddArgs(extraJvmArgs...) } + if err := configureHealthMutators(details); err != nil { + return err + } + return nil } @@ -1049,6 +1054,7 @@ func cohPost2206(details *run_details.RunDetails) { if strings.EqualFold("true", useOperator) { details.AddSystemPropertyArg(v1.SysPropOperatorHealthEnabled, "true") } else { + details.ResolvedOperatorHealth = false details.AddSystemPropertyArg(v1.SysPropOperatorHealthEnabled, "false") details.SetSystemPropertyFromEnvVarOrDefault(v1.EnvVarCohHealthPort, v1.SysPropCoherenceHealthHttpPort, fmt.Sprintf("%d", v1.DefaultHealthPort)) } diff --git a/test/e2e/local/job-with-ready-action.yaml b/test/e2e/local/job-with-ready-action.yaml index 992581509..804a8a2e2 100644 --- a/test/e2e/local/job-with-ready-action.yaml +++ b/test/e2e/local/job-with-ready-action.yaml @@ -1,3 +1,7 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at +# http://oss.oracle.com/licenses/upl. + apiVersion: coherence.oracle.com/v1 kind: CoherenceJob metadata: @@ -9,12 +13,10 @@ spec: imagePullPolicy: ${TEST_IMAGE_PULL_POLICY} restartPolicy: Never readyAction: - httpGet: - port: rest - path: test - ports: - - name: rest - port: 8080 + exec: + command: + - java + - -version readinessProbe: initialDelaySeconds: 10 periodSeconds: 10 diff --git a/test/e2e/local/job_test.go b/test/e2e/local/job_test.go index ad0139407..03014333c 100644 --- a/test/e2e/local/job_test.go +++ b/test/e2e/local/job_test.go @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2024, Oracle and/or its affiliates. + * Copyright (c) 2020, 2026, Oracle and/or its affiliates. * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl. */ @@ -86,7 +86,7 @@ func TestJobWithSingleFailedReplica(t *testing.T) { g.Expect(err).NotTo(HaveOccurred()) } -func TestJobWithReadyAction(t *testing.T) { +func TestJobWithReadyExecAction(t *testing.T) { // Make sure we defer clean-up when we're done!! testContext.CleanupAfterTest(t) g := NewWithT(t) diff --git a/test/e2e/remote/job-with-ready-http-action.yaml b/test/e2e/remote/job-with-ready-http-action.yaml new file mode 100644 index 000000000..7e3479839 --- /dev/null +++ b/test/e2e/remote/job-with-ready-http-action.yaml @@ -0,0 +1,24 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at +# http://oss.oracle.com/licenses/upl. + +apiVersion: coherence.oracle.com/v1 +kind: CoherenceJob +metadata: + name: job-with-ready-http +spec: + cluster: ready-http-test + replicas: 3 + image: ${TEST_APPLICATION_IMAGE} + imagePullPolicy: ${TEST_IMAGE_PULL_POLICY} + restartPolicy: Never + readyAction: + httpGet: + port: rest + path: test + ports: + - name: rest + port: 8080 + readinessProbe: + initialDelaySeconds: 10 + periodSeconds: 10 diff --git a/test/e2e/remote/job_ready_action_test.go b/test/e2e/remote/job_ready_action_test.go new file mode 100644 index 000000000..de4e81ec6 --- /dev/null +++ b/test/e2e/remote/job_ready_action_test.go @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ + +package remote + +import ( + "fmt" + "testing" + "time" + + . "github.com/onsi/gomega" + coh "github.com/oracle/coherence-operator/api/v1" + "github.com/oracle/coherence-operator/test/e2e/helper" +) + +func TestJobWithReadyHTTPAction(t *testing.T) { + testContext.CleanupAfterTest(t) + g := NewWithT(t) + + name := "job-with-ready-http" + jobs, _ := helper.AssertCoherenceJobs(testContext, t, "job-with-ready-http-action.yaml") + + job, ok := jobs[name] + g.Expect(ok).To(BeTrue(), fmt.Sprintf("did not find expected '%s' deployment", name)) + + condition := remoteJobProbesExecuted{count: int(job.GetReplicas())} + _, err := helper.WaitForCoherenceJobCondition(testContext, job.Namespace, job.Name, condition, time.Second*10, time.Minute*5) + g.Expect(err).NotTo(HaveOccurred()) +} + +type remoteJobProbesExecuted struct { + count int +} + +func (in remoteJobProbesExecuted) Test(d coh.CoherenceResource) bool { + status := d.GetStatus() + if len(status.JobProbes) == 0 { + return false + } + + success := 0 + for _, s := range status.JobProbes { + if s.Success != nil && *s.Success { + success++ + } + } + return success == in.count +} + +func (in remoteJobProbesExecuted) String() string { + return fmt.Sprintf("Job HTTP ready actions executed on %d pods", in.count) +} diff --git a/test/health-mutator/README.md b/test/health-mutator/README.md new file mode 100644 index 000000000..5330f7aa8 --- /dev/null +++ b/test/health-mutator/README.md @@ -0,0 +1,443 @@ + + +# Protected Health mutator acceptance fixtures + +This directory contains repository-only qualification fixtures for protected +Coherence Health mutators. The procedures use test applications, generated +credentials, disposable namespaces, failure injection, and destructive +cleanup. They are not production deployment instructions. + +Customer documentation and reusable manifests are published in the +[Protected Coherence Health Endpoints example](../../examples/096_protected_health_endpoints/README.adoc). + +## Qualification scope and provenance + +Qualify the exact Operator image digest, Coherence artifact, Java version, +application image digest, and Kubernetes version used for a deployment. + +Complete runtime qualification used Coherence CE 15.1.1-0-5 with normal +Java/classpath packaging and proved `3 -> blocked zero -> 0 -> 3` plus +persistent recovery. Coherence CE 22.06.18 passed startup and +HTTPS/authentication smoke checks only. Base CE 22.06 fails closed at the +secure-mutator capability boundary. Multi-worker Pod placement remains +unqualified. + +## Managed automated fixture + +The automated fixture validates the recommended +`spec.healthMutatorConnection.secure: true` path. Use a separate test namespace +and a test application image built with the affected hardened Coherence +artifact. The image must: + +- contain `com.oracle.coherence.k8s.testing.RestServer` and + `test-cache-config.xml`; +- be available to every node; and +- be selected by immutable digest. + +The scripts generate 30-day test credentials and TLS material, create immutable +Secrets, deploy two storage-enabled members with active persistence, and prove: + +- the unconfigured hardened mutator returns HTTP 403; +- read-only Health remains available over verified HTTPS; +- missing and invalid Basic credentials return HTTP 401; +- valid credentials can suspend and resume services; +- invalid Operator CA trust blocks scale-to-zero; +- repairing trust allows scale-to-zero; and +- the persistent canary is recovered after restoring two members. + +From the repository root: + +```bash +test/health-mutator/apply.sh \ + kind-coh40013873-qual \ + coh40013873-qual \ + registry.example/operator-test@sha256:REPLACE_WITH_VERIFIED_DIGEST \ + /tmp/coh40013873-health-v1 + +test/health-mutator/verify.sh \ + kind-coh40013873-qual \ + coh40013873-qual \ + /tmp/coh40013873-health-v1 +``` + +The arguments are the explicit Kubernetes context, new disposable namespace, +pinned application-image digest, and new material directory. Preserve the +rendered manifests, Pod placement, events, exact image identities, Kubernetes +version, PVC identities, and canary results before deleting the namespace. + +## Application-owned manual workflow + +Use this workflow when the application, rather than +`spec.healthMutatorConnection`, owns the Coherence Health server's TLS, JAAS, +and mutator properties. The Operator owns only the request it uses during safe +shutdown or scale-to-zero. + +The supplied qualification fixture is `application-owned.yaml`. It shows the exact +application-owned configuration qualified with Coherence 15.1.1-0-5. The +application main class, cache configuration, and persistence-canary endpoint +are qualification-image details; replace them with the corresponding parts of +the application being tested. + +### Ownership boundary + +The `Coherence` resource deliberately omits `healthMutatorConnection`. + +The application owns: + +- enabling the Coherence Health mutators; +- selecting the Health server authentication mode; +- selecting and configuring the TLS socket provider; +- mounting the server keystore and password files; +- providing the `CoherenceREST` JAAS context; and +- declaring HTTPS kubelet probes. + +The Operator owns only `spec.suspendProbe`. For the Basic-over-TLS workflow it +reads the client username, password, and CA from namespace-local Secrets and +sends a verified HTTPS `PUT /suspend`. + +### Preconditions + +- Use a disposable namespace and a StatefulSet-backed `Coherence` resource. +- The application must use Coherence 15.1.1-0-5 or another version that exposes + the hardened PUT mutator contract. +- The selected application launcher must place the Coherence and Operator + support classes on the member JVM classpath. +- Do not select the legacy Operator health server with + `COHERENCE_OPERATOR_HEALTH_CHECK=true`. +- Use a digest-pinned application image. +- The cluster needs a default `ReadWriteOnce` StorageClass for this demo. + +### 1. Generate test credentials and TLS material + +From the root of this Operator repository: + +```bash +export NAMESPACE=coh-app-owned-health +export CONTEXT=YOUR_CONTEXT +export MATERIAL=/tmp/coh-app-owned-health-material + +test/health-mutator/prepare.sh "$MATERIAL" health.example +``` + +The generated certificate contains `health.example` as a DNS subject +alternative name. The Operator will connect to a Pod IP but verify this name. + +For production, replace this generated material with certificates and random +credentials issued through the normal security process. + +### 2. Create immutable, versioned Secrets + +```bash +kubectl --context "$CONTEXT" create namespace "$NAMESPACE" + +kubectl --context "$CONTEXT" -n "$NAMESPACE" create secret generic health-auth-v1 \ + --from-file=username="$MATERIAL/username" \ + --from-file=password="$MATERIAL/password" + +kubectl --context "$CONTEXT" -n "$NAMESPACE" create secret generic health-stores-v1 \ + --from-file=server.jks="$MATERIAL/server.jks" \ + --from-file=store-password="$MATERIAL/store-password" + +kubectl --context "$CONTEXT" -n "$NAMESPACE" create secret generic health-ca-v1 \ + --from-file=ca.pem="$MATERIAL/ca.pem" + +for secret in health-auth-v1 health-stores-v1 health-ca-v1; do + kubectl --context "$CONTEXT" -n "$NAMESPACE" patch secret "$secret" \ + --type=merge -p '{"immutable":true}' +done +``` + +The credential files contain exact bytes without trailing newlines. The JAAS +login module performs an exact comparison, so an accidental newline changes +the credential. + +### 3. Understand the application-owned server properties + +The full example supplies these member JVM properties: + +```text +coherence.health.http.mutators.enabled=true +coherence.health.http.auth=basic +coherence.health.http.provider=HealthSSLProvider +coherence.health.security.keystore=file:/application/health-tls/server.jks +coherence.health.security.keystore.password=/application/health-tls/store-password +coherence.health.security.key.password=/application/health-tls/store-password +coherence.health.security.keystore.type=JKS +java.security.auth.login.config=/application/health-jaas/jaas.conf +``` + +Their responsibilities are: + +- `mutators.enabled=true` permits PUT `/suspend` and `/resume`. +- `auth=basic` requires an HTTP Basic identity for mutator routes. +- `provider=HealthSSLProvider` selects the Operator-supplied socket-provider + definition used by the Coherence Health listener. +- The `coherence.health.security.*` properties identify the application-mounted + server keystore and password files consumed by that provider. One-way TLS + does not use a member-side truststore. +- `java.security.auth.login.config` selects the application-owned JAAS file. + +These direct properties are valid only because `healthMutatorConnection` is +absent. They conflict with managed secure mode and are rejected when both +approaches are configured. + +The `CoherenceREST` JAAS entry uses the Operator support library's +`SecretLoginModule`, but its file paths are application-owned mounts: + +```text +CoherenceREST { + com.oracle.coherence.k8s.SecretLoginModule required + usernameFile="/application/health-auth/username" + passwordFile="/application/health-auth/password"; +}; +``` + +An application can substitute another compatible JAAS LoginModule, but it must +retain the exact `CoherenceREST` context and accept Coherence's +`UsernameAndPassword` identity assertion. + +### 4. Render and apply the complete resource + +Build the application image with the Coherence version being qualified. For +Kind, load it into every Kubernetes node. For a shared cluster, push it to a +registry reachable by every node. In either case, obtain and use its immutable +manifest digest, not a mutable tag. The image used by this exact example must +contain `com.oracle.coherence.k8s.testing.HealthMutatorDemo` and +`test-cache-config.xml`; when qualifying another application, update those two +fields in the resource. + +```bash +export IMAGE_DIGEST='registry.example/demo@sha256:REPLACE_WITH_DIGEST' + +envsubst '${NAMESPACE} ${IMAGE_DIGEST}' \ + < test/health-mutator/application-owned.yaml \ + > "$MATERIAL/application-owned-health.yaml" + +kubectl --context "$CONTEXT" apply \ + -f "$MATERIAL/application-owned-health.yaml" + +until kubectl --context "$CONTEXT" -n "$NAMESPACE" get \ + statefulset/application-owned-health >/dev/null 2>&1; do sleep 2; done + +kubectl --context "$CONTEXT" -n "$NAMESPACE" rollout status \ + statefulset/application-owned-health --timeout=10m +``` + +The example starts three storage-enabled members with active persistence. + +### 5. Verify the server before testing Operator suspension + +Confirm that the running JVM contains the application-owned settings and that +the kubelet probes use HTTPS: + +```bash +kubectl --context "$CONTEXT" -n "$NAMESPACE" get statefulset application-owned-health \ + -o jsonpath='{range .spec.template.spec.containers[*]}{.name}{" ready="}{.readinessProbe.httpGet.scheme}{":"}{.readinessProbe.httpGet.port}{" live="}{.livenessProbe.httpGet.scheme}{":"}{.livenessProbe.httpGet.port}{"\n"}{end}' +``` + +Expected Coherence-container output: + +```text +coherence ready=HTTPS:8089 live=HTTPS:8089 +``` + +Port-forward one member: + +```bash +kubectl --context "$CONTEXT" -n "$NAMESPACE" port-forward \ + pod/application-owned-health-0 18089:8089 18080:8080 +``` + +In another terminal, verify the protocol. Read-only health stays +unauthenticated; mutations require valid Basic credentials: + +```bash +curl --fail --cacert "$MATERIAL/ca.pem" \ + --resolve health.example:18089:127.0.0.1 \ + https://health.example:18089/ready + +curl --silent --output /dev/null --write-out '%{http_code}\n' \ + --request PUT --cacert "$MATERIAL/ca.pem" \ + --resolve health.example:18089:127.0.0.1 \ + https://health.example:18089/suspend +# Expected: 401 + +curl --fail --request PUT \ + --user "$(<"$MATERIAL/username"):$(<"$MATERIAL/password")" \ + --cacert "$MATERIAL/ca.pem" \ + --resolve health.example:18089:127.0.0.1 \ + https://health.example:18089/suspend + +curl --fail --request PUT \ + --user "$(<"$MATERIAL/username"):$(<"$MATERIAL/password")" \ + --cacert "$MATERIAL/ca.pem" \ + --resolve health.example:18089:127.0.0.1 \ + https://health.example:18089/resume +``` + +Always resume after a direct suspension check before continuing. + +### 6. Establish a persistent canary + +The qualification application exposes `/canaryStart` and `/canaryCheck` on +port 8080. With the same port-forward running: + +```bash +curl --fail http://127.0.0.1:18080/canaryStart +# Expected with the default partition count: stored 257 entries + +kubectl --context "$CONTEXT" -n "$NAMESPACE" get pvc -o name | sort \ + > "$MATERIAL/pvcs-before-zero.txt" +``` + +For another application, replace these two endpoints with an application-level +write and read that prove data is present in every required persistent cache. +Record enough information to distinguish recovered data from newly created +data after restart. + +### 7. Prove that bad Operator trust fails closed + +Create a bad CA Secret and point only the custom Operator suspension action to +it. This does not change the server certificate or kubelet probes. + +```bash +printf 'not a certificate\n' > "$MATERIAL/bad-ca.pem" + +kubectl --context "$CONTEXT" -n "$NAMESPACE" create secret generic health-bad-ca-v1 \ + --from-file=ca.pem="$MATERIAL/bad-ca.pem" + +kubectl --context "$CONTEXT" -n "$NAMESPACE" patch \ + coherence.coherence.oracle.com application-owned-health --type=merge \ + -p '{"spec":{"suspendProbe":{"timeoutSeconds":60,"http":{"method":"PUT","scheme":"HTTPS","path":"/suspend","port":8089,"basicAuth":{"username":{"name":"health-auth-v1","key":"username"},"password":{"name":"health-auth-v1","key":"password"}},"tls":{"caSecret":{"name":"health-bad-ca-v1","key":"ca.pem"},"serverName":"health.example"}}}}}' + +kubectl --context "$CONTEXT" -n "$NAMESPACE" patch \ + coherence.coherence.oracle.com application-owned-health --type=merge \ + -p '{"spec":{"replicas":0}}' +``` + +The Operator must report a suspension failure and keep the StatefulSet and all +three Pods: + +```bash +kubectl --context "$CONTEXT" -n "$NAMESPACE" get events \ + --field-selector involvedObject.kind=Coherence,involvedObject.name=application-owned-health,type=Warning + +kubectl --context "$CONTEXT" -n "$NAMESPACE" get statefulset,pods +``` + +Do not proceed unless the transition was blocked. + +### 8. Repair trust and reach zero + +Restore the valid CA reference. The unchanged `replicas: 0` request should be +retried and complete: + +```bash +kubectl --context "$CONTEXT" -n "$NAMESPACE" patch \ + coherence.coherence.oracle.com application-owned-health --type=merge \ + -p '{"spec":{"suspendProbe":{"timeoutSeconds":60,"http":{"method":"PUT","scheme":"HTTPS","path":"/suspend","port":8089,"basicAuth":{"username":{"name":"health-auth-v1","key":"username"},"password":{"name":"health-auth-v1","key":"password"}},"tls":{"caSecret":{"name":"health-ca-v1","key":"ca.pem"},"serverName":"health.example"}}}}}' + +kubectl --context "$CONTEXT" -n "$NAMESPACE" wait --for=delete \ + statefulset/application-owned-health --timeout=10m + +for _ in $(seq 1 120); do + count=$(kubectl --context "$CONTEXT" -n "$NAMESPACE" get pods \ + -l coherenceDeployment=application-owned-health -o name | wc -l | tr -d ' ') + [[ "$count" = 0 ]] && break + sleep 2 +done + +test "$(kubectl --context "$CONTEXT" -n "$NAMESPACE" get pods \ + -l coherenceDeployment=application-owned-health -o name | wc -l | tr -d ' ')" = 0 + +kubectl --context "$CONTEXT" -n "$NAMESPACE" get pvc -o name | sort \ + > "$MATERIAL/pvcs-at-zero.txt" +diff -u "$MATERIAL/pvcs-before-zero.txt" "$MATERIAL/pvcs-at-zero.txt" +``` + +### 9. Restore three members and verify persistent recovery + +```bash +kubectl --context "$CONTEXT" -n "$NAMESPACE" patch \ + coherence.coherence.oracle.com application-owned-health --type=merge \ + -p '{"spec":{"replicas":3}}' + +until kubectl --context "$CONTEXT" -n "$NAMESPACE" get \ + statefulset/application-owned-health >/dev/null 2>&1; do sleep 2; done + +kubectl --context "$CONTEXT" -n "$NAMESPACE" rollout status \ + statefulset/application-owned-health --timeout=10m + +kubectl --context "$CONTEXT" -n "$NAMESPACE" port-forward \ + pod/application-owned-health-0 18089:8089 18080:8080 +``` + +In another terminal: + +```bash +curl --fail http://127.0.0.1:18080/canaryCheck +# Expected: recovered 257 entries +``` + +Record the final PVC names and require an empty diff: + +```bash +kubectl --context "$CONTEXT" -n "$NAMESPACE" get pvc -o name | sort \ + > "$MATERIAL/pvcs-after-restart.txt" +diff -u "$MATERIAL/pvcs-before-zero.txt" "$MATERIAL/pvcs-after-restart.txt" +``` + +### 10. Preserve evidence and clean up + +Retain the rendered resource, Pod placement, warning events, image digest, +Operator image/revision, Kubernetes version, PVC lists, and canary results. + +After copying the evidence to durable storage: + +```bash +kubectl --context "$CONTEXT" delete namespace "$NAMESPACE" +``` + +Namespace deletion can delete dynamically provisioned volumes according to the +StorageClass reclaim policy. + +### Validated result + +On 2026-09-28 this exact application-owned Basic-over-TLS manifest passed in +the disposable Kubernetes 1.34.0/linux-arm64 Kind environment with Operator +commit `fd9192e8bd0bdcd82b35df2eb4f9eecbc837e80f` and application image: + +```text +docker.io/library/coherence-health-mutator-demo@sha256:7e17b851a0ed02885bb23915f108e023d45b9794eb79cf4c0d5b3f0750edd5ec +``` + +The run proved: + +- generated application-owned kubelet probes used HTTPS on port 8089; +- an unauthenticated PUT returned 401; +- authenticated suspend and resume returned 200; +- invalid Operator CA trust blocked scale-to-zero with three ready members; +- restoring the CA completed the transition from three members to zero Pods; +- all three PVC identities were retained; and +- restoring three members recovered all 257 persistent canary entries. + +The Kind environment had one Kubernetes worker/control-plane node. This proves +three Coherence members, not scheduling across three physical worker nodes. + +### Other application-owned authentication modes + +Coherence 15.1.1-0-5 also exposes `cert` and `cert+basic` mutator authentication +modes. They require an application-owned socket provider that requests and +validates client certificates. + +The Operator's custom HTTP action supports server verification and +Secret-backed Basic credentials, but it has no client-certificate fields. +Therefore it cannot natively call `cert` or `cert+basic` endpoints. Use a +custom `suspendProbe.exec` command capable of presenting the client certificate +and key, or use managed Basic-over-one-way-TLS. Treat the custom exec path as +application-owned and qualify it separately; it is not covered by the demo's +Basic-over-TLS acceptance result. diff --git a/test/health-mutator/admission_test.go b/test/health-mutator/admission_test.go new file mode 100644 index 000000000..a7bb68449 --- /dev/null +++ b/test/health-mutator/admission_test.go @@ -0,0 +1,345 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ +package healthmutator + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "testing" + + coh "github.com/oracle/coherence-operator/api/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +// This gate requires an isolated API server, never the user's current kube context. +func TestHealthActionAdmission(t *testing.T) { + if os.Getenv("KUBEBUILDER_ASSETS") == "" { + t.Skip("KUBEBUILDER_ASSETS is required for the admission certification gate") + } + environment := &envtest.Environment{CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, ErrorIfCRDPathMissing: true} + config, err := environment.Start() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := environment.Stop(); err != nil { + t.Error(err) + } + }) + scheme := runtime.NewScheme() + if err := coh.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + c, err := client.New(config, client.Options{Scheme: scheme}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + if err := c.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "health-test"}}); err != nil { + t.Fatal(err) + } + valid := &coh.Coherence{ObjectMeta: metav1.ObjectMeta{Name: "valid", Namespace: "health-test"}} + valid.Spec.SuspendProbe = &coh.Probe{HTTP: &coh.HTTPAction{Method: "PUT", HTTPEndpoint: coh.HTTPEndpoint{Path: "/suspend", Port: intstr.FromInt(6676)}}} + if err := c.Create(ctx, valid); err != nil { + t.Fatal(err) + } + stored := &coh.Coherence{} + if err := c.Get(ctx, client.ObjectKeyFromObject(valid), stored); err != nil { + t.Fatal(err) + } + if stored.Spec.SuspendProbe.HTTP == nil || stored.Spec.SuspendProbe.HTTP.Method != "PUT" { + t.Fatal("HTTP action was pruned") + } + invalid := valid.DeepCopy() + invalid.ResourceVersion = "" + invalid.Name = "invalid" + invalid.Spec.SuspendProbe.Exec = &corev1.ExecAction{Command: []string{"true"}} + if err := c.Create(ctx, invalid); err == nil { + t.Fatal("admission accepted multiple actions") + } + selector := corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "health-auth-v1"}, Key: "value"} + authenticated := &coh.Coherence{ObjectMeta: metav1.ObjectMeta{Name: "authenticated-pod-probe", Namespace: "health-test"}} + authenticated.Spec.SuspendProbe = &coh.Probe{HTTP: &coh.HTTPAction{ + HTTPEndpoint: coh.HTTPEndpoint{Path: "/suspend", Port: intstr.FromInt(6676), Scheme: corev1.URISchemeHTTPS}, + Method: "PUT", + BasicAuth: &coh.HTTPBasicAuth{Username: selector, Password: selector}, + }} + if err := c.Create(ctx, authenticated); err != nil { + t.Fatalf("admission rejected an authenticated target-Pod probe: %v", err) + } + customHost := authenticated.DeepCopy() + customHost.ResourceVersion = "" + customHost.Name = "authenticated-custom-host-probe" + customHost.Spec.SuspendProbe.HTTP.Host = "health.example" + if err := c.Create(ctx, customHost); err == nil { + t.Fatal("admission accepted Basic authentication with a custom host") + } + legacyEmpty := &coh.Coherence{ObjectMeta: metav1.ObjectMeta{Name: "legacy-empty-probe", Namespace: "health-test"}} + legacyEmpty.Spec.SuspendProbe = &coh.Probe{} + if err := c.Create(ctx, legacyEmpty); err != nil { + t.Fatalf("admission rejected a legacy empty probe: %v", err) + } + legacyEmpty.Annotations = map[string]string{"compatibility-test": "updated"} + if err := c.Update(ctx, legacyEmpty); err != nil { + t.Fatalf("admission rejected an update containing a legacy empty probe: %v", err) + } + legacyMultiple := &coh.Coherence{ObjectMeta: metav1.ObjectMeta{Name: "legacy-multiple-probe-actions", Namespace: "health-test"}} + legacyMultiple.Spec.SuspendProbe = &coh.Probe{ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{Command: []string{"true"}}, + HTTPGet: &corev1.HTTPGetAction{Path: "/suspend", Port: intstr.FromInt(6676)}, + }} + if err := c.Create(ctx, legacyMultiple); err != nil { + t.Fatalf("admission rejected legacy multiple probe actions: %v", err) + } + invalid = valid.DeepCopy() + invalid.ResourceVersion = "" + invalid.Name = "incomplete-secure" + invalid.Spec.SuspendProbe = nil + invalid.Spec.HealthMutatorConnection = &coh.HealthMutatorConnection{Secure: true} + if err := c.Create(ctx, invalid); err == nil { + t.Fatal("admission accepted incomplete secure configuration") + } + managed := &coh.Coherence{ObjectMeta: metav1.ObjectMeta{Name: "managed", Namespace: "health-test"}} + managed.Spec.HealthMutatorConnection = &coh.HealthMutatorConnection{ + Secure: true, + BasicAuth: &coh.HTTPBasicAuth{Username: selector, Password: selector}, + TLS: &coh.HTTPClientTLS{CASecret: corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "health-ca-v1"}, Key: "ca.pem"}, ServerName: "health.example"}, + JAAS: &coh.HealthJAAS{}, + ServerTLS: &coh.SSLSpec{ + Enabled: ptr.To(true), + RequireClientCert: ptr.To(false), + Secrets: ptr.To("health-stores-v1"), + KeyStore: ptr.To("server.jks"), + KeyStorePasswordFile: ptr.To("store-password"), + KeyPasswordFile: ptr.To("store-password"), + }, + } + if err := c.Create(ctx, managed); err != nil { + t.Fatal(err) + } + storedManaged := &coh.Coherence{} + if err := c.Get(ctx, client.ObjectKeyFromObject(managed), storedManaged); err != nil { + t.Fatal(err) + } + connection := storedManaged.Spec.HealthMutatorConnection + if connection == nil || connection.BasicAuth == nil || connection.TLS == nil || connection.ServerTLS == nil || connection.JAAS == nil || connection.JAAS.Mode != "packaged" { + t.Fatal("complete healthMutatorConnection was pruned or not defaulted") + } + if connection.ServerTLS.TrustStore != nil || connection.ServerTLS.TrustStorePasswordFile != nil { + t.Fatal("admission added a server truststore to one-way TLS") + } + legacy := managed.DeepCopy() + legacy.ResourceVersion = "" + legacy.Name = "managed-with-legacy-truststore" + legacy.Spec.HealthMutatorConnection.ServerTLS.TrustStore = ptr.To("trust.jks") + legacy.Spec.HealthMutatorConnection.ServerTLS.TrustStorePasswordFile = ptr.To("store-password") + if err := c.Create(ctx, legacy); err != nil { + t.Fatalf("admission rejected previously valid truststore fields: %v", err) + } + conflict := managed.DeepCopy() + conflict.ResourceVersion = "" + conflict.Name = "managed-conflict" + conflict.Spec.SuspendProbe = valid.Spec.SuspendProbe.DeepCopy() + if err := c.Create(ctx, conflict); err == nil { + t.Fatal("admission accepted managed security with a custom suspend probe") + } + job := &coh.CoherenceJob{ObjectMeta: metav1.ObjectMeta{Name: "job", Namespace: "health-test"}} + job.Spec.ReadyAction = &coh.CoherenceJobProbe{Probe: *valid.Spec.SuspendProbe.DeepCopy()} + if err := c.Create(ctx, job); err != nil { + t.Fatal(err) + } + storedJob := &coh.CoherenceJob{} + if err := c.Get(ctx, client.ObjectKeyFromObject(job), storedJob); err != nil { + t.Fatal(err) + } + if storedJob.Spec.ReadyAction == nil || storedJob.Spec.ReadyAction.HTTP == nil { + t.Fatal("Job HTTP action was pruned") + } + rawJob := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": coh.GroupVersion.String(), + "kind": "CoherenceJob", + "metadata": map[string]interface{}{ + "name": "job-with-managed-field", + "namespace": "health-test", + }, + "spec": map[string]interface{}{ + "healthMutatorConnection": map[string]interface{}{"secure": false}, + }, + }} + if err := c.Create(ctx, rawJob); err != nil { + t.Fatal(err) + } + storedRawJob := &unstructured.Unstructured{} + storedRawJob.SetAPIVersion(coh.GroupVersion.String()) + storedRawJob.SetKind("CoherenceJob") + if err := c.Get(ctx, client.ObjectKeyFromObject(rawJob), storedRawJob); err != nil { + t.Fatal(err) + } + if _, found, err := unstructured.NestedFieldNoCopy(storedRawJob.Object, "spec", "healthMutatorConnection"); err != nil || found { + t.Fatalf("managed StatefulSet field survived CoherenceJob admission: found=%t err=%v", found, err) + } + + t.Run("list-map key defaults", func(t *testing.T) { + resource := newUnstructuredResource("Coherence", "list-map-defaults", map[string]interface{}{ + "network": map[string]interface{}{ + "dnsConfig": map[string]interface{}{ + "options": []interface{}{ + map[string]interface{}{"value": "2"}, + map[string]interface{}{"name": "ndots", "value": "1"}, + }, + }, + }, + "actions": []interface{}{ + map[string]interface{}{ + "name": "pull-secret-defaulting", + "job": map[string]interface{}{ + "spec": map[string]interface{}{ + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "restartPolicy": "Never", + "containers": []interface{}{ + map[string]interface{}{"name": "test", "image": "example.invalid/test:latest"}, + }, + "imagePullSecrets": []interface{}{ + map[string]interface{}{}, + map[string]interface{}{"name": "registry-credentials"}, + }, + }, + }, + }, + }, + }, + }, + }) + storedResource := createAndGetUnstructured(t, ctx, c, resource) + assertNestedListMapNames(t, storedResource.Object, []string{"", "ndots"}, "spec", "network", "dnsConfig", "options") + storedActions, found, err := unstructured.NestedSlice(storedResource.Object, "spec", "actions") + if err != nil || !found || len(storedActions) != 1 { + t.Fatalf("actions were not stored: count=%d found=%t err=%v", len(storedActions), found, err) + } + storedAction, ok := storedActions[0].(map[string]interface{}) + if !ok { + t.Fatalf("stored action has type %T", storedActions[0]) + } + assertNestedListMapNames(t, storedAction, []string{"", "registry-credentials"}, "job", "spec", "template", "spec", "imagePullSecrets") + + jobResource := newUnstructuredResource("CoherenceJob", "job-list-map-defaults", map[string]interface{}{ + "network": map[string]interface{}{ + "dnsConfig": map[string]interface{}{ + "options": []interface{}{ + map[string]interface{}{"value": "2"}, + map[string]interface{}{"name": "ndots", "value": "1"}, + }, + }, + }, + }) + storedJobResource := createAndGetUnstructured(t, ctx, c, jobResource) + assertNestedListMapNames(t, storedJobResource.Object, []string{"", "ndots"}, "spec", "network", "dnsConfig", "options") + }) + + t.Run("actions limit", func(t *testing.T) { + actions := make([]coh.Action, 128) + for i := range actions { + actions[i].Name = fmt.Sprintf("action-%03d", i) + } + atLimit := &coh.Coherence{ObjectMeta: metav1.ObjectMeta{Name: "actions-at-limit", Namespace: "health-test"}} + atLimit.Spec.Actions = actions + if err := c.Create(ctx, atLimit); err != nil { + t.Fatalf("admission rejected 128 actions: %v", err) + } + storedAtLimit := &coh.Coherence{} + if err := c.Get(ctx, client.ObjectKeyFromObject(atLimit), storedAtLimit); err != nil { + t.Fatal(err) + } + if len(storedAtLimit.Spec.Actions) != 128 { + t.Fatalf("stored %d actions, want 128", len(storedAtLimit.Spec.Actions)) + } + + overLimit := &coh.Coherence{ObjectMeta: metav1.ObjectMeta{Name: "actions-over-limit", Namespace: "health-test"}} + overLimit.Spec.Actions = append(append([]coh.Action{}, actions...), coh.Action{Name: "action-128"}) + if err := c.Create(ctx, overLimit); err == nil { + t.Fatal("admission accepted 129 actions") + } + }) + + if os.Getenv("HEALTH_MUTATOR_RUNNER_SUITE") == "true" { + kubeconfig := clientcmdapi.Config{Clusters: map[string]*clientcmdapi.Cluster{"test": {Server: config.Host, CertificateAuthorityData: config.CAData}}, AuthInfos: map[string]*clientcmdapi.AuthInfo{"test": {ClientCertificateData: config.CertData, ClientKeyData: config.KeyData}}, Contexts: map[string]*clientcmdapi.Context{"test": {Cluster: "test", AuthInfo: "test"}}, CurrentContext: "test"} + path := filepath.Join(t.TempDir(), "kubeconfig") + if err := clientcmd.WriteToFile(kubeconfig, path); err != nil { + t.Fatal(err) + } + command := exec.Command("go", "test", "./pkg/runner") + command.Dir = filepath.Join("..", "..") + command.Env = append(os.Environ(), "KUBECONFIG="+path) + output, err := command.CombinedOutput() + if err != nil { + t.Fatalf("runner suite failed: %s", output) + } + } + +} + +func newUnstructuredResource(kind, name string, spec map[string]interface{}) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": coh.GroupVersion.String(), + "kind": kind, + "metadata": map[string]interface{}{ + "name": name, + "namespace": "health-test", + }, + "spec": spec, + }} +} + +func createAndGetUnstructured(t *testing.T, ctx context.Context, c client.Client, resource *unstructured.Unstructured) *unstructured.Unstructured { + t.Helper() + if err := c.Create(ctx, resource); err != nil { + t.Fatal(err) + } + stored := &unstructured.Unstructured{} + stored.SetAPIVersion(resource.GetAPIVersion()) + stored.SetKind(resource.GetKind()) + if err := c.Get(ctx, client.ObjectKeyFromObject(resource), stored); err != nil { + t.Fatal(err) + } + return stored +} + +func assertNestedListMapNames(t *testing.T, object map[string]interface{}, expected []string, fields ...string) { + t.Helper() + items, found, err := unstructured.NestedSlice(object, fields...) + if err != nil || !found { + t.Fatalf("list-map field %v was not stored: found=%t err=%v", fields, found, err) + } + if len(items) != len(expected) { + t.Fatalf("list-map field %v has %d items, want %d", fields, len(items), len(expected)) + } + for i, item := range items { + entry, ok := item.(map[string]interface{}) + if !ok { + t.Fatalf("list-map field %v item %d has type %T", fields, i, item) + } + name, found, err := unstructured.NestedString(entry, "name") + if err != nil || !found || name != expected[i] { + t.Fatalf("list-map field %v item %d name=%q found=%t err=%v, want %q", fields, i, name, found, err, expected[i]) + } + } +} diff --git a/test/health-mutator/application-owned.yaml b/test/health-mutator/application-owned.yaml new file mode 100644 index 000000000..eb979c497 --- /dev/null +++ b/test/health-mutator/application-owned.yaml @@ -0,0 +1,122 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at +# http://oss.oracle.com/licenses/upl. +# Repository qualification fixture for application-owned Basic-over-TLS. +# Render IMAGE_DIGEST and NAMESPACE with envsubst before applying. Replace the +# application main class, cache configuration, and test REST port with the +# corresponding settings and recovery proof for another application. +apiVersion: v1 +kind: ConfigMap +metadata: + name: application-health-jaas-v1 + namespace: ${NAMESPACE} +data: + jaas.conf: | + CoherenceREST { + com.oracle.coherence.k8s.SecretLoginModule required + usernameFile="/application/health-auth/username" + passwordFile="/application/health-auth/password"; + }; +--- +apiVersion: coherence.oracle.com/v1 +kind: Coherence +metadata: + name: application-owned-health + namespace: ${NAMESPACE} +spec: + replicas: 3 + image: ${IMAGE_DIGEST} + imagePullPolicy: IfNotPresent + healthPort: 8089 + + securityContext: + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + containerSecurityContext: + runAsUser: 65532 + runAsGroup: 65532 + runAsNonRoot: true + + application: + # Qualification-image entry point; this is not configured by the Operator's + # health client. + main: com.oracle.coherence.k8s.testing.HealthMutatorDemo + + # The application owns every Coherence Health security property. There is + # deliberately no healthMutatorConnection block. + jvm: + args: + - -Dcoherence.health.http.mutators.enabled=true + - -Dcoherence.health.http.auth=basic + - -Dcoherence.health.http.provider=HealthSSLProvider + - -Dcoherence.health.security.keystore=file:/application/health-tls/server.jks + - -Dcoherence.health.security.keystore.password=/application/health-tls/store-password + - -Dcoherence.health.security.key.password=/application/health-tls/store-password + - -Dcoherence.health.security.keystore.type=JKS + - -Djava.security.auth.login.config=/application/health-jaas/jaas.conf + + volumes: + - name: application-health-auth + secret: + secretName: health-auth-v1 + - name: application-health-tls + secret: + secretName: health-stores-v1 + - name: application-health-jaas + configMap: + name: application-health-jaas-v1 + volumeMounts: + - name: application-health-auth + mountPath: /application/health-auth + readOnly: true + - name: application-health-tls + mountPath: /application/health-tls + readOnly: true + - name: application-health-jaas + mountPath: /application/health-jaas + readOnly: true + + # This block configures only the Operator client. The application-owned JVM + # arguments, JAAS file, and volumes above configure the member server. + suspendProbe: + timeoutSeconds: 60 + http: + method: PUT + scheme: HTTPS + path: /suspend + port: 8089 + basicAuth: + username: {name: health-auth-v1, key: username} + password: {name: health-auth-v1, key: password} + tls: + caSecret: {name: health-ca-v1, key: ca.pem} + serverName: health.example + + # Custom probes are necessary because healthMutatorConnection is absent, so + # the Operator does not rewrite generated probes to HTTPS. + readinessProbe: + httpGet: + scheme: HTTPS + path: /ready + port: 8089 + livenessProbe: + httpGet: + scheme: HTTPS + path: /live + port: 8089 + + coherence: + # Qualification-image cache configuration. + cacheConfig: test-cache-config.xml + persistence: + mode: active + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + ports: + # Qualification-only canary endpoints; not part of Coherence Health. + - name: test-rest + port: 8080 diff --git a/test/health-mutator/apply.sh b/test/health-mutator/apply.sh new file mode 100755 index 000000000..3b3d61bb7 --- /dev/null +++ b/test/health-mutator/apply.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at +# http://oss.oracle.com/licenses/upl. +set -euo pipefail + +context=${1:?Kubernetes context is required} +namespace=${2:?namespace is required} +image_digest=${3:?pinned test application image digest is required} +material=${4:?new material directory is required} + +if [[ "$image_digest" != *@sha256:* ]]; then + echo "The test application image must be pinned by digest" >&2 + exit 1 +fi +if [[ -e "$material" ]]; then + echo "Material directory already exists: $material" >&2 + exit 1 +fi + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +bash "$script_dir/prepare.sh" "$material" health.example + +kubectl --context "$context" create namespace "$namespace" --dry-run=client -o yaml | kubectl --context "$context" apply -f - +kubectl --context "$context" -n "$namespace" create secret generic health-auth-v1 \ + --from-file=username="$material/username" \ + --from-file=password="$material/password" \ + --dry-run=client -o yaml | kubectl --context "$context" apply -f - +kubectl --context "$context" -n "$namespace" create secret generic health-stores-v1 \ + --from-file=server.jks="$material/server.jks" \ + --from-file=store-password="$material/store-password" \ + --dry-run=client -o yaml | kubectl --context "$context" apply -f - +kubectl --context "$context" -n "$namespace" create secret generic health-ca-v1 \ + --from-file=ca.pem="$material/ca.pem" \ + --dry-run=client -o yaml | kubectl --context "$context" apply -f - +for secret in health-auth-v1 health-stores-v1 health-ca-v1; do + kubectl --context "$context" -n "$namespace" patch secret "$secret" --type=merge -p '{"immutable":true}' +done + +export IMAGE_DIGEST=$image_digest +export NAMESPACE=$namespace +envsubst "\${IMAGE_DIGEST} \${NAMESPACE}" < "$script_dir/coherence.yaml" > "$material/coherence.yaml" +envsubst "\${IMAGE_DIGEST} \${NAMESPACE}" < "$script_dir/coherence-unconfigured.yaml" > "$material/coherence-unconfigured.yaml" +kubectl --context "$context" apply -f "$material/coherence-unconfigured.yaml" +for _ in $(seq 1 120); do + if kubectl --context "$context" -n "$namespace" get statefulset health-secure >/dev/null 2>&1; then + break + fi + sleep 2 +done +if ! kubectl --context "$context" -n "$namespace" get statefulset health-secure >/dev/null 2>&1; then + echo "StatefulSet was not created" >&2 + exit 1 +fi +kubectl --context "$context" -n "$namespace" rollout status statefulset/health-secure --timeout=10m diff --git a/test/health-mutator/coherence-unconfigured.yaml b/test/health-mutator/coherence-unconfigured.yaml new file mode 100644 index 000000000..3e0bd0473 --- /dev/null +++ b/test/health-mutator/coherence-unconfigured.yaml @@ -0,0 +1,28 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at +# http://oss.oracle.com/licenses/upl. +# This is the reported starting state, before managed health security is applied. +apiVersion: coherence.oracle.com/v1 +kind: Coherence +metadata: + name: health-secure + namespace: ${NAMESPACE} +spec: + replicas: 2 + image: ${IMAGE_DIGEST} + healthPort: 8089 + application: + main: com.oracle.coherence.k8s.testing.RestServer + coherence: + cacheConfig: test-cache-config.xml + persistence: + mode: active + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + ports: + - name: test-rest + port: 8080 diff --git a/test/health-mutator/coherence.yaml b/test/health-mutator/coherence.yaml new file mode 100644 index 000000000..dc36182be --- /dev/null +++ b/test/health-mutator/coherence.yaml @@ -0,0 +1,46 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at +# http://oss.oracle.com/licenses/upl. +# Render IMAGE_DIGEST and NAMESPACE using envsubst before applying. +apiVersion: coherence.oracle.com/v1 +kind: Coherence +metadata: + name: health-secure + namespace: ${NAMESPACE} +spec: + replicas: 2 + image: ${IMAGE_DIGEST} + healthPort: 8089 + application: + main: com.oracle.coherence.k8s.testing.RestServer + coherence: + cacheConfig: test-cache-config.xml + persistence: + mode: active + persistentVolumeClaim: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + ports: + - name: test-rest + port: 8080 + healthMutatorConnection: + secure: true + basicAuth: + username: {name: health-auth-v1, key: username} + password: {name: health-auth-v1, key: password} + tls: + caSecret: {name: health-ca-v1, key: ca.pem} + serverName: health.example + serverTLS: + enabled: true + requireClientCert: false + secrets: health-stores-v1 + keyStore: server.jks + keyStoreType: JKS + keyStorePasswordFile: store-password + keyPasswordFile: store-password + jaas: + mode: packaged diff --git a/test/health-mutator/extract-coherence-jar.sh b/test/health-mutator/extract-coherence-jar.sh new file mode 100755 index 000000000..8a96b6070 --- /dev/null +++ b/test/health-mutator/extract-coherence-jar.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at +# http://oss.oracle.com/licenses/upl. +set -euo pipefail + +image=${1:?Coherence image is required} +version=${2:?Coherence version is required} +output=${3:?output jar path is required} +container=$(docker create "$image") +cleanup() { + docker rm -f "$container" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +source="/app/libs/coherence-${version}.jar" +mkdir -p "$(dirname "$output")" +if ! docker cp "${container}:${source}" "$output"; then + echo "No Coherence jar found at ${source} in ${image}" >&2 + exit 1 +fi +test -s "$output" diff --git a/test/health-mutator/prepare.sh b/test/health-mutator/prepare.sh new file mode 100755 index 000000000..e22fbfc3b --- /dev/null +++ b/test/health-mutator/prepare.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at +# http://oss.oracle.com/licenses/upl. +set -euo pipefail +umask 077 +output=${1:?output directory required} +dns=${2:?certificate DNS name required} +mkdir "$output" +openssl rand -hex 24 > "$output/store-password" +# Credential files intentionally have no trailing newline. +printf 'health-operator' > "$output/username" +printf '%s' "$(openssl rand -hex 32)" > "$output/password" +keytool -genkeypair -alias health -keyalg RSA -keysize 2048 -validity 30 \ + -dname "CN=$dns" -ext "SAN=dns:$dns" -storetype JKS \ + -keystore "$output/server.jks" -storepass:file "$output/store-password" \ + -keypass:file "$output/store-password" >/dev/null 2>&1 +keytool -exportcert -rfc -alias health -keystore "$output/server.jks" \ + -storepass:file "$output/store-password" -file "$output/ca.pem" >/dev/null 2>&1 diff --git a/test/health-mutator/verify.sh b/test/health-mutator/verify.sh new file mode 100755 index 000000000..9c64a29e8 --- /dev/null +++ b/test/health-mutator/verify.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at +# http://oss.oracle.com/licenses/upl. +set -euo pipefail + +context=${1:?Kubernetes context is required} +namespace=${2:?namespace is required} +material=${3:?material directory is required} +health_port=${HEALTH_MUTATOR_LOCAL_PORT:-18089} +rest_port=${HEALTH_MUTATOR_REST_PORT:-18080} +port_forward_pid= + +stop_forward() { + if [[ -n "$port_forward_pid" ]]; then + kill "$port_forward_pid" >/dev/null 2>&1 || true + wait "$port_forward_pid" >/dev/null 2>&1 || true + port_forward_pid= + fi +} +trap stop_forward EXIT + +start_forward() { + scheme=$1 + stop_forward + pod=$(kubectl --context "$context" -n "$namespace" get pod \ + -l coherenceDeployment=health-secure \ + -o jsonpath='{.items[0].metadata.name}') + kubectl --context "$context" -n "$namespace" port-forward "pod/$pod" \ + "$health_port:8089" "$rest_port:8080" > "$material/port-forward.log" 2>&1 & + port_forward_pid=$! + for _ in $(seq 1 60); do + if [[ "$scheme" == https ]]; then + if curl --noproxy '*' --silent --show-error --fail \ + --cacert "$material/ca.pem" \ + --resolve "health.example:$health_port:127.0.0.1" \ + "https://health.example:$health_port/ready" >/dev/null 2>&1; then + return + fi + elif curl --noproxy '*' --silent --show-error --fail \ + "http://127.0.0.1:$health_port/ready" >/dev/null 2>&1; then + return + fi + sleep 1 + done + echo "Health endpoint did not become reachable; see $material/port-forward.log" >&2 + exit 1 +} + +status() { + curl --noproxy '*' --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --cacert "$material/ca.pem" \ + --resolve "health.example:$health_port:127.0.0.1" "$@" +} + +expect_status() { + expected=$1 + shift + actual=$(status "$@") + if [[ "$actual" != "$expected" ]]; then + echo "Expected HTTP $expected but received $actual" >&2 + exit 1 + fi +} + +start_forward http +unconfigured_status=$(curl --noproxy '*' --silent --show-error --output /dev/null --write-out '%{http_code}' \ + --request PUT "http://127.0.0.1:$health_port/suspend") +if [[ "$unconfigured_status" != 403 ]]; then + echo "Expected the unconfigured hardened mutator to return HTTP 403 but received $unconfigured_status" >&2 + exit 1 +fi +stop_forward +kubectl --context "$context" apply -f "$material/coherence.yaml" +for _ in $(seq 1 120); do + annotation=$(kubectl --context "$context" -n "$namespace" get statefulset health-secure \ + -o jsonpath='{.spec.template.metadata.annotations.coherence\.oracle\.com/health-mutator-connection}' 2>/dev/null || true) + if [[ "$annotation" == *health-auth-v1* ]]; then + break + fi + sleep 2 +done +if [[ "$annotation" != *health-auth-v1* ]]; then + echo "StatefulSet did not adopt the managed health configuration" >&2 + exit 1 +fi +kubectl --context "$context" -n "$namespace" rollout status statefulset/health-secure --timeout=10m + +start_forward https +expect_status 200 "https://health.example:$health_port/ready" +expect_status 401 --request PUT "https://health.example:$health_port/suspend" +expect_status 401 --request PUT --user 'health-operator:wrong' "https://health.example:$health_port/suspend" +curl --noproxy '*' --silent --show-error --fail "http://127.0.0.1:$rest_port/canaryStart" >/dev/null +expect_status 200 --request PUT \ + --user "$(<"$material/username"):$(<"$material/password")" \ + "https://health.example:$health_port/suspend" +expect_status 200 --request PUT \ + --user "$(<"$material/username"):$(<"$material/password")" \ + "https://health.example:$health_port/resume" +if curl --noproxy '*' --silent --show-error --fail \ + --cacert "$material/password" \ + --resolve "health.example:$health_port:127.0.0.1" \ + "https://health.example:$health_port/ready" >/dev/null 2>&1; then + echo "Invalid CA unexpectedly passed TLS verification" >&2 + exit 1 +fi +stop_forward + +kubectl --context "$context" -n "$namespace" create secret generic health-ca-failure-v1 \ + --from-file=ca.pem="$material/ca.pem" --dry-run=client -o yaml | \ + kubectl --context "$context" apply -f - +kubectl --context "$context" -n "$namespace" patch coherence.coherence.oracle.com health-secure --type=merge \ + -p '{"spec":{"healthMutatorConnection":{"tls":{"caSecret":{"name":"health-ca-failure-v1","key":"ca.pem"},"serverName":"health.example"}}}}' +for _ in $(seq 1 120); do + annotation=$(kubectl --context "$context" -n "$namespace" get statefulset health-secure \ + -o jsonpath='{.spec.template.metadata.annotations.coherence\.oracle\.com/health-mutator-connection}' 2>/dev/null || true) + if [[ "$annotation" == *health-ca-failure-v1* ]]; then + break + fi + sleep 2 +done +if [[ "$annotation" != *health-ca-failure-v1* ]]; then + echo "StatefulSet did not adopt the failure-injection CA reference" >&2 + exit 1 +fi +kubectl --context "$context" -n "$namespace" rollout status statefulset/health-secure --timeout=10m +printf 'not a certificate\n' > "$material/bad-ca.pem" +kubectl --context "$context" -n "$namespace" create secret generic health-ca-failure-v1 \ + --from-file=ca.pem="$material/bad-ca.pem" --dry-run=client -o yaml | \ + kubectl --context "$context" apply -f - +kubectl --context "$context" -n "$namespace" patch coherence.coherence.oracle.com health-secure --type=merge -p '{"spec":{"replicas":0}}' +for _ in $(seq 1 120); do + warning_events=$(kubectl --context "$context" -n "$namespace" get events \ + --field-selector involvedObject.kind=Coherence,involvedObject.name=health-secure,type=Warning \ + -o jsonpath='{range .items[?(@.reason=="Scaling")]}{.message}{"\n"}{end}' 2>/dev/null || true) + if [[ "$warning_events" == *"failed suspending Coherence services"* ]]; then + break + fi + sleep 2 +done +if [[ "$warning_events" != *"failed suspending Coherence services"* ]]; then + echo "Operator did not report a failed suspension with invalid trust" >&2 + exit 1 +fi +if ! kubectl --context "$context" -n "$namespace" get statefulset health-secure >/dev/null 2>&1; then + echo "Scale-to-zero was not blocked by invalid trust" >&2 + exit 1 +fi + +kubectl --context "$context" -n "$namespace" create secret generic health-ca-failure-v1 \ + --from-file=ca.pem="$material/ca.pem" --dry-run=client -o yaml | \ + kubectl --context "$context" apply -f - +for _ in $(seq 1 120); do + if ! kubectl --context "$context" -n "$namespace" get statefulset health-secure >/dev/null 2>&1; then + break + fi + sleep 2 +done +if kubectl --context "$context" -n "$namespace" get statefulset health-secure >/dev/null 2>&1; then + echo "StatefulSet did not reach scale-to-zero after restoring trust" >&2 + exit 1 +fi + +kubectl --context "$context" -n "$namespace" patch coherence.coherence.oracle.com health-secure --type=merge -p '{"spec":{"replicas":2}}' +for _ in $(seq 1 120); do + if kubectl --context "$context" -n "$namespace" get statefulset health-secure >/dev/null 2>&1; then + break + fi + sleep 2 +done +if ! kubectl --context "$context" -n "$namespace" get statefulset health-secure >/dev/null 2>&1; then + echo "StatefulSet was not recreated" >&2 + exit 1 +fi +kubectl --context "$context" -n "$namespace" rollout status statefulset/health-secure --timeout=10m +start_forward https +curl --noproxy '*' --silent --show-error --fail "http://127.0.0.1:$rest_port/canaryCheck" >/dev/null +echo "Unconfigured 403, authenticated suspension, fail-closed TLS, scale-to-zero, restart, and persistent canary verification passed" diff --git a/utils/crd-fixup/main.go b/utils/crd-fixup/main.go new file mode 100644 index 000000000..6f4167ce4 --- /dev/null +++ b/utils/crd-fixup/main.go @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl. + */ +// crd-fixup supplies the existing string zero value for optional list-map keys. +// Kubernetes requires every list-map key to be required or defaulted. Embedded +// upstream API structs can expose optional keys even when the containing CRD list +// is a map. Defaulting preserves the Go zero-value behavior without changing +// server-side apply list semantics. +package main + +import ( + "fmt" + "os" + "sigs.k8s.io/yaml" +) + +func main() { + for _, name := range os.Args[1:] { + data, err := os.ReadFile(name) + if err != nil { + panic(err) + } + var schema map[string]interface{} + if err := yaml.Unmarshal(data, &schema); err != nil { + panic(err) + } + fix(schema) + data, err = yaml.Marshal(schema) + if err != nil { + panic(err) + } + if err := os.WriteFile(name, data, 0644); err != nil { + panic(err) + } + } +} + +func fix(node interface{}) { + switch value := node.(type) { + case map[string]interface{}: + if keys, ok := value["x-kubernetes-list-map-keys"].([]interface{}); ok { + items, _ := value["items"].(map[string]interface{}) + properties, _ := items["properties"].(map[string]interface{}) + required, _ := items["required"].([]interface{}) + for _, key := range keys { + property, _ := properties[fmt.Sprint(key)].(map[string]interface{}) + isRequired := false + for _, name := range required { + if name == key { + isRequired = true + } + } + if property != nil && !isRequired && property["default"] == nil && property["type"] == "string" { + property["default"] = "" + } + } + } + for _, child := range value { + fix(child) + } + case []interface{}: + for _, child := range value { + fix(child) + } + } +}