Update startupProbe to use HTTP health endpoint, retain legacy option.

.github/workflows/build-test-publish.yml

@@ -324,6 +324,9 @@ jobs:
         KIND_NODE_IMAGE: ${{ env.KIND_NODE_IMAGE }}
         RABBITMQ_IMAGE: ${{ matrix.rabbitmq-image }}
         RABBITMQ_SERVICE_TYPE: NodePort
+        # RabbitMQ 3.13.7 predates the /health/checks/reached-target-cluster-size HTTP endpoint,
+        # so fall back to the legacy rabbitmqctl-based startup probe for that image only.
+        RABBITMQ_CLUSTER_ANNOTATION: ${{ matrix.rabbitmq-image == 'rabbitmq:3.13.7-management' && 'rabbitmq.com/legacy-startup-probe=true' || '' }}
       # 'make deploy-kind' builds the image locally. We should avoid using that target, because we have already built
       # the image in a previous job
       run: |
@@ -395,6 +398,9 @@ jobs:
         KIND_NODE_IMAGE: ${{ env.KIND_OLDEST_NODE_IMAGE }}
         RABBITMQ_IMAGE: ${{ matrix.rabbitmq-image }}
         RABBITMQ_SERVICE_TYPE: NodePort
+        # RabbitMQ 3.13.7 predates the /health/checks/reached-target-cluster-size HTTP endpoint,
+        # so fall back to the legacy rabbitmqctl-based startup probe for that image only.
+        RABBITMQ_CLUSTER_ANNOTATION: ${{ matrix.rabbitmq-image == 'rabbitmq:3.13.7-management' && 'rabbitmq.com/legacy-startup-probe=true' || '' }}
       # make system-tests will install required tools e.g. ginkgo
       run: |
         make cert-manager

api/v1beta1/rabbitmqcluster_types.go

@@ -27,6 +27,7 @@ const (
 	RabbitmqVersionAnnotation              = "rabbitmq.com/version"
 	ErlangVersionAnnotation                = "rabbitmq.com/erlang-version"
 	VersionNotAnnotated                    = "VersionNotAnnotated"
+	LegacyStartupProbeAnnotation           = "rabbitmq.com/legacy-startup-probe"
 )
 
 // +kubebuilder:object:root=true

docs/examples/community-plugins/rabbitmq.yaml

@@ -2,6 +2,8 @@ apiVersion: rabbitmq.com/v1beta1
 kind: RabbitmqCluster
 metadata:
   name: community-plugins
+  annotations:
+    "rabbitmq.com/legacy-startup-probe": "true"
 spec:
   replicas: 1
   image: rabbitmq:3.13-management

internal/controller/rabbitmqcluster_controller_test.go

@@ -97,11 +97,10 @@ var _ = Describe("RabbitmqClusterController", func() {
 				rabbitmqContainer := sts.Spec.Template.Spec.Containers[0]
 				Expect(rabbitmqContainer.Name).To(Equal("rabbitmq"))
 				Expect(rabbitmqContainer.StartupProbe).NotTo(BeNil())
-				Expect(rabbitmqContainer.StartupProbe.ProbeHandler.Exec).NotTo(BeNil())
-				Expect(rabbitmqContainer.StartupProbe.ProbeHandler.Exec.Command).To(Equal([]string{
-					"/bin/bash", "-c",
-					"rabbitmqctl eval 'rabbit_nodes:reached_target_cluster_size().' | grep -q '^true$'",
-				}))
+				Expect(rabbitmqContainer.StartupProbe.ProbeHandler.HTTPGet).NotTo(BeNil())
+				Expect(rabbitmqContainer.StartupProbe.ProbeHandler.HTTPGet.Path).To(Equal("/api/health/checks/reached-target-cluster-size"))
+				Expect(rabbitmqContainer.StartupProbe.ProbeHandler.HTTPGet.Port).To(Equal(intstr.FromString("management")))
+				Expect(rabbitmqContainer.StartupProbe.ProbeHandler.HTTPGet.Scheme).To(Equal(corev1.URISchemeHTTP))
 				Expect(rabbitmqContainer.StartupProbe.InitialDelaySeconds).To(BeEquivalentTo(10))
 				Expect(rabbitmqContainer.StartupProbe.TimeoutSeconds).To(BeEquivalentTo(5))
 				Expect(rabbitmqContainer.StartupProbe.PeriodSeconds).To(BeEquivalentTo(10))

internal/resource/statefulset.go

@@ -641,19 +641,7 @@ func (builder *StatefulSetBuilder) podTemplateSpec(previousPodAnnotations map[st
 						SuccessThreshold:    1,
 						FailureThreshold:    3,
 					},
-					// TODO: Update this probe once we have an HTTP API endpoint for this
-					StartupProbe: &corev1.Probe{
-						ProbeHandler: corev1.ProbeHandler{
-							Exec: &corev1.ExecAction{
-								Command: []string{"/bin/bash", "-c",
-									"rabbitmqctl eval 'rabbit_nodes:reached_target_cluster_size().' | grep -q '^true$'"},
-							},
-						},
-						InitialDelaySeconds: 10,
-						TimeoutSeconds:      5,
-						PeriodSeconds:       10,
-						FailureThreshold:    30,
-					},
+					StartupProbe: builder.startupProbe(),
 					Lifecycle: &corev1.Lifecycle{
 						PreStop: &corev1.LifecycleHandler{
 							Exec: &corev1.ExecAction{
@@ -704,6 +692,44 @@ func (builder *StatefulSetBuilder) rabbitmqConfigurationIsSet() bool {
 		builder.Instance.Spec.Rabbitmq.ErlangInetConfig != ""
 }
 
+func (builder *StatefulSetBuilder) startupProbe() *corev1.Probe {
+	if _, ok := builder.Instance.Annotations[rabbitmqv1beta1.LegacyStartupProbeAnnotation]; ok {
+		return &corev1.Probe{
+			ProbeHandler: corev1.ProbeHandler{
+				Exec: &corev1.ExecAction{
+					Command: []string{"/bin/bash", "-c",
+						"[[ \"true\" == \"$(rabbitmqctl eval 'rabbit_nodes:reached_target_cluster_size().')\" ]]"},
+				},
+			},
+			InitialDelaySeconds: 10,
+			TimeoutSeconds:      5,
+			PeriodSeconds:       10,
+			FailureThreshold:    30,
+		}
+	}
+
+	port := intstr.FromString("management")
+	scheme := corev1.URISchemeHTTP
+	if builder.Instance.DisableNonTLSListeners() {
+		port = intstr.FromString("management-tls")
+		scheme = corev1.URISchemeHTTPS
+	}
+
+	return &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   "/api/health/checks/reached-target-cluster-size",
+				Port:   port,
+				Scheme: scheme,
+			},
+		},
+		InitialDelaySeconds: 10,
+		TimeoutSeconds:      5,
+		PeriodSeconds:       10,
+		FailureThreshold:    30,
+	}
+}
+
 func defaultUserCredentialUpdater(instance *rabbitmqv1beta1.RabbitmqCluster) corev1.Container {
 	managementURI := "http://127.0.0.1:15672"
 	if instance.TLSEnabled() {

internal/resource/statefulset_test.go

@@ -763,6 +763,15 @@ var _ = Describe("StatefulSet", func() {
 					Expect(TCPProbe.Port.Type).To(Equal(intstr.String))
 					Expect(TCPProbe.Port.StrVal).To(Equal("amqps"))
 				})
+
+				It("sets StartupProbe to use HTTPS on management-tls port", func() {
+					Expect(stsBuilder.Update(statefulSet)).To(Succeed())
+					container := extractContainer(statefulSet.Spec.Template.Spec.Containers, "rabbitmq")
+					Expect(container.StartupProbe.ProbeHandler.HTTPGet).NotTo(BeNil())
+					Expect(container.StartupProbe.ProbeHandler.HTTPGet.Path).To(Equal("/api/health/checks/reached-target-cluster-size"))
+					Expect(container.StartupProbe.ProbeHandler.HTTPGet.Port).To(Equal(intstr.FromString("management-tls")))
+					Expect(container.StartupProbe.ProbeHandler.HTTPGet.Scheme).To(Equal(corev1.URISchemeHTTPS))
+				})
 			})
 		})
 
@@ -898,23 +907,47 @@ var _ = Describe("StatefulSet", func() {
 			Expect(container.Env).To(ConsistOf(requiredEnvVariables))
 		})
 
-		It("sets default StartupProbe with rabbitmqctl eval command", func() {
+		It("sets default StartupProbe using HTTP management API", func() {
 			stsBuilder := builder.StatefulSet()
 			Expect(stsBuilder.Update(statefulSet)).To(Succeed())
 
 			container := extractContainer(statefulSet.Spec.Template.Spec.Containers, "rabbitmq")
 			Expect(container.StartupProbe).NotTo(BeNil())
-			Expect(container.StartupProbe.ProbeHandler.Exec).NotTo(BeNil())
-			Expect(container.StartupProbe.ProbeHandler.Exec.Command).To(Equal([]string{
-				"/bin/bash", "-c",
-				"rabbitmqctl eval 'rabbit_nodes:reached_target_cluster_size().' | grep -q '^true$'",
-			}))
+			Expect(container.StartupProbe.ProbeHandler.HTTPGet).NotTo(BeNil())
+			Expect(container.StartupProbe.ProbeHandler.HTTPGet.Path).To(Equal("/api/health/checks/reached-target-cluster-size"))
+			Expect(container.StartupProbe.ProbeHandler.HTTPGet.Port).To(Equal(intstr.FromString("management")))
+			Expect(container.StartupProbe.ProbeHandler.HTTPGet.Scheme).To(Equal(corev1.URISchemeHTTP))
 			Expect(container.StartupProbe.InitialDelaySeconds).To(BeEquivalentTo(10))
 			Expect(container.StartupProbe.TimeoutSeconds).To(BeEquivalentTo(5))
 			Expect(container.StartupProbe.PeriodSeconds).To(BeEquivalentTo(10))
 			Expect(container.StartupProbe.FailureThreshold).To(BeEquivalentTo(30))
 		})
 
+		When("annotation rabbitmq.com/legacy-startup-probe is present", func() {
+			BeforeEach(func() {
+				instance.Annotations = map[string]string{
+					rabbitmqv1beta1.LegacyStartupProbeAnnotation: "true",
+				}
+			})
+
+			It("uses legacy exec StartupProbe with rabbitmqctl eval command", func() {
+				stsBuilder := builder.StatefulSet()
+				Expect(stsBuilder.Update(statefulSet)).To(Succeed())
+
+				container := extractContainer(statefulSet.Spec.Template.Spec.Containers, "rabbitmq")
+				Expect(container.StartupProbe).NotTo(BeNil())
+				Expect(container.StartupProbe.ProbeHandler.Exec).NotTo(BeNil())
+				Expect(container.StartupProbe.ProbeHandler.Exec.Command).To(Equal([]string{
+					"/bin/bash", "-c",
+					"[[ \"true\" == \"$(rabbitmqctl eval 'rabbit_nodes:reached_target_cluster_size().')\" ]]",
+				}))
+				Expect(container.StartupProbe.InitialDelaySeconds).To(BeEquivalentTo(10))
+				Expect(container.StartupProbe.TimeoutSeconds).To(BeEquivalentTo(5))
+				Expect(container.StartupProbe.PeriodSeconds).To(BeEquivalentTo(10))
+				Expect(container.StartupProbe.FailureThreshold).To(BeEquivalentTo(30))
+			})
+		})
+
 		Context("ExternalSecret", func() {
 			When("SecretBackend.ExternalSecret is set", func() {
 				JustBeforeEach(func() {

test/system/utils_test.go

@@ -444,6 +444,17 @@ func newRabbitmqCluster(namespace, instanceName string) *rabbitmqv1beta1.Rabbitm
 		}
 	}
 
+	// RABBITMQ_CLUSTER_ANNOTATION, in "key=value" form, annotates every cluster
+	// created by the system tests so the operator can detect them (e.g. for StartupProbe).
+	if annotationSpec := os.Getenv("RABBITMQ_CLUSTER_ANNOTATION"); annotationSpec != "" {
+		if key, value, found := strings.Cut(annotationSpec, "="); found {
+			if cluster.Annotations == nil {
+				cluster.Annotations = map[string]string{}
+			}
+			cluster.Annotations[key] = value
+		}
+	}
+
 	return cluster
 }