Add TLS support for metrics endpoints with service CA certificates

This commit implements TLS encryption for both SSP operator and template validator metrics services.
The implementation handles both OLM and non-OLM deployments with proper certificate management.

Changes:
- Add --olm-deployment flag to distinguish OLM-managed deployments
- Create separate ServiceMonitors for SSP and template validator metrics
- Configure TLS settings with appropriate CA bundles based on deployment type
- Move common service names to constants package for reusability
- Update ServiceMonitor selector labels to differentiate between services

Technical details:
- Non-OLM deployments use service-ca.crt from ConfigMap
- OLM deployments use ssp-operator-service-cert Secret with olmCAKey

Signed-off-by: Michal Vavrinec <[email protected]>

Author: Ruclo

data/olm-catalog/ssp-operator.clusterserviceversion.yaml

@@ -343,6 +343,7 @@ spec:
               containers:
               - args:
                 - --leader-elect
+                - --olm-deployment
                 command:
                 - /manager
                 env:

hack/csv-generator.go

@@ -20,15 +20,16 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
+	"os"
+
 	"github.com/blang/semver/v4"
 	"github.com/operator-framework/api/pkg/lib/version"
 	csvv1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
 	"github.com/spf13/cobra"
-	"io"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/yaml"
-	"os"
 	sigsyaml "sigs.k8s.io/yaml"
 
 	"kubevirt.io/ssp-operator/internal/env"
@@ -120,6 +121,8 @@ func runGenerator() error {
 		removeCerts(&csv)
 	}
 
+	addOLMArg(&csv)
+
 	relatedImages, err := buildRelatedImages(f)
 	if err != nil {
 		return err
@@ -144,6 +147,16 @@ func runGenerator() error {
 	return nil
 }
 
+func addOLMArg(csv *csvv1.ClusterServiceVersion) {
+	templateSpec := &csv.Spec.InstallStrategy.StrategySpec.DeploymentSpecs[0].Spec.Template.Spec
+	for i, container := range templateSpec.Containers {
+		if container.Name == "manager" {
+			templateSpec.Containers[i].Args = append(container.Args, "--olm-deployment")
+			break
+		}
+	}
+}
+
 func dumpFiles(path string) error {
 	files, err := os.ReadDir(path)
 	if err != nil {

internal/common/constants.go

@@ -0,0 +1,7 @@
+package common
+
+const (
+	SspOperatorMetricsServiceName       = "ssp-operator-metrics"
+	TemplateValidatorMetricsServiceName = "template-validator-metrics"
+	VirtTemplateValidator               = "virt-template-validator"
+)

internal/common/request.go

@@ -23,7 +23,9 @@ type Request struct {
 	VersionCache    VersionCache
 	TopologyMode    osconfv1.TopologyMode
 
-	CrdList crd_watch.CrdList
+	CrdList            crd_watch.CrdList
+	OLMDeployment      bool
+	SSPServiceHostname string
 }
 
 func (r *Request) IsSingleReplicaTopologyMode() bool {

internal/controllers/services_controller.go

@@ -26,7 +26,6 @@ import (
 
 const (
 	ServiceManagedByLabelValue = "ssp-operator-services"
-	MetricsServiceName         = "ssp-operator-metrics"
 	OperatorName               = "ssp-operator"
 	ServiceControllerName      = "service-controller"
 )
@@ -38,13 +37,14 @@ func ServiceObject(namespace string, appKubernetesPartOfValue string) *v1.Servic
 		common.AppKubernetesVersionLabel:   env.GetOperatorVersion(),
 		common.AppKubernetesComponentLabel: ServiceControllerName,
 		metrics.PrometheusLabelKey:         metrics.PrometheusLabelValue,
+		metrics.MetricsServiceKey:          common.SspOperatorMetricsServiceName,
 	}
 	if appKubernetesPartOfValue != "" {
 		labels[common.AppKubernetesPartOfLabel] = appKubernetesPartOfValue
 	}
 	return &v1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      MetricsServiceName,
+			Name:      common.SspOperatorMetricsServiceName,
 			Namespace: namespace,
 			Labels:    labels,
 		},
@@ -136,7 +136,7 @@ func (s *serviceReconciler) setupController(mgr ctrl.Manager) error {
 		Named("service-controller").
 		For(&v1.Service{}, builder.WithPredicates(predicate.NewPredicateFuncs(
 			func(object client.Object) bool {
-				return object.GetName() == MetricsServiceName && object.GetNamespace() == s.operatorNamespace
+				return object.GetName() == common.SspOperatorMetricsServiceName && object.GetNamespace() == s.operatorNamespace
 			}))).
 		Complete(s)
 }

internal/controllers/setup.go

@@ -43,7 +43,7 @@ func StartControllers(ctx context.Context, mgr controllerruntime.Manager, contro
 	return nil
 }
 
-func CreateControllers(ctx context.Context, apiReader client.Reader) ([]Controller, error) {
+func CreateControllers(ctx context.Context, apiReader client.Reader, olmDeployment bool, sspServiceHostname string) ([]Controller, error) {
 	runningOnOpenShift, err := env.RunningOnOpenshift(ctx, apiReader)
 	if err != nil {
 		return nil, fmt.Errorf("failed to check if running on openshift: %w", err)
@@ -102,7 +102,7 @@ func CreateControllers(ctx context.Context, apiReader client.Reader) ([]Controll
 		serviceController,
 		NewWebhookConfigurationController(),
 		NewVmController(),
-		NewSspController(infrastructureTopology, sspOperands),
+		NewSspController(infrastructureTopology, sspOperands, olmDeployment, sspServiceHostname),
 	}, nil
 }
 

internal/controllers/ssp_controller.go

@@ -61,24 +61,28 @@ const (
 
 // sspController reconciles a SSP object
 type sspController struct {
-	log              logr.Logger
-	operands         []operands.Operand
-	lastSspSpec      ssp.SSPSpec
-	subresourceCache common.VersionCache
-	topologyMode     osconfv1.TopologyMode
-	areCrdsMissing   bool
+	log                logr.Logger
+	operands           []operands.Operand
+	lastSspSpec        ssp.SSPSpec
+	subresourceCache   common.VersionCache
+	topologyMode       osconfv1.TopologyMode
+	areCrdsMissing     bool
+	olmDeployment      bool
+	sspServiceHostname string
 
 	client         client.Client
 	uncachedReader client.Reader
 	crdList        crd_watch.CrdList
 }
 
-func NewSspController(infrastructureTopology osconfv1.TopologyMode, operands []operands.Operand) Controller {
+func NewSspController(infrastructureTopology osconfv1.TopologyMode, operands []operands.Operand, olmDeployment bool, sspServiceHostname string) Controller {
 	return &sspController{
-		log:              ctrl.Log.WithName("controllers").WithName("SSP"),
-		operands:         operands,
-		subresourceCache: common.VersionCache{},
-		topologyMode:     infrastructureTopology,
+		log:                ctrl.Log.WithName("controllers").WithName("SSP"),
+		operands:           operands,
+		subresourceCache:   common.VersionCache{},
+		topologyMode:       infrastructureTopology,
+		olmDeployment:      olmDeployment,
+		sspServiceHostname: sspServiceHostname,
 	}
 }
 
@@ -175,16 +179,18 @@ func (s *sspController) Reconcile(ctx context.Context, req ctrl.Request) (res ct
 	sspChanged := s.clearCacheIfNeeded(instance)
 
 	sspRequest := &common.Request{
-		Request:         req,
-		Client:          s.client,
-		UncachedReader:  s.uncachedReader,
-		Context:         ctx,
-		Instance:        instance,
-		InstanceChanged: sspChanged,
-		Logger:          reqLogger,
-		VersionCache:    s.subresourceCache,
-		TopologyMode:    s.topologyMode,
-		CrdList:         s.crdList,
+		Request:            req,
+		Client:             s.client,
+		UncachedReader:     s.uncachedReader,
+		Context:            ctx,
+		Instance:           instance,
+		InstanceChanged:    sspChanged,
+		Logger:             reqLogger,
+		VersionCache:       s.subresourceCache,
+		TopologyMode:       s.topologyMode,
+		CrdList:            s.crdList,
+		OLMDeployment:      s.olmDeployment,
+		SSPServiceHostname: s.sspServiceHostname,
 	}
 
 	if !isInitialized(sspRequest.Instance) {

internal/operands/metrics/reconcile.go

@@ -50,7 +50,8 @@ func (m *metrics) WatchClusterTypes() []operands.WatchType {
 
 func (m *metrics) Reconcile(request *common.Request) ([]common.ReconcileResult, error) {
 	return common.CollectResourceStatus(request,
-		reconcilePrometheusMonitor,
+		reconcileValidatorMetricsMonitor,
+		reconcileSspMetricsMonitor,
 		reconcilePrometheusRule,
 		reconcileMonitoringRbacRole,
 		reconcileMonitoringRbacRoleBinding,
@@ -75,9 +76,16 @@ const (
 	operandComponent = common.AppComponentMonitoring
 )
 
-func reconcilePrometheusMonitor(request *common.Request) (common.ReconcileResult, error) {
+func reconcileSspMetricsMonitor(request *common.Request) (common.ReconcileResult, error) {
 	return common.CreateOrUpdate(request).
-		NamespacedResource(newServiceMonitorCR(request.Namespace)).
+		NamespacedResource(newSspServiceMonitor(*request)).
+		WithAppLabels(operandName, operandComponent).
+		Reconcile()
+}
+
+func reconcileValidatorMetricsMonitor(request *common.Request) (common.ReconcileResult, error) {
+	return common.CreateOrUpdate(request).
+		NamespacedResource(newValidatorServiceMonitor(*request)).
 		WithAppLabels(operandName, operandComponent).
 		Reconcile()
 }

internal/operands/metrics/reconcile_test.go

@@ -74,7 +74,8 @@ var _ = Describe("Metrics operand", func() {
 		Expect(err).ToNot(HaveOccurred())
 
 		ExpectResourceExists(prometheusRule, request)
-		ExpectResourceExists(newServiceMonitorCR(namespace), request)
+		ExpectResourceExists(newSspServiceMonitor(request), request)
+		ExpectResourceExists(newValidatorServiceMonitor(request), request)
 		ExpectResourceExists(newMonitoringClusterRole(), request)
 		ExpectResourceExists(newMonitoringClusterRoleBinding(), request)
 	})

internal/operands/metrics/resources.go

@@ -1,12 +1,14 @@
 package metrics
 
 import (
+	"fmt"
 	promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	v1 "k8s.io/api/core/v1"
 	rbac "k8s.io/api/rbac/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/ptr"
 
-	"kubevirt.io/ssp-operator/pkg/monitoring/rules"
+	"kubevirt.io/ssp-operator/internal/common"
 )
 
 const (
@@ -18,6 +20,7 @@ const (
 	PrometheusClusterRoleName    = "prometheus-k8s-ssp"
 	PrometheusServiceAccountName = "prometheus-k8s"
 	MetricsPortName              = "http-metrics"
+	MetricsServiceKey            = "metrics.ssp.kubevirt.io"
 )
 
 func newMonitoringClusterRole() *rbac.ClusterRole {
@@ -61,31 +64,87 @@ func ServiceMonitorLabels() map[string]string {
 	}
 }
 
-func newServiceMonitorCR(namespace string) *promv1.ServiceMonitor {
-	return &promv1.ServiceMonitor{
+func serviceCABundle() promv1.SecretOrConfigMap {
+	return promv1.SecretOrConfigMap{
+		ConfigMap: &v1.ConfigMapKeySelector{
+			LocalObjectReference: v1.LocalObjectReference{
+				Name: "openshift-service-ca.crt",
+			},
+			Key: "service-ca.crt",
+		},
+	}
+}
+
+func olmManagedCABundle() promv1.SecretOrConfigMap {
+	return promv1.SecretOrConfigMap{
+		Secret: &v1.SecretKeySelector{
+			LocalObjectReference: v1.LocalObjectReference{
+				Name: "ssp-operator-service-cert",
+			},
+			Key: "olmCAKey",
+		},
+	}
+}
+
+func getCAConfigForServiceMonitor(olmDeployment bool) promv1.SecretOrConfigMap {
+	if olmDeployment {
+		return olmManagedCABundle()
+	}
+	return serviceCABundle()
+}
+
+func newValidatorServiceMonitor(request common.Request) *promv1.ServiceMonitor {
+	tlsConfig := &promv1.TLSConfig{
+		SafeTLSConfig: promv1.SafeTLSConfig{
+			CA: serviceCABundle(),
+		},
+	}
+
+	tlsConfig.ServerName = ptr.To(fmt.Sprintf("%s.%s.svc", common.VirtTemplateValidator, request.Namespace))
+	serviceMonitor := newServiceMonitor(common.TemplateValidatorMetricsServiceName, request.Namespace, tlsConfig, metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			MetricsServiceKey: common.TemplateValidatorMetricsServiceName,
+		},
+	})
+	return &serviceMonitor
+}
+
+func newSspServiceMonitor(request common.Request) *promv1.ServiceMonitor {
+	tlsConfig := &promv1.TLSConfig{
+		SafeTLSConfig: promv1.SafeTLSConfig{
+			CA: getCAConfigForServiceMonitor(request.OLMDeployment),
+		},
+	}
+	tlsConfig.ServerName = ptr.To(request.SSPServiceHostname)
+
+	serviceMonitor := newServiceMonitor(common.SspOperatorMetricsServiceName, request.Namespace, tlsConfig, metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			MetricsServiceKey: common.SspOperatorMetricsServiceName,
+		},
+	})
+	return &serviceMonitor
+}
+
+func newServiceMonitor(name,
+	namespace string,
+	tlsConfig *promv1.TLSConfig,
+	selector metav1.LabelSelector) promv1.ServiceMonitor {
+	return promv1.ServiceMonitor{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: namespace,
-			Name:      rules.RuleName,
+			Name:      name,
 			Labels:    ServiceMonitorLabels(),
 		},
 		Spec: promv1.ServiceMonitorSpec{
 			NamespaceSelector: promv1.NamespaceSelector{
 				Any: true,
 			},
-			Selector: metav1.LabelSelector{
-				MatchLabels: map[string]string{
-					PrometheusLabelKey: PrometheusLabelValue,
-				},
-			},
+			Selector: selector,
 			Endpoints: []promv1.Endpoint{
 				{
-					Port:   MetricsPortName,
-					Scheme: ptr.To(promv1.Scheme("https")),
-					TLSConfig: &promv1.TLSConfig{
-						SafeTLSConfig: promv1.SafeTLSConfig{
-							InsecureSkipVerify: ptr.To(true),
-						},
-					},
+					Port:        MetricsPortName,
+					Scheme:      ptr.To(promv1.Scheme("https")),
+					TLSConfig:   tlsConfig,
 					HonorLabels: true,
 				},
 			},