Skip to content

Commit 7204d52

Browse files
RucloRuclo
committed
use tls for prometheus metrics in ssp operator and template validator
Signed-off-by: Michal Vavrinec <[email protected]>
1 parent 8a83fdf commit 7204d52

File tree

6 files changed

+182
-55
lines changed

6 files changed

+182
-55
lines changed

‎internal/controllers/services_controller.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ func ServiceObject(namespace string, appKubernetesPartOfValue string) *v1.Servic
3838
common.AppKubernetesVersionLabel: env.GetOperatorVersion(),
3939
common.AppKubernetesComponentLabel: ServiceControllerName,
4040
metrics.PrometheusLabelKey: metrics.PrometheusLabelValue,
41+
metrics.MetricsServiceKey: MetricsServiceName,
4142
}
4243
if appKubernetesPartOfValue != "" {
4344
labels[common.AppKubernetesPartOfLabel] = appKubernetesPartOfValue

‎internal/operands/metrics/reconcile.go‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,8 @@ func (m *metrics) WatchClusterTypes() []operands.WatchType {
5050

5151
func (m *metrics) Reconcile(request *common.Request) ([]common.ReconcileResult, error) {
5252
return common.CollectResourceStatus(request,
53-
reconcilePrometheusMonitor,
53+
reconcileValidatorMetricsMonitor,
54+
reconcileSspMetricsMonitor,
5455
reconcilePrometheusRule,
5556
reconcileMonitoringRbacRole,
5657
reconcileMonitoringRbacRoleBinding,
@@ -75,9 +76,16 @@ const (
7576
operandComponent = common.AppComponentMonitoring
7677
)
7778

78-
func reconcilePrometheusMonitor(request *common.Request) (common.ReconcileResult, error) {
79+
func reconcileSspMetricsMonitor(request *common.Request) (common.ReconcileResult, error) {
7980
return common.CreateOrUpdate(request).
80-
NamespacedResource(newServiceMonitorCR(request.Namespace)).
81+
NamespacedResource(newSspServiceMonitor(request.Namespace)).
82+
WithAppLabels(operandName, operandComponent).
83+
Reconcile()
84+
}
85+
86+
func reconcileValidatorMetricsMonitor(request *common.Request) (common.ReconcileResult, error) {
87+
return common.CreateOrUpdate(request).
88+
NamespacedResource(newValidatorServiceMonitor(request.Namespace)).
8189
WithAppLabels(operandName, operandComponent).
8290
Reconcile()
8391
}

‎internal/operands/metrics/reconcile_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ var _ = Describe("Metrics operand", func() {
7474
Expect(err).ToNot(HaveOccurred())
7575

7676
ExpectResourceExists(prometheusRule, request)
77-
ExpectResourceExists(newServiceMonitorCR(namespace), request)
77+
ExpectResourceExists(newSspServiceMonitor(namespace), request)
7878
ExpectResourceExists(newMonitoringClusterRole(), request)
7979
ExpectResourceExists(newMonitoringClusterRoleBinding(), request)
8080
})

‎internal/operands/metrics/resources.go‎

Lines changed: 133 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,43 @@
11
package metrics
22

33
import (
4+
"crypto/x509"
5+
"encoding/pem"
6+
"fmt"
7+
"os"
8+
"path/filepath"
9+
410
promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
11+
v1 "k8s.io/api/core/v1"
512
rbac "k8s.io/api/rbac/v1"
613
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
714
"k8s.io/utils/ptr"
8-
915
"kubevirt.io/ssp-operator/pkg/monitoring/rules"
1016
)
1117

1218
const (
13-
MonitorNamespace = "openshift-monitoring"
14-
defaultRunbookURLTemplate = "https://kubevirt.io/monitoring/runbooks/%s"
15-
runbookURLTemplateEnv = "RUNBOOK_URL_TEMPLATE"
16-
PrometheusLabelKey = "prometheus.ssp.kubevirt.io"
17-
PrometheusLabelValue = "true"
18-
PrometheusClusterRoleName = "prometheus-k8s-ssp"
19-
PrometheusServiceAccountName = "prometheus-k8s"
20-
MetricsPortName = "http-metrics"
19+
MonitorNamespace = "openshift-monitoring"
20+
defaultRunbookURLTemplate = "https://kubevirt.io/monitoring/runbooks/%s"
21+
runbookURLTemplateEnv = "RUNBOOK_URL_TEMPLATE"
22+
PrometheusLabelKey = "prometheus.ssp.kubevirt.io"
23+
PrometheusLabelValue = "true"
24+
PrometheusClusterRoleName = "prometheus-k8s-ssp"
25+
PrometheusServiceAccountName = "prometheus-k8s"
26+
MetricsPortName = "http-metrics"
27+
CertFilename = "tls.crt"
28+
DefaultCertsDirectory = "/tmp/k8s-webhook-server/serving-certs"
29+
TemplateValidatorMetricsServiceName = "template-validator-metrics"
30+
MetricsServiceName = "ssp-operator-metrics"
31+
MetricsServiceKey = "metrics.ssp.kubevirt.io"
2132
)
2233

34+
// Variable to store OLM deployment info (set from main)
35+
var isOLMDeployment bool
36+
37+
func SetOLMDeployment(isOLM bool) {
38+
isOLMDeployment = isOLM
39+
}
40+
2341
func newMonitoringClusterRole() *rbac.ClusterRole {
2442
return &rbac.ClusterRole{
2543
ObjectMeta: metav1.ObjectMeta{
@@ -61,31 +79,122 @@ func ServiceMonitorLabels() map[string]string {
6179
}
6280
}
6381

64-
func newServiceMonitorCR(namespace string) *promv1.ServiceMonitor {
65-
return &promv1.ServiceMonitor{
82+
func extractHostnameFromCert(certPath string) (string, error) {
83+
certBytes, err := os.ReadFile(certPath)
84+
if err != nil {
85+
return "", fmt.Errorf("failed to read certificate file: %w", err)
86+
}
87+
88+
block, _ := pem.Decode(certBytes)
89+
if block == nil {
90+
return "", fmt.Errorf("failed to parse certificate PEM")
91+
}
92+
93+
cert, err := x509.ParseCertificate(block.Bytes)
94+
if err != nil {
95+
return "", fmt.Errorf("failed to parse certificate: %w", err)
96+
}
97+
98+
if cert.Subject.CommonName != "" {
99+
return cert.Subject.CommonName, nil
100+
}
101+
102+
if len(cert.DNSNames) > 0 {
103+
return cert.DNSNames[0], nil
104+
}
105+
106+
return "", fmt.Errorf("no hostname found in certificate")
107+
}
108+
109+
// getCAConfigForServiceMonitor returns the appropriate CA configuration
110+
func getCAConfigForServiceMonitor() *promv1.SecretOrConfigMap {
111+
if isOLMDeployment {
112+
// OLM deployment: use ssp-operator-service-cert secret with olmCAKey
113+
return &promv1.SecretOrConfigMap{
114+
Secret: &v1.SecretKeySelector{
115+
LocalObjectReference: v1.LocalObjectReference{
116+
Name: "ssp-operator-service-cert",
117+
},
118+
Key: "olmCAKey",
119+
},
120+
}
121+
}
122+
123+
// Service-CA deployment: use openshift-service-ca.crt configmap
124+
return &promv1.SecretOrConfigMap{
125+
ConfigMap: &v1.ConfigMapKeySelector{
126+
LocalObjectReference: v1.LocalObjectReference{
127+
Name: "openshift-service-ca.crt",
128+
},
129+
Key: "service-ca.crt",
130+
},
131+
}
132+
}
133+
134+
func newValidatorServiceMonitor(namespace string) *promv1.ServiceMonitor {
135+
tlsConfig := &promv1.TLSConfig{
136+
SafeTLSConfig: promv1.SafeTLSConfig{
137+
InsecureSkipVerify: ptr.To(false),
138+
CA: promv1.SecretOrConfigMap{
139+
ConfigMap: &v1.ConfigMapKeySelector{
140+
LocalObjectReference: v1.LocalObjectReference{
141+
Name: "openshift-service-ca.crt",
142+
},
143+
Key: "service-ca.crt",
144+
},
145+
},
146+
},
147+
}
148+
tlsConfig.ServerName = ptr.To("virt-template-validator.kubevirt.svc")
149+
150+
serviceMonitor := newServiceMonitor(TemplateValidatorMetricsServiceName, namespace, tlsConfig, metav1.LabelSelector{
151+
MatchLabels: map[string]string{
152+
MetricsServiceKey: TemplateValidatorMetricsServiceName,
153+
},
154+
})
155+
return &serviceMonitor
156+
}
157+
158+
func newSspServiceMonitor(namespace string) *promv1.ServiceMonitor {
159+
certPath := filepath.Join(DefaultCertsDirectory, CertFilename)
160+
hostname, _ := extractHostnameFromCert(certPath)
161+
162+
tlsConfig := &promv1.TLSConfig{
163+
SafeTLSConfig: promv1.SafeTLSConfig{
164+
InsecureSkipVerify: ptr.To(false),
165+
// Use appropriate CA based on deployment type
166+
CA: *getCAConfigForServiceMonitor(),
167+
},
168+
}
169+
170+
if hostname != "" {
171+
tlsConfig.ServerName = &hostname
172+
}
173+
174+
serviceMonitor := newServiceMonitor(rules.RuleName, namespace, tlsConfig, metav1.LabelSelector{})
175+
return &serviceMonitor
176+
}
177+
178+
func newServiceMonitor(name,
179+
namespace string,
180+
tlsConfig *promv1.TLSConfig,
181+
selector metav1.LabelSelector) promv1.ServiceMonitor {
182+
return promv1.ServiceMonitor{
66183
ObjectMeta: metav1.ObjectMeta{
67184
Namespace: namespace,
68-
Name: rules.RuleName,
185+
Name: name,
69186
Labels: ServiceMonitorLabels(),
70187
},
71188
Spec: promv1.ServiceMonitorSpec{
72189
NamespaceSelector: promv1.NamespaceSelector{
73190
Any: true,
74191
},
75-
Selector: metav1.LabelSelector{
76-
MatchLabels: map[string]string{
77-
PrometheusLabelKey: PrometheusLabelValue,
78-
},
79-
},
192+
Selector: selector,
80193
Endpoints: []promv1.Endpoint{
81194
{
82-
Port: MetricsPortName,
83-
Scheme: "https",
84-
TLSConfig: &promv1.TLSConfig{
85-
SafeTLSConfig: promv1.SafeTLSConfig{
86-
InsecureSkipVerify: ptr.To(true),
87-
},
88-
},
195+
Port: MetricsPortName,
196+
Scheme: "https",
197+
TLSConfig: tlsConfig,
89198
HonorLabels: true,
90199
},
91200
},

‎internal/operands/template-validator/resources.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -372,6 +372,7 @@ func newValidatingWebhook(serviceNamespace string) *admission.ValidatingWebhookC
372372
func PrometheusServiceLabels() map[string]string {
373373
return map[string]string{
374374
metrics.PrometheusLabelKey: metrics.PrometheusLabelValue,
375+
metrics.MetricsServiceKey: MetricsServiceName,
375376
}
376377
}
377378

‎main.go‎

Lines changed: 35 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,14 @@ import (
4141
"sigs.k8s.io/controller-runtime/pkg/client"
4242
"sigs.k8s.io/controller-runtime/pkg/healthz"
4343
"sigs.k8s.io/controller-runtime/pkg/log/zap"
44-
"sigs.k8s.io/controller-runtime/pkg/metrics"
44+
k8smetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
4545
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
4646
"sigs.k8s.io/controller-runtime/pkg/webhook"
4747

4848
ssp "kubevirt.io/ssp-operator/api/v1beta3"
4949
"kubevirt.io/ssp-operator/internal/common"
5050
"kubevirt.io/ssp-operator/internal/controllers"
51+
"kubevirt.io/ssp-operator/internal/operands/metrics"
5152
sspMetrics "kubevirt.io/ssp-operator/pkg/monitoring/metrics/ssp-operator"
5253
"kubevirt.io/ssp-operator/pkg/monitoring/rules"
5354
"kubevirt.io/ssp-operator/webhooks"
@@ -162,7 +163,7 @@ func (s *prometheusServer) NeedLeaderElection() bool {
162163

163164
func (s *prometheusServer) Start(ctx context.Context) error {
164165
setupLog.Info("Starting Prometheus metrics endpoint server with TLS")
165-
handler := promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{})
166+
handler := promhttp.HandlerFor(k8smetrics.Registry, promhttp.HandlerOpts{})
166167
mux := http.NewServeMux()
167168
mux.Handle("/metrics", handler)
168169

@@ -225,10 +226,15 @@ func main() {
225226

226227
ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
227228

228-
err := createCertificateSymlinks()
229-
if err != nil {
230-
setupLog.Error(err, "Error creating certificate symlinks")
231-
os.Exit(1)
229+
if deployedOnOLM() {
230+
err := createCertificateSymlinks()
231+
if err != nil {
232+
setupLog.Error(err, "Error creating certificate symlinks")
233+
os.Exit(1)
234+
}
235+
metrics.SetOLMDeployment(true)
236+
} else {
237+
metrics.SetOLMDeployment(false)
232238
}
233239

234240
ctx := ctrl.SetupSignalHandler()
@@ -311,33 +317,35 @@ func main() {
311317
}
312318
}
313319

314-
func createCertificateSymlinks() error {
320+
func deployedOnOLM() bool {
315321
olmDir, olmDirErr := os.Stat(olmTLSDir)
316322
_, sdkDirErr := os.Stat(sdkTLSDir)
317-
318-
// If certificates are generated by OLM, we should use OLM certificates mount path
319323
if olmDirErr == nil && olmDir.IsDir() && os.IsNotExist(sdkDirErr) {
320-
// For some reason, OLM maps the cert/key files to apiserver.crt/apiserver.key
321-
// instead of tls.crt/tls.key like the SDK expects. Creating symlinks to allow
322-
// the operator to find and use them.
323-
setupLog.Info("OLM cert directory found, copying cert files")
324+
setupLog.Info("OLM cert directory found")
325+
return true
326+
}
327+
setupLog.Info("OLM cert directory not found")
328+
return false
329+
}
324330

325-
err := os.MkdirAll(sdkTLSDir, 0755)
326-
if err != nil {
327-
return fmt.Errorf("failed to create %s: %w", sdkTLSCrt, err)
328-
}
331+
func createCertificateSymlinks() error {
332+
// For some reason, OLM maps the cert/key files to apiserver.crt/apiserver.key
333+
// instead of tls.crt/tls.key like the SDK expects. Creating symlinks to allow
334+
// the operator to find and use them.
329335

330-
err = os.Symlink(path.Join(olmTLSDir, olmTLSCrt), path.Join(sdkTLSDir, sdkTLSCrt))
331-
if err != nil {
332-
return err
333-
}
336+
err := os.MkdirAll(sdkTLSDir, 0755)
337+
if err != nil {
338+
return fmt.Errorf("failed to create %s: %w", sdkTLSCrt, err)
339+
}
334340

335-
err = os.Symlink(path.Join(olmTLSDir, olmTLSKey), path.Join(sdkTLSDir, sdkTLSKey))
336-
if err != nil {
337-
return err
338-
}
339-
} else {
340-
setupLog.Info("OLM cert directory not found, using default cert directory")
341+
err = os.Symlink(path.Join(olmTLSDir, olmTLSCrt), path.Join(sdkTLSDir, sdkTLSCrt))
342+
if err != nil {
343+
return err
344+
}
345+
346+
err = os.Symlink(path.Join(olmTLSDir, olmTLSKey), path.Join(sdkTLSDir, sdkTLSKey))
347+
if err != nil {
348+
return err
341349
}
342350

343351
return nil

0 commit comments

Comments
 (0)