diff --git a/test/extended/prometheus/collection_profiles.go b/test/extended/prometheus/collection_profiles.go index dbf74e9923a7..c1fa076239e9 100644 --- a/test/extended/prometheus/collection_profiles.go +++ b/test/extended/prometheus/collection_profiles.go @@ -18,17 +18,34 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes" ) +// These constants are defined in the Cluster Monitoring Operator and need to +// be kept in sync. const ( - projectName = "monitoring-collection-profiles" - + // collectionProfileFeatureLabel is the Kubernetes label identifying the + // collection profile associated to the monitoring resource (ServiceMonitor + // or PodMonitor) collectionProfileFeatureLabel = "monitoring.openshift.io/collection-profile" - collectionProfileFull = "full" - collectionProfileDefault = collectionProfileFull - collectionProfileMinimal = "minimal" - collectionProfileNone = "" + + // collectionProfileFull is the profile enabling the collection of all metrics. + collectionProfileFull = "full" + + // collectionProfileMinimal is the profile enabling the collection of + // metrics used for Telemetry, alerting and dashboards. + collectionProfileMinimal = "minimal" + + collectionProfileEmpty = "" + + // collectionProfileDefault is the default collection profile (currently: full). + collectionProfileDefault = collectionProfileFull +) + +const ( + projectName = "monitoring-collection-profiles" operatorName = "cluster-monitoring-operator" operatorNamespaceName = "openshift-monitoring" @@ -38,18 +55,19 @@ const ( pollInterval = 5 * time.Second ) -var ( - collectionProfilesSupportedList = []string{ - collectionProfileFull, - collectionProfileMinimal, - } -) - type runner struct { - kclient kubernetes.Interface - mclient *prometheusoperatorv1client.MonitoringV1Client - pclient prometheusv1.API + kclient kubernetes.Interface + mclient *prometheusoperatorv1client.MonitoringV1Client + pclient prometheusv1.API + + // originalOperatorConfiguration is the copy of the CMO configuration's + // configmap to be restored when the test suite finishes. originalOperatorConfiguration *v1.ConfigMap + + // collectionProfilesSupportedList is the list of all collection profiles + // supported by the Cluster Monitoring Operator. It is populated at runtime + // to account for new profiles being added over time. + collectionProfilesSupportedList []string } // NOTE: The nested `Context` containers inside the following `Describe` container are used to group certain tests based on the environments they demand. @@ -74,6 +92,7 @@ var _ = g.Describe("[sig-instrumentation][OCPFeatureGate:MetricsCollectionProfil } r.pclient = oc.NewPrometheusClient(tctx) + // Save the current configuration and enabled the default collection profile. var operatorConfiguration *v1.ConfigMap o.Eventually(func() error { operatorConfiguration, err = r.kclient.CoreV1().ConfigMaps(operatorNamespaceName).Get(tctx, operatorConfigurationName, metav1.GetOptions{}) @@ -81,7 +100,7 @@ var _ = g.Describe("[sig-instrumentation][OCPFeatureGate:MetricsCollectionProfil if errors.IsNotFound(err) { g.By("initially, creating a configuration for the operator as it did not exist") operatorConfiguration = nil - return r.makeCollectionProfileConfigurationFor(tctx, collectionProfileDefault) + return r.configureCollectionProfile(tctx, collectionProfileDefault) } return err @@ -90,33 +109,47 @@ var _ = g.Describe("[sig-instrumentation][OCPFeatureGate:MetricsCollectionProfil return nil }, pollTimeout, pollInterval).Should(o.BeNil()) r.originalOperatorConfiguration = operatorConfiguration + + // Discover all supported collection profiles. + var supportedProfiles []string + o.Eventually(func() error { + var err error + supportedProfiles, err = r.getSupportedCollectionProfiles(tctx) + return err + }, pollTimeout, pollInterval).Should(o.BeNil()) + g.GinkgoWriter.Printf("supported collection profiles: %v\n", supportedProfiles) + r.collectionProfilesSupportedList = supportedProfiles }) + // Restore the Cluster Monitoring Operator's configuration. g.AfterAll(func() { - shouldDeleteConfiguration := false currentConfiguration, err := r.kclient.CoreV1().ConfigMaps(operatorNamespaceName).Get(tctx, operatorConfigurationName, metav1.GetOptions{}) o.Expect(err).To(o.BeNil()) + if r.originalOperatorConfiguration != nil { currentConfiguration.Data = r.originalOperatorConfiguration.Data g.By("restoring the original configuration for the operator") _, err = r.kclient.CoreV1().ConfigMaps(operatorNamespaceName).Update(tctx, currentConfiguration, metav1.UpdateOptions{}) } else { - shouldDeleteConfiguration = true - g.By("cleaning up the configuration for the operator as it did not exist pre-job") + g.By("deleting the cluster monitoring operator's configuration since it did not exist pre-job") err = r.kclient.CoreV1().ConfigMaps(operatorNamespaceName).Delete(tctx, operatorConfigurationName, metav1.DeleteOptions{}) } o.Expect(err).To(o.BeNil()) o.Eventually(func() error { - if shouldDeleteConfiguration { - _, err := r.kclient.CoreV1().ConfigMaps(operatorNamespaceName).Get(tctx, operatorConfigurationName, metav1.GetOptions{}) + if r.originalOperatorConfiguration != nil { + return nil + } + + _, err := r.kclient.CoreV1().ConfigMaps(operatorNamespaceName).Get(tctx, operatorConfigurationName, metav1.GetOptions{}) + if err != nil { if errors.IsNotFound(err) { return nil } - return fmt.Errorf("ConfigMap %q still exists after deletion attempt", operatorConfigurationName) + return err } - return nil + return fmt.Errorf("ConfigMap %q still exists after deletion attempt", operatorConfigurationName) }, pollTimeout, pollInterval).Should(o.BeNil()) }) @@ -124,31 +157,23 @@ var _ = g.Describe("[sig-instrumentation][OCPFeatureGate:MetricsCollectionProfil profile := collectionProfileDefault g.BeforeAll(func() { - err := r.makeCollectionProfileConfigurationFor(tctx, profile) + err := r.configureCollectionProfile(tctx, profile) o.Expect(err).To(o.BeNil()) o.Eventually(func() error { - enabled, err := r.isProfileEnabled(tctx, profile) - if err != nil { - return err - } - if !enabled { - return fmt.Errorf("collection profile %q is not enabled", profile) - } - - return nil + return r.assertCollectionProfileEnabled(tctx, profile) }, pollTimeout, pollInterval).Should(o.BeNil()) }) - g.It("should expose default metrics", func() { + g.It("should expose all metrics", func() { o.Eventually(func() error { - defaultOnlyMetric := "prometheus_engine_query_log_enabled" - defaultMetricQuery := fmt.Sprintf("max(%s)", defaultOnlyMetric) - queryResponse, err := helper.RunQuery(tctx, r.pclient, defaultMetricQuery) + const sentinelMetricForDefaultProfile = "prometheus_engine_query_log_enabled" + queryResponse, err := helper.RunQuery(tctx, r.pclient, fmt.Sprintf("max(%s)", sentinelMetricForDefaultProfile)) if err != nil { return err } + if len(queryResponse.Data.Result) == 0 { - return fmt.Errorf("expected %q to be present", defaultOnlyMetric) + return fmt.Errorf("expected %q to be present", sentinelMetricForDefaultProfile) } return nil @@ -158,8 +183,9 @@ var _ = g.Describe("[sig-instrumentation][OCPFeatureGate:MetricsCollectionProfil g.Context("in a heterogeneous environment,", func() { g.It("should expose information about the applied collection profile using meta-metrics", func() { - for _, profile := range collectionProfilesSupportedList { - err := r.makeCollectionProfileConfigurationFor(tctx, profile) + for _, profile := range r.collectionProfilesSupportedList { + g.GinkgoWriter.Printf("enabling collection profile: %s\n", profile) + err := r.configureCollectionProfile(tctx, profile) o.Expect(err).To(o.BeNil()) o.Eventually(func() error { @@ -168,6 +194,7 @@ var _ = g.Describe("[sig-instrumentation][OCPFeatureGate:MetricsCollectionProfil if err != nil { return err } + if len(queryResponse.Data.Result) == 0 { return fmt.Errorf("no result found for profile %q", profile) } @@ -176,58 +203,69 @@ var _ = g.Describe("[sig-instrumentation][OCPFeatureGate:MetricsCollectionProfil }, pollTimeout, pollInterval).Should(o.BeNil()) } }) - g.It("should have at least one implementation for each collection profile", func() { - for _, profile := range collectionProfilesSupportedList { - err := r.makeCollectionProfileConfigurationFor(tctx, profile) - o.Expect(err).To(o.BeNil()) - o.Eventually(func() error { - monitors, err := r.fetchMonitorsFor(tctx, [2]string{collectionProfileFeatureLabel, profile}) - if err != nil { - return err - } - if len(monitors.Items) == 0 { - return fmt.Errorf("no monitors found with collection profile %q", profile) + g.It("should implement all collection profiles or none", func() { + // Retrieve all service monitors implementing the default collection profile. + var monitors []*prometheusoperatorv1.ServiceMonitor + o.Eventually(func() error { + serviceMonitors, err := r.getServiceMonitors(tctx, metav1.NamespaceAll, label{key: collectionProfileFeatureLabel, value: collectionProfileDefault}) + if err != nil { + return err + } + monitors = serviceMonitors.Items + return nil + }, pollTimeout, pollInterval).Should(o.BeNil()) + + // For each service monitor implementing the default collection + // profile, ensure that all other collection profiles are also + // implemented. + for _, monitor := range monitors { + g.GinkgoWriter.Printf("checking ServiceMonitor %s/%s\n", monitor.Namespace, monitor.Name) + for _, profile := range r.collectionProfilesSupportedList { + if profile == collectionProfileDefault { + continue } - return nil - }, pollTimeout, pollInterval).Should(o.BeNil()) + o.Eventually(func() error { + selectors := []label{{key: collectionProfileFeatureLabel, value: profile}} + for k, v := range monitor.Labels { + if k == collectionProfileFeatureLabel { + continue + } + selectors = append(selectors, label{key: k, value: v}) + } + + monitors, err := r.getServiceMonitors(tctx, monitor.Namespace, selectors...) + if err != nil { + return err + } + + if len(monitors.Items) == 0 { + return fmt.Errorf("%s/%s: no ServiceMonitor found for collection profile %q", monitor.Namespace, monitor.Name, profile) + } + + return nil + }, time.Minute, pollInterval).Should(o.BeNil()) + } } }) + g.It("should revert to default collection profile when an empty collection profile value is specified", func() { - err := r.makeCollectionProfileConfigurationFor(tctx, collectionProfileNone) + err := r.configureCollectionProfile(tctx, collectionProfileEmpty) o.Expect(err).To(o.BeNil()) o.Eventually(func() error { - enabled, err := r.isProfileEnabled(tctx, collectionProfileFull) - if err != nil { - return err - } - if !enabled { - return fmt.Errorf("collection profile %q is not enabled", collectionProfileFull) - } - - return nil + return r.assertCollectionProfileEnabled(tctx, collectionProfileFull) }, pollTimeout, pollInterval).Should(o.BeNil()) }) }) g.Context("in a homogeneous minimal environment,", func() { - profile := collectionProfileMinimal - g.BeforeAll(func() { - err := r.makeCollectionProfileConfigurationFor(tctx, profile) + err := r.configureCollectionProfile(tctx, collectionProfileMinimal) o.Expect(err).To(o.BeNil()) o.Eventually(func() error { - enabled, err := r.isProfileEnabled(tctx, profile) - if err != nil { - return err - } - if !enabled { - return fmt.Errorf("collection profile %q is not enabled", profile) - } - - return nil + return r.assertCollectionProfileEnabled(tctx, collectionProfileMinimal) }, pollTimeout, pollInterval).Should(o.BeNil()) }) @@ -237,15 +275,17 @@ var _ = g.Describe("[sig-instrumentation][OCPFeatureGate:MetricsCollectionProfil var kubeStateMetricsMonitor *prometheusoperatorv1.ServiceMonitor o.Eventually(func() error { - monitors, err := r.fetchMonitorsFor(tctx, [2]string{collectionProfileFeatureLabel, profile}, [2]string{appNameSelector, appName}) + monitors, err := r.getServiceMonitorsForOpenShiftMonitoring(tctx, label{key: collectionProfileFeatureLabel, value: collectionProfileMinimal}, label{key: appNameSelector, value: appName}) if err != nil { return err } + if len(monitors.Items) == 0 { - return fmt.Errorf("no monitors found with collection profile: %q and %#v=%q", profile, appNameSelector, appName) + return fmt.Errorf("no ServiceMonitor found with collection profile: %q and %#v=%q", collectionProfileMinimal, appNameSelector, appName) } + if len(monitors.Items) > 1 { - return fmt.Errorf("more than one monitor found with collection profile: %q and %#v=%q", profile, appNameSelector, appName) + return fmt.Errorf("more than one ServiceMonitor found with collection profile: %q and %#v=%q", collectionProfileMinimal, appNameSelector, appName) } kubeStateMetricsMonitor = monitors.Items[0] @@ -312,86 +352,107 @@ var _ = g.Describe("[sig-instrumentation][OCPFeatureGate:MetricsCollectionProfil }) }) -func (r runner) isProfileEnabled(ctx context.Context, profile string) (bool, error) { +func (r runner) assertCollectionProfileEnabled(ctx context.Context, profile string) error { vectorExpression := "max(profile:cluster_monitoring_operator_collection_profile:max{profile=\"%s\"}) == 1" queryResponse, err := helper.RunQuery(ctx, r.pclient, fmt.Sprintf(vectorExpression, profile)) if err != nil { - return false, err + return err } if len(queryResponse.Data.Result) == 0 { - return false, nil + return fmt.Errorf("collection profile %q is not enabled", profile) } - return true, nil + return nil } -func (r runner) fetchMonitorsFor(ctx context.Context, selectors ...[2]string) (*prometheusoperatorv1.ServiceMonitorList, error) { - managedMonitorsSelectors := []string{ - fmt.Sprintf("%s=%s", "app.kubernetes.io/managed-by", operatorName), - } +type label struct { + key string + value string +} + +// getServiceMonitorsForOpenShiftMonitoring returns all service monitors managed by the Cluster Monitoring Operator. +func (r runner) getServiceMonitorsForOpenShiftMonitoring(ctx context.Context, selectors ...label) (*prometheusoperatorv1.ServiceMonitorList, error) { + return r.getServiceMonitors(ctx, operatorNamespaceName, append([]label{{key: "app.kubernetes.io/managed-by", value: operatorName}}, selectors...)...) +} + +// getServiceMonitors returns service monitors in the given namespace (or all namespaces if empty) matching the given label selectors. +func (r runner) getServiceMonitors(ctx context.Context, namespace string, selectors ...label) (*prometheusoperatorv1.ServiceMonitorList, error) { + var labelSelectors []string for _, selector := range selectors { - managedMonitorsSelectors = append(managedMonitorsSelectors, fmt.Sprintf("%s=%s", selector[0], selector[1])) + labelSelectors = append(labelSelectors, fmt.Sprintf("%s=%s", selector.key, selector.value)) } return r.mclient.ServiceMonitors(operatorNamespaceName).List(ctx, metav1.ListOptions{ - LabelSelector: strings.Join(managedMonitorsSelectors, ","), + LabelSelector: strings.Join(labelSelectors, ","), }) } -func (r runner) makeCollectionProfileConfigurationFor(ctx context.Context, collectionProfile string) error { - dataConfigYAMLPrometheusK8s := fmt.Sprintf("collectionProfile: %s", collectionProfile) - dataConfigYAMLPrometheusK8sStructured := map[string]interface{}{ - "collectionProfile": collectionProfile, +// getSupportedCollectionProfiles returns the list of supported collection +// profiles interpolating from the monitor resources installed by the Cluster +// Monitoring Operator. +func (r runner) getSupportedCollectionProfiles(ctx context.Context) ([]string, error) { + monitors, err := r.getServiceMonitorsForOpenShiftMonitoring(ctx) + if err != nil { + return nil, err + } + + seen := sets.New[string]() + for _, monitor := range monitors.Items { + if profile, ok := monitor.Labels[collectionProfileFeatureLabel]; ok && profile != collectionProfileEmpty { + seen.Insert(profile) + } } - dataConfigYAML := fmt.Sprintf("prometheusK8s:\n %s", dataConfigYAMLPrometheusK8s) - configurationEnableCollectionProfiles := &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: operatorConfigurationName, - Namespace: operatorNamespaceName, - }, - Data: map[string]string{ - "config.yaml": dataConfigYAML, - }, + + profiles := sets.List(seen) + if len(profiles) < 2 { + return nil, fmt.Errorf("expected at least 2 supported collection profiles, got %d: %v", len(profiles), profiles) } + return profiles, nil +} + +// configureCollectionProfile udpates the Cluster Monitoring +// Operator's configuration to enable a given collection profile. +func (r runner) configureCollectionProfile(ctx context.Context, collectionProfile string) error { configuration, err := r.kclient.CoreV1().ConfigMaps(operatorNamespaceName).Get(ctx, operatorConfigurationName, metav1.GetOptions{}) - if err != nil && errors.IsNotFound(err) { - _, err = r.kclient.CoreV1().ConfigMaps(operatorNamespaceName).Create(ctx, configurationEnableCollectionProfiles, metav1.CreateOptions{}) - if err != nil { - return err - } - } else { - gotDataConfigYAML, ok := configuration.Data["config.yaml"] - if !ok { - configuration.Data = make(map[string]string) - configuration.Data["config.yaml"] = dataConfigYAML - } else { - var gotDataConfigYAMLMap map[string]interface{} - err = yaml.Unmarshal([]byte(gotDataConfigYAML), &gotDataConfigYAMLMap) - if err != nil { - return err - } - if _, ok := gotDataConfigYAMLMap["prometheusK8s"]; !ok { - gotDataConfigYAMLMap["prometheusK8s"] = dataConfigYAMLPrometheusK8sStructured - } else { - gotDataConfigYAMLMap["prometheusK8s"].(map[string]interface{})["collectionProfile"] = collectionProfile - } - gotDataConfigYAMLRaw, err := yaml.Marshal(gotDataConfigYAMLMap) - if err != nil { - return err - } - gotDataConfigYAML = string(gotDataConfigYAMLRaw) - configuration.Data["config.yaml"] = gotDataConfigYAML - } - currentConfiguration, err := r.kclient.CoreV1().ConfigMaps(operatorNamespaceName).Get(ctx, operatorConfigurationName, metav1.GetOptions{}) - if err != nil { - return err + create := errors.IsNotFound(err) + if err != nil && !create { + return err + } + + if create { + configuration = &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: operatorConfigurationName, + Namespace: operatorNamespaceName, + }, + Data: map[string]string{}, } - currentConfiguration.Data = configuration.Data - _, err = r.kclient.CoreV1().ConfigMaps(operatorNamespaceName).Update(ctx, currentConfiguration, metav1.UpdateOptions{}) - if err != nil { + } + + var configMap map[string]interface{} + if raw, ok := configuration.Data["config.yaml"]; ok { + if err := yaml.Unmarshal([]byte(raw), &configMap); err != nil { return err } } + if configMap == nil { + configMap = make(map[string]interface{}) + } - return nil + if err := unstructured.SetNestedField(configMap, collectionProfile, "prometheusK8s", "collectionProfile"); err != nil { + return err + } + + raw, err := yaml.Marshal(configMap) + if err != nil { + return err + } + configuration.Data["config.yaml"] = string(raw) + + if create { + _, err = r.kclient.CoreV1().ConfigMaps(operatorNamespaceName).Create(ctx, configuration, metav1.CreateOptions{}) + } else { + _, err = r.kclient.CoreV1().ConfigMaps(operatorNamespaceName).Update(ctx, configuration, metav1.UpdateOptions{}) + } + return err }