diff --git a/.gitignore b/.gitignore index 5b455fe03d..a8bdf3ad06 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ gha-creds-*.json # The directory Mix will write compiled artifacts to. /_build/ +/tmp/ # If you run "mix test --cover", coverage assets end up here. /cover/ diff --git a/go/client/models_gen.go b/go/client/models_gen.go index 886555bf34..0ec5525df6 100644 --- a/go/client/models_gen.go +++ b/go/client/models_gen.go @@ -2345,8 +2345,10 @@ type ClusterInsightComponent struct { Namespace *string `json:"namespace,omitempty"` Name string `json:"name"` Priority *InsightComponentPriority `json:"priority,omitempty"` - Cluster *Cluster `json:"cluster,omitempty"` - Insight *AiInsight `json:"insight,omitempty"` + // the plural service managing this component, if any + Service *ServiceDeployment `json:"service,omitempty"` + Cluster *Cluster `json:"cluster,omitempty"` + Insight *AiInsight `json:"insight,omitempty"` // the raw kubernetes resource itself, this is an expensive fetch and should be used sparingly Resource *KubernetesUnstructured `json:"resource,omitempty"` } @@ -2358,6 +2360,8 @@ type ClusterInsightComponentAttributes struct { Namespace *string `json:"namespace,omitempty"` Name string `json:"name"` Priority *InsightComponentPriority `json:"priority,omitempty"` + // the plural service that manages this component, if any + ServiceID *string `json:"serviceId,omitempty"` } // A reference to a built ISO image to be used for flashing new edge clusters diff --git a/go/deployment-operator/Makefile b/go/deployment-operator/Makefile index ccafa8505a..c8285757d3 100644 --- a/go/deployment-operator/Makefile +++ b/go/deployment-operator/Makefile @@ -59,9 +59,16 @@ genmock: TOOL = mockery genmock: --tool ## generates mocks before running tests $(MOCKERY) +# The chart lives in the pluralsh/deployment-operator repository, point this at the crds directory of a checkout of it. +CHART_CRDS_DIR ?= ../../charts/deployment-operator/crds + .PHONY: codegen-chart-crds -codegen-chart-crds: ## copy CRDs to the helm chart - @cp -a ./config/crd/bases/. ../../charts/deployment-operator/crds +codegen-chart-crds: ## copy CRDs to the helm chart, set CHART_CRDS_DIR to the crds directory of the chart + @if [ -d "$(CHART_CRDS_DIR)" ]; then \ + cp -a ./config/crd/bases/. "$(CHART_CRDS_DIR)"; \ + else \ + echo "skipping chart CRDs, $(CHART_CRDS_DIR) does not exist, set CHART_CRDS_DIR to the crds directory of the chart"; \ + fi .PHONY: velero-crds velero-crds: ## download velero CRDs diff --git a/go/deployment-operator/api/v1alpha1/custominsightpriority_types.go b/go/deployment-operator/api/v1alpha1/custominsightpriority_types.go new file mode 100644 index 0000000000..416a5534bb --- /dev/null +++ b/go/deployment-operator/api/v1alpha1/custominsightpriority_types.go @@ -0,0 +1,174 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // OverlappingConditionType is set on a CustomInsightPriority that lost some of its objects to a CustomInsightPriority with a higher precedence. + OverlappingConditionType ConditionType = "Overlapping" + + // InvalidScriptConditionReason means the script could not be compiled, so it is not evaluated. + InvalidScriptConditionReason ConditionReason = "InvalidScript" + + // EvaluationFailedConditionReason means the script failed on the last evaluation, so its objects use the built-in priority. + EvaluationFailedConditionReason ConditionReason = "EvaluationFailed" + + // OverlapConditionReason is the reason of the Overlapping condition. + OverlapConditionReason ConditionReason = "Overlap" +) + +// InsightEngine is the language an insight priority script is written in. +// +kubebuilder:validation:Enum=python +type InsightEngine string + +const ( + // InsightEnginePython evaluates the script as sandboxed Python. + InsightEnginePython InsightEngine = "python" +) + +// InsightTargetKind selects objects by kind. +type InsightTargetKind struct { + // Group of the kind, empty for the core group. + // +kubebuilder:validation:Optional + Group string `json:"group,omitempty"` + + // Version of the kind, any version matches when empty. + // +kubebuilder:validation:Optional + Version string `json:"version,omitempty"` + + // Kind of the objects, for example StatefulSet. + // +kubebuilder:validation:MinLength=1 + Kind string `json:"kind"` +} + +// InsightTarget selects the failing objects a script is responsible for. An object has to match every +// non-empty field, so an empty target matches everything. +type InsightTarget struct { + // Kinds the script is responsible for. + // +kubebuilder:validation:Optional + Kinds []InsightTargetKind `json:"kinds,omitempty"` + + // Namespaces the script is responsible for. Cluster scoped objects only match when this is empty. + // +kubebuilder:validation:Optional + Namespaces []string `json:"namespaces,omitempty"` + + // LabelSelector selects objects by their labels. + // +kubebuilder:validation:Optional + LabelSelector *metav1.LabelSelector `json:"labelSelector,omitempty"` +} + +// CustomInsightPrioritySpec defines how the insight priority of failing objects is decided. +type CustomInsightPrioritySpec struct { + // Target selects the failing objects this script sets the priority of. + // +kubebuilder:validation:Optional + Target InsightTarget `json:"target,omitempty"` + + // Precedence decides which script owns an object that several targets match: the highest wins and ties are + // broken by name. A script never sees objects it does not own, and scripts are not chained. + // +kubebuilder:validation:Optional + Precedence int `json:"precedence,omitempty"` + + // Engine the script is written for. + // +kubebuilder:default=python + // +kubebuilder:validation:Optional + Engine InsightEngine `json:"engine,omitempty"` + + // Script defines `evaluate(input)`, called with the objects this script owns in `input["rows"]`, and returns a + // dictionary from the `uid` of a row to `{"priority": "LOW|MEDIUM|HIGH|CRITICAL|IGNORE", "reason": "..."}`. + // Rows that are left out keep their built-in priority, IGNORE removes a row from the insights and the cluster health score. + // + // Objects annotated with deployments.plural.sh/insight-priority are decided by the annotation and never reach the script. + // + // See https://docs.plural.sh/plural-features/continuous-deployment/deployment-operator/insight-priority for the full guide. + Script string `json:"script"` +} + +// InsightEvaluation is the outcome of the last time the script ran. +type InsightEvaluation struct { + // Time the script last ran with a different outcome than the one before. + Time metav1.Time `json:"time,omitempty"` + + // Matched is the number of failing objects the script owned. + Matched int `json:"matched"` + + // Overridden is the number of objects the script gave a priority. + Overridden int `json:"overridden"` + + // Ignored is the number of objects the script ignored. + Ignored int `json:"ignored"` + + // DurationMs is how long the script took. + DurationMs int64 `json:"durationMs"` +} + +// CustomInsightPriorityStatus defines the observed state of CustomInsightPriority +type CustomInsightPriorityStatus struct { + // LastEvaluation is the outcome of the last evaluation, empty until the script ran. + // +kubebuilder:validation:Optional + LastEvaluation *InsightEvaluation `json:"lastEvaluation,omitempty"` + + // LastError is the error of the last evaluation, empty when it succeeded. + // +kubebuilder:validation:Optional + LastError string `json:"lastError,omitempty"` + + // Represents the observations of a CustomInsightPriority's current state. + // +patchMergeKey=type + // +patchStrategy=merge + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status +//+kubebuilder:printcolumn:name="Ready",type="string",JSONPath=".status.conditions[?(@.type==\"Ready\")].status" +//+kubebuilder:printcolumn:name="Precedence",type="integer",JSONPath=".spec.precedence" +//+kubebuilder:printcolumn:name="Matched",type="integer",JSONPath=".status.lastEvaluation.matched" + +// CustomInsightPriority sets the priority of failing objects in the cluster insights and the cluster health score. +type CustomInsightPriority struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CustomInsightPrioritySpec `json:"spec,omitempty"` + Status CustomInsightPriorityStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// CustomInsightPriorityList contains a list of CustomInsightPriority +type CustomInsightPriorityList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CustomInsightPriority `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CustomInsightPriority{}, &CustomInsightPriorityList{}) +} + +func (c *CustomInsightPriority) SetCondition(condition metav1.Condition) { + meta.SetStatusCondition(&c.Status.Conditions, condition) +} + +func (c *CustomInsightPriority) RemoveCondition(conditionType ConditionType) { + meta.RemoveStatusCondition(&c.Status.Conditions, conditionType.String()) +} diff --git a/go/deployment-operator/api/v1alpha1/zz_generated.deepcopy.go b/go/deployment-operator/api/v1alpha1/zz_generated.deepcopy.go index 742c544d6b..f2615f28aa 100644 --- a/go/deployment-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/go/deployment-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -1225,6 +1225,108 @@ func (in *CustomHealthStatus) DeepCopy() *CustomHealthStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CustomInsightPriority) DeepCopyInto(out *CustomInsightPriority) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomInsightPriority. +func (in *CustomInsightPriority) DeepCopy() *CustomInsightPriority { + if in == nil { + return nil + } + out := new(CustomInsightPriority) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CustomInsightPriority) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CustomInsightPriorityList) DeepCopyInto(out *CustomInsightPriorityList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CustomInsightPriority, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomInsightPriorityList. +func (in *CustomInsightPriorityList) DeepCopy() *CustomInsightPriorityList { + if in == nil { + return nil + } + out := new(CustomInsightPriorityList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CustomInsightPriorityList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CustomInsightPrioritySpec) DeepCopyInto(out *CustomInsightPrioritySpec) { + *out = *in + in.Target.DeepCopyInto(&out.Target) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomInsightPrioritySpec. +func (in *CustomInsightPrioritySpec) DeepCopy() *CustomInsightPrioritySpec { + if in == nil { + return nil + } + out := new(CustomInsightPrioritySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CustomInsightPriorityStatus) DeepCopyInto(out *CustomInsightPriorityStatus) { + *out = *in + if in.LastEvaluation != nil { + in, out := &in.LastEvaluation, &out.LastEvaluation + *out = new(InsightEvaluation) + (*in).DeepCopyInto(*out) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomInsightPriorityStatus. +func (in *CustomInsightPriorityStatus) DeepCopy() *CustomInsightPriorityStatus { + if in == nil { + return nil + } + out := new(CustomInsightPriorityStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ExaConnection) DeepCopyInto(out *ExaConnection) { *out = *in @@ -1703,6 +1805,67 @@ func (in *IngressReplicaSpec) DeepCopy() *IngressReplicaSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InsightEvaluation) DeepCopyInto(out *InsightEvaluation) { + *out = *in + in.Time.DeepCopyInto(&out.Time) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InsightEvaluation. +func (in *InsightEvaluation) DeepCopy() *InsightEvaluation { + if in == nil { + return nil + } + out := new(InsightEvaluation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InsightTarget) DeepCopyInto(out *InsightTarget) { + *out = *in + if in.Kinds != nil { + in, out := &in.Kinds, &out.Kinds + *out = make([]InsightTargetKind, len(*in)) + copy(*out, *in) + } + if in.Namespaces != nil { + in, out := &in.Namespaces, &out.Namespaces + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.LabelSelector != nil { + in, out := &in.LabelSelector, &out.LabelSelector + *out = new(metav1.LabelSelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InsightTarget. +func (in *InsightTarget) DeepCopy() *InsightTarget { + if in == nil { + return nil + } + out := new(InsightTarget) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InsightTargetKind) DeepCopyInto(out *InsightTargetKind) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InsightTargetKind. +func (in *InsightTargetKind) DeepCopy() *InsightTargetKind { + if in == nil { + return nil + } + out := new(InsightTargetKind) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubecostExtractor) DeepCopyInto(out *KubecostExtractor) { *out = *in diff --git a/go/deployment-operator/cmd/agent/args/args.go b/go/deployment-operator/cmd/agent/args/args.go index 62af9d464e..49ed62bbb7 100644 --- a/go/deployment-operator/cmd/agent/args/args.go +++ b/go/deployment-operator/cmd/agent/args/args.go @@ -94,6 +94,9 @@ const ( defaultClusterPingInterval = "2m" defaultClusterPingIntervalDuration = 2 * time.Minute + defaultInsightGracePeriod = "10m" + defaultInsightGracePeriodDuration = 10 * time.Minute + defaultRuntimeServicePingInterval = "3m" defaultRuntimeServicePingIntervalDuration = 3 * time.Minute @@ -174,6 +177,7 @@ var ( argWorkqueueQPS = flag.Int("workqueue-qps", 10, "The maximum number of items to process per second.") argWorkqueueBurst = flag.Int("workqueue-burst", 50, "The maximum number of items to process at a time.") argClusterPingInterval = flag.String("cluster-ping-interval", defaultClusterPingInterval, "Time interval to ping cluster.") + argInsightGracePeriod = flag.String("insight-grace-period", defaultInsightGracePeriod, "How long a component that stopped failing is still reported as a cluster insight, to keep flapping components from regenerating their insight. Set to 0 to disable.") argRuntimeServicePingInterval = flag.String("runtime-service-ping-interval", defaultRuntimeServicePingInterval, "Time interval to register runtime services.") argStackPollInterval = flag.String("stack-poll-interval", defaultStackPollInterval, "Time interval to poll stack resources from the Console API. Use '0s' to disable.") argSentinelPollInterval = flag.String("sentinel-poll-interval", defaultSentinelPollInterval, "Time interval to poll sentinel run jobs from the Console API. Use '0s' to disable.") @@ -511,6 +515,16 @@ func ClusterPingInterval() time.Duration { return duration } +func InsightGracePeriod() time.Duration { + duration, err := time.ParseDuration(*argInsightGracePeriod) + if err != nil || duration < 0 { + klog.ErrorS(err, "Could not parse insight-grace-period", "value", *argInsightGracePeriod, "default", defaultInsightGracePeriod) + return defaultInsightGracePeriodDuration + } + + return duration +} + func RuntimeServicesPingInterval() time.Duration { duration, err := time.ParseDuration(*argRuntimeServicePingInterval) if err != nil { diff --git a/go/deployment-operator/cmd/agent/kubernetes.go b/go/deployment-operator/cmd/agent/kubernetes.go index c28a7fbc63..d0812406af 100644 --- a/go/deployment-operator/cmd/agent/kubernetes.go +++ b/go/deployment-operator/cmd/agent/kubernetes.go @@ -31,6 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "github.com/pluralsh/console/go/deployment-operator/api/v1alpha1" "github.com/pluralsh/console/go/deployment-operator/cmd/agent/args" "github.com/pluralsh/console/go/deployment-operator/internal/controller" "github.com/pluralsh/console/go/deployment-operator/internal/utils" @@ -39,6 +40,7 @@ import ( consoleclient "github.com/pluralsh/console/go/deployment-operator/pkg/client" consolectrl "github.com/pluralsh/console/go/deployment-operator/pkg/controller" "github.com/pluralsh/console/go/deployment-operator/pkg/controller/service" + "github.com/pluralsh/console/go/deployment-operator/pkg/insightpriority" ) func emptyDiskHealthCheck(_ *http.Request) error { @@ -138,6 +140,7 @@ func registerKubeReconcilersOrDie( enableKubecostProxy bool, consoleURL, deployToken string, userGroupCache cache.UserGroupCache, + insightRegistry *insightpriority.Registry, ) { rolloutsClient, dynamicClient, kubeClient := initKubeClientsOrDie(config) @@ -240,6 +243,8 @@ func registerKubeReconcilersOrDie( }).SetupWithManager(manager); err != nil { setupLog.Error(err, "unable to create controller", "controller", "HealthConvert") } + registerCustomInsightPriority(manager, insightRegistry) + if err := (&controller.StackRunJobReconciler{ Client: manager.GetClient(), Scheme: manager.GetScheme(), @@ -368,3 +373,27 @@ func registerKubeReconcilersOrDie( setupLog.Error(err, "unable to create controller", "controller", "SentinelRunJob") } } + +// registerCustomInsightPriority registers the controller of CustomInsightPriority scripts when their CRD is installed. +// Helm does not upgrade the CRDs of a chart, so an agent that is upgraded without them has to start without the feature, +// a controller watching a missing CRD would fail the whole manager. +func registerCustomInsightPriority(manager ctrl.Manager, registry *insightpriority.Registry) { + gk := schema.GroupKind{Group: v1alpha1.GroupVersion.Group, Kind: "CustomInsightPriority"} + if _, err := manager.GetRESTMapper().RESTMapping(gk, v1alpha1.GroupVersion.Version); err != nil { + if meta.IsNoMatchError(err) { + setupLog.Info("CustomInsightPriority CRD is not installed, insight priorities can not be customized", "controller", "CustomInsightPriority") + } else { + setupLog.Error(err, "unable to check for the CustomInsightPriority CRD", "controller", "CustomInsightPriority") + } + return + } + + if err := (&controller.CustomInsightPriorityReconciler{ + Client: manager.GetClient(), + Scheme: manager.GetScheme(), + Registry: registry, + Recorder: manager.GetEventRecorder("custominsightpriority-controller"), + }).SetupWithManager(manager); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "CustomInsightPriority") + } +} diff --git a/go/deployment-operator/cmd/agent/main.go b/go/deployment-operator/cmd/agent/main.go index e5e5283384..c9eb938617 100644 --- a/go/deployment-operator/cmd/agent/main.go +++ b/go/deployment-operator/cmd/agent/main.go @@ -47,6 +47,7 @@ import ( consolectrl "github.com/pluralsh/console/go/deployment-operator/pkg/controller" "github.com/pluralsh/console/go/deployment-operator/pkg/controller/namespaces" "github.com/pluralsh/console/go/deployment-operator/pkg/controller/service" + "github.com/pluralsh/console/go/deployment-operator/pkg/insightpriority" "github.com/pluralsh/console/go/deployment-operator/pkg/ping" pythonruntime "github.com/pluralsh/console/go/deployment-operator/pkg/python" "github.com/pluralsh/console/go/deployment-operator/pkg/scraper" @@ -180,7 +181,8 @@ func main() { userGroupCache := cache.NewUserGroupCache(extConsoleClient) registerConsoleReconcilersOrDie(consoleManager, mapper, clientSet, kubeManager.GetClient(), dynamicClient, dbStore, kubeManager.GetScheme(), extConsoleClient, supervisor, discoveryCache, namespaceCache, svcCache, cacheStore.Dir()) - registerKubeReconcilersOrDie(ctx, clientSet, kubeManager, consoleManager, config, extConsoleClient, discoveryCache, args.EnableKubecostProxy(), args.ConsoleUrl(), args.DeployToken(), userGroupCache) + insightRegistry := insightpriority.NewRegistry() + registerKubeReconcilersOrDie(ctx, clientSet, kubeManager, consoleManager, config, extConsoleClient, discoveryCache, args.EnableKubecostProxy(), args.ConsoleUrl(), args.DeployToken(), userGroupCache, insightRegistry) svcReconciler := consoleManager.GetReconcilerOrDie(service.Identifier).(*service.ServiceReconciler) nsReconciler := consoleManager.GetReconcilerOrDie(namespaces.Identifier).(*namespaces.NamespaceReconciler) @@ -218,10 +220,11 @@ func main() { // Start the console manager in background. runConsoleManagerInBackgroundOrDie(ctx, consoleManager) - pinger := ping.NewOrDie(extConsoleClient, config, kubeManager.GetClient(), discoveryCache, dbStore) + pinger := ping.NewOrDie(extConsoleClient, config, kubeManager.GetClient(), discoveryCache, dbStore, ping.WithInsightGracePeriod(args.InsightGracePeriod()), ping.WithInsightPriorities(insightpriority.NewEvaluator(insightRegistry))) - // Start cluster pinger - ping.RunClusterPingerInBackgroundOrDie(ctx, pinger, args.ClusterPingInterval()) + // Start cluster pinger. Unlike the rest of the agent, it waits for all synchronizers to load the store + // instead of giving up at the supervisor cache sync timeout, as it reports on the whole cluster. + ping.RunClusterPingerInBackgroundOrDie(ctx, pinger, args.ClusterPingInterval(), supervisor.WaitUntilSynced) // Start runtime services pinger ping.RunRuntimeServicePingerInBackgroundOrDie(ctx, pinger, args.RuntimeServicesPingInterval()) diff --git a/go/deployment-operator/config/crd/bases/deployments.plural.sh_custominsightpriorities.yaml b/go/deployment-operator/config/crd/bases/deployments.plural.sh_custominsightpriorities.yaml new file mode 100644 index 0000000000..0a09641aae --- /dev/null +++ b/go/deployment-operator/config/crd/bases/deployments.plural.sh_custominsightpriorities.yaml @@ -0,0 +1,259 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: custominsightpriorities.deployments.plural.sh +spec: + group: deployments.plural.sh + names: + kind: CustomInsightPriority + listKind: CustomInsightPriorityList + plural: custominsightpriorities + singular: custominsightpriority + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .spec.precedence + name: Precedence + type: integer + - jsonPath: .status.lastEvaluation.matched + name: Matched + type: integer + name: v1alpha1 + schema: + openAPIV3Schema: + description: CustomInsightPriority sets the priority of failing objects in + the cluster insights and the cluster health score. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CustomInsightPrioritySpec defines how the insight priority + of failing objects is decided. + properties: + engine: + default: python + description: Engine the script is written for. + enum: + - python + type: string + precedence: + description: |- + Precedence decides which script owns an object that several targets match: the highest wins and ties are + broken by name. A script never sees objects it does not own, and scripts are not chained. + type: integer + script: + description: |- + Script defines `evaluate(input)`, called with the objects this script owns in `input["rows"]`, and returns a + dictionary from the `uid` of a row to `{"priority": "LOW|MEDIUM|HIGH|CRITICAL|IGNORE", "reason": "..."}`. + Rows that are left out keep their built-in priority, IGNORE removes a row from the insights and the cluster health score. + + Objects annotated with deployments.plural.sh/insight-priority are decided by the annotation and never reach the script. + + See https://docs.plural.sh/plural-features/continuous-deployment/deployment-operator/insight-priority for the full guide. + type: string + target: + description: Target selects the failing objects this script sets the + priority of. + properties: + kinds: + description: Kinds the script is responsible for. + items: + description: InsightTargetKind selects objects by kind. + properties: + group: + description: Group of the kind, empty for the core group. + type: string + kind: + description: Kind of the objects, for example StatefulSet. + minLength: 1 + type: string + version: + description: Version of the kind, any version matches when + empty. + type: string + required: + - kind + type: object + type: array + labelSelector: + description: LabelSelector selects objects by their labels. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: Namespaces the script is responsible for. Cluster + scoped objects only match when this is empty. + items: + type: string + type: array + type: object + required: + - script + type: object + status: + description: CustomInsightPriorityStatus defines the observed state of + CustomInsightPriority + properties: + conditions: + description: Represents the observations of a CustomInsightPriority's + current state. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + lastError: + description: LastError is the error of the last evaluation, empty + when it succeeded. + type: string + lastEvaluation: + description: LastEvaluation is the outcome of the last evaluation, + empty until the script ran. + properties: + durationMs: + description: DurationMs is how long the script took. + format: int64 + type: integer + ignored: + description: Ignored is the number of objects the script ignored. + type: integer + matched: + description: Matched is the number of failing objects the script + owned. + type: integer + overridden: + description: Overridden is the number of objects the script gave + a priority. + type: integer + time: + description: Time the script last ran with a different outcome + than the one before. + format: date-time + type: string + required: + - durationMs + - ignored + - matched + - overridden + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/go/deployment-operator/config/rbac/role.yaml b/go/deployment-operator/config/rbac/role.yaml index be621f1780..f95457ce1d 100644 --- a/go/deployment-operator/config/rbac/role.yaml +++ b/go/deployment-operator/config/rbac/role.yaml @@ -4,6 +4,13 @@ kind: ClusterRole metadata: name: manager-role rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch - apiGroups: - apps resources: @@ -34,6 +41,7 @@ rules: resources: - agentruntimes - customhealths + - custominsightpriorities - imagewarmers - kubecostextractors - pipelinegates @@ -50,6 +58,7 @@ rules: resources: - agentruntimes/finalizers - customhealths/finalizers + - custominsightpriorities/finalizers - imagewarmers/finalizers - kubecostextractors/finalizers - pipelinegates/finalizers @@ -60,6 +69,7 @@ rules: resources: - agentruntimes/status - customhealths/status + - custominsightpriorities/status - imagewarmers/status - kubecostextractors/status - pipelinegates/status diff --git a/go/deployment-operator/docs/api.md b/go/deployment-operator/docs/api.md index c38721f91c..b48be5d669 100644 --- a/go/deployment-operator/docs/api.md +++ b/go/deployment-operator/docs/api.md @@ -14,6 +14,7 @@ Package v1alpha1 contains API Schema definitions for the deployments v1alpha1 AP - [AgentRuntime](#agentruntime) - [ClusterDrain](#clusterdrain) - [CustomHealth](#customhealth) +- [CustomInsightPriority](#custominsightpriority) - [ImageWarmer](#imagewarmer) - [IngressReplica](#ingressreplica) - [KubecostExtractor](#kubecostextractor) @@ -604,6 +605,45 @@ _Appears in:_ +#### CustomInsightPriority + + + +CustomInsightPriority sets the priority of failing objects in the cluster insights and the cluster health score. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `deployments.plural.sh/v1alpha1` | | | +| `kind` _string_ | `CustomInsightPriority` | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[CustomInsightPrioritySpec](#custominsightpriorityspec)_ | | | | + + +#### CustomInsightPrioritySpec + + + +CustomInsightPrioritySpec defines how the insight priority of failing objects is decided. + + + +_Appears in:_ +- [CustomInsightPriority](#custominsightpriority) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `target` _[InsightTarget](#insighttarget)_ | Target selects the failing objects this script sets the priority of. | | Optional: \{\}
| +| `precedence` _integer_ | Precedence decides which script owns an object that several targets match: the highest wins and ties are
broken by name. A script never sees objects it does not own, and scripts are not chained. | | Optional: \{\}
| +| `engine` _[InsightEngine](#insightengine)_ | Engine the script is written for. | python | Enum: [python]
Optional: \{\}
| +| `script` _string_ | Script defines `evaluate(input)`, called with the objects this script owns in `input["rows"]`, and returns a
dictionary from the `uid` of a row to `\{"priority": "LOW\|MEDIUM\|HIGH\|CRITICAL\|IGNORE", "reason": "..."\}`.
Rows that are left out keep their built-in priority, IGNORE removes a row from the insights and the cluster health score.
Objects annotated with deployments.plural.sh/insight-priority are decided by the annotation and never reach the script.
See https://docs.plural.sh/plural-features/continuous-deployment/deployment-operator/insight-priority for the full guide. | | | + + + + #### ExaConnection @@ -863,6 +903,80 @@ _Appears in:_ | `hostMappings` _object (keys:string, values:string)_ | | | Required: \{\}
| +#### InsightEngine + +_Underlying type:_ _string_ + +InsightEngine is the language an insight priority script is written in. + +_Validation:_ +- Enum: [python] + +_Appears in:_ +- [CustomInsightPrioritySpec](#custominsightpriorityspec) + +| Field | Description | +| --- | --- | +| `python` | InsightEnginePython evaluates the script as sandboxed Python.
| + + +#### InsightEvaluation + + + +InsightEvaluation is the outcome of the last time the script ran. + + + +_Appears in:_ +- [CustomInsightPriorityStatus](#custominsightprioritystatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `time` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#time-v1-meta)_ | Time the script last ran with a different outcome than the one before. | | | +| `matched` _integer_ | Matched is the number of failing objects the script owned. | | | +| `overridden` _integer_ | Overridden is the number of objects the script gave a priority. | | | +| `ignored` _integer_ | Ignored is the number of objects the script ignored. | | | +| `durationMs` _integer_ | DurationMs is how long the script took. | | | + + +#### InsightTarget + + + +InsightTarget selects the failing objects a script is responsible for. An object has to match every +non-empty field, so an empty target matches everything. + + + +_Appears in:_ +- [CustomInsightPrioritySpec](#custominsightpriorityspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `kinds` _[InsightTargetKind](#insighttargetkind) array_ | Kinds the script is responsible for. | | Optional: \{\}
| +| `namespaces` _string array_ | Namespaces the script is responsible for. Cluster scoped objects only match when this is empty. | | Optional: \{\}
| +| `labelSelector` _[LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#labelselector-v1-meta)_ | LabelSelector selects objects by their labels. | | Optional: \{\}
| + + +#### InsightTargetKind + + + +InsightTargetKind selects objects by kind. + + + +_Appears in:_ +- [InsightTarget](#insighttarget) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `group` _string_ | Group of the kind, empty for the core group. | | Optional: \{\}
| +| `version` _string_ | Version of the kind, any version matches when empty. | | Optional: \{\}
| +| `kind` _string_ | Kind of the objects, for example StatefulSet. | | MinLength: 1
| + + #### KubecostExtractor diff --git a/go/deployment-operator/go.mod b/go/deployment-operator/go.mod index c366bcebcf..5222f8dbee 100644 --- a/go/deployment-operator/go.mod +++ b/go/deployment-operator/go.mod @@ -64,7 +64,6 @@ require ( github.com/pluralsh/controller-reconcile-helper v0.1.0 github.com/prometheus/client_golang v1.23.2 github.com/robfig/cron/v3 v3.0.1 - github.com/sahilm/fuzzy v0.1.1 github.com/samber/lo v1.53.0 github.com/sirupsen/logrus v1.9.4 github.com/spf13/pflag v1.0.10 diff --git a/go/deployment-operator/go.sum b/go/deployment-operator/go.sum index 903a661f24..b77fcbfd00 100644 --- a/go/deployment-operator/go.sum +++ b/go/deployment-operator/go.sum @@ -764,8 +764,6 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= -github.com/sahilm/fuzzy v0.1.1 h1:ceu5RHF8DGgoi+/dR5PsECjCDH1BE3Fnmpo7aVXOdRA= -github.com/sahilm/fuzzy v0.1.1/go.mod h1:VFvziUEIMCrT6A6tw2RFIXPXXmzXbOsSHF0DOI8ZK9Y= github.com/samber/lo v1.53.0 h1:t975lj2py4kJPQ6haz1QMgtId2gtmfktACxIXArw3HM= github.com/samber/lo v1.53.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= github.com/samber/oops v1.19.0 h1:sfZAwC8MmTXBRRyNc4Z1utuTPBx+hFKF5fJ9DEQRZfw= diff --git a/go/deployment-operator/internal/controller/custominsightpriority_controller.go b/go/deployment-operator/internal/controller/custominsightpriority_controller.go new file mode 100644 index 0000000000..88cc48ce26 --- /dev/null +++ b/go/deployment-operator/internal/controller/custominsightpriority_controller.go @@ -0,0 +1,161 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "strings" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/events" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/pluralsh/console/go/deployment-operator/api/v1alpha1" + "github.com/pluralsh/console/go/deployment-operator/internal/utils" + "github.com/pluralsh/console/go/deployment-operator/pkg/insightpriority" +) + +// CustomInsightPriorityReconciler registers the scripts that decide the priority of failing objects in the cluster +// insights, and reports how they are doing. The scripts run when the cluster is pinged, not here. +type CustomInsightPriorityReconciler struct { + client.Client + Scheme *runtime.Scheme + Registry *insightpriority.Registry + Recorder events.EventRecorder +} + +//+kubebuilder:rbac:groups=deployments.plural.sh,resources=custominsightpriorities,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=deployments.plural.sh,resources=custominsightpriorities/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=deployments.plural.sh,resources=custominsightpriorities/finalizers,verbs=update +//+kubebuilder:rbac:groups="",resources=events,verbs=create;patch + +// Reconcile compiles the script and registers it when it is valid, then copies the result of its last evaluation to the status. +func (r *CustomInsightPriorityReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ reconcile.Result, reterr error) { + logger := log.FromContext(ctx) + + cr := &v1alpha1.CustomInsightPriority{} + if err := r.Get(ctx, req.NamespacedName, cr); err != nil { + if client.IgnoreNotFound(err) == nil { + r.Registry.Remove(req.NamespacedName) + return ctrl.Result{}, nil + } + + logger.Error(err, "Unable to fetch CustomInsightPriority") + return ctrl.Result{}, err + } + + scope, err := NewDefaultScope(ctx, r.Client, cr) + if err != nil { + logger.Error(err, "Failed to create scope") + return ctrl.Result{}, err + } + defer func() { + if err := scope.PatchObject(); err != nil && reterr == nil { + reterr = err + } + }() + + script, err := r.compile(ctx, cr) + if err != nil { + // A script that does not compile is never evaluated, so its objects keep their built-in priority. + r.Registry.Remove(req.NamespacedName) + cr.Status.LastEvaluation = nil + cr.Status.LastError = "" + cr.RemoveCondition(v1alpha1.OverlappingConditionType) + utils.MarkCondition(cr.SetCondition, v1alpha1.ReadyConditionType, v1.ConditionFalse, v1alpha1.InvalidScriptConditionReason, err.Error()) + return ctrl.Result{}, nil + } + + r.Registry.Set(script) + r.syncStatus(cr) + return ctrl.Result{}, nil +} + +func (r *CustomInsightPriorityReconciler) compile(ctx context.Context, cr *v1alpha1.CustomInsightPriority) (insightpriority.Script, error) { + if engine := cr.Spec.Engine; engine != "" && engine != v1alpha1.InsightEnginePython { + return insightpriority.Script{}, fmt.Errorf("unsupported engine %q, only %q is available", engine, v1alpha1.InsightEnginePython) + } + + script, err := insightpriority.NewScript(cr) + if err != nil { + return insightpriority.Script{}, err + } + + if err := insightpriority.Compile(ctx, cr.Spec.Script); err != nil { + return insightpriority.Script{}, err + } + + return script, nil +} + +// syncStatus reflects the last evaluation of the script in the status. +func (r *CustomInsightPriorityReconciler) syncStatus(cr *v1alpha1.CustomInsightPriority) { + key := client.ObjectKeyFromObject(cr) + result, evaluated := r.Registry.Result(key) + if !evaluated { + cr.Status.LastEvaluation = nil + cr.Status.LastError = "" + cr.RemoveCondition(v1alpha1.OverlappingConditionType) + utils.MarkCondition(cr.SetCondition, v1alpha1.ReadyConditionType, v1.ConditionTrue, v1alpha1.ReadyConditionReason, "") + return + } + + cr.Status.LastEvaluation = &v1alpha1.InsightEvaluation{ + Time: v1.NewTime(result.Time), + Matched: result.Matched, + Overridden: result.Overridden, + Ignored: result.Ignored, + DurationMs: result.Duration.Milliseconds(), + } + + if result.Err != "" { + if cr.Status.LastError != result.Err { + r.Recorder.Eventf(cr, nil, "Warning", string(v1alpha1.EvaluationFailedConditionReason), "Evaluate", "%s", result.Err) + } + cr.Status.LastError = result.Err + utils.MarkCondition(cr.SetCondition, v1alpha1.ReadyConditionType, v1.ConditionFalse, v1alpha1.EvaluationFailedConditionReason, result.Err) + } else { + cr.Status.LastError = "" + utils.MarkCondition(cr.SetCondition, v1alpha1.ReadyConditionType, v1.ConditionTrue, v1alpha1.ReadyConditionReason, "") + } + + if len(result.LostTo) > 0 { + winners := make([]string, 0, len(result.LostTo)) + for _, winner := range result.LostTo { + winners = append(winners, winner.String()) + } + utils.MarkCondition(cr.SetCondition, v1alpha1.OverlappingConditionType, v1.ConditionTrue, v1alpha1.OverlapConditionReason, + "some objects are owned by a script with a higher precedence: "+strings.Join(winners, ", ")) + } else { + cr.RemoveCondition(v1alpha1.OverlappingConditionType) + } +} + +// SetupWithManager sets up the controller with the Manager. +func (r *CustomInsightPriorityReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.CustomInsightPriority{}). + WatchesRawSource(source.Channel(r.Registry.Events(), &handler.EnqueueRequestForObject{})). + Complete(r) +} diff --git a/go/deployment-operator/internal/controller/custominsightpriority_controller_unit_test.go b/go/deployment-operator/internal/controller/custominsightpriority_controller_unit_test.go new file mode 100644 index 0000000000..a0d2cbce10 --- /dev/null +++ b/go/deployment-operator/internal/controller/custominsightpriority_controller_unit_test.go @@ -0,0 +1,152 @@ +package controller + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/events" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/pluralsh/console/go/deployment-operator/api/v1alpha1" + "github.com/pluralsh/console/go/deployment-operator/pkg/insightpriority" + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/insights" +) + +func TestCustomInsightPriorityReconciler(t *testing.T) { + ctx := context.Background() + key := types.NamespacedName{Namespace: "plrl-deploy-operator", Name: "payments"} + validScript := "def evaluate(input):\n return {r['uid']: {'priority': 'CRITICAL'} for r in input['rows']}\n" + + setup := func(t *testing.T, spec v1alpha1.CustomInsightPrioritySpec) (*CustomInsightPriorityReconciler, *events.FakeRecorder) { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, v1alpha1.AddToScheme(scheme)) + cr := &v1alpha1.CustomInsightPriority{ObjectMeta: metav1.ObjectMeta{Namespace: key.Namespace, Name: key.Name, Generation: 1}, Spec: spec} + recorder := events.NewFakeRecorder(10) + return &CustomInsightPriorityReconciler{ + Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(cr).WithStatusSubresource(cr).Build(), + Scheme: scheme, + Registry: insightpriority.NewRegistry(), + Recorder: recorder, + }, recorder + } + + reconcile := func(t *testing.T, r *CustomInsightPriorityReconciler) *v1alpha1.CustomInsightPriority { + t.Helper() + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + require.NoError(t, err) + cr := &v1alpha1.CustomInsightPriority{} + require.NoError(t, r.Get(ctx, key, cr)) + return cr + } + + condition := func(cr *v1alpha1.CustomInsightPriority, conditionType v1alpha1.ConditionType) *metav1.Condition { + for i := range cr.Status.Conditions { + if cr.Status.Conditions[i].Type == conditionType.String() { + return &cr.Status.Conditions[i] + } + } + return nil + } + + t.Run("registers a valid script and reports it ready", func(t *testing.T) { + r, _ := setup(t, v1alpha1.CustomInsightPrioritySpec{Script: validScript, Precedence: 7}) + + cr := reconcile(t, r) + + ready := condition(cr, v1alpha1.ReadyConditionType) + require.NotNil(t, ready) + assert.Equal(t, metav1.ConditionTrue, ready.Status) + scripts := r.Registry.Scripts() + require.Len(t, scripts, 1) + assert.Equal(t, 7, scripts[0].Precedence) + assert.Nil(t, cr.Status.LastEvaluation, "it has not run yet") + }) + + t.Run("never registers a script that does not compile", func(t *testing.T) { + r, _ := setup(t, v1alpha1.CustomInsightPrioritySpec{Script: "def evaluate(input:\n"}) + + cr := reconcile(t, r) + + ready := condition(cr, v1alpha1.ReadyConditionType) + require.NotNil(t, ready) + assert.Equal(t, metav1.ConditionFalse, ready.Status) + assert.Equal(t, v1alpha1.InvalidScriptConditionReason.String(), ready.Reason) + assert.Empty(t, r.Registry.Scripts()) + }) + + t.Run("rejects an engine that is not available", func(t *testing.T) { + r, _ := setup(t, v1alpha1.CustomInsightPrioritySpec{Script: validScript, Engine: "rego"}) + + cr := reconcile(t, r) + + assert.Equal(t, metav1.ConditionFalse, condition(cr, v1alpha1.ReadyConditionType).Status) + assert.Empty(t, r.Registry.Scripts()) + }) + + t.Run("forgets a deleted script", func(t *testing.T) { + r, _ := setup(t, v1alpha1.CustomInsightPrioritySpec{Script: validScript}) + reconcile(t, r) + require.NoError(t, r.Delete(ctx, &v1alpha1.CustomInsightPriority{ObjectMeta: metav1.ObjectMeta{Namespace: key.Namespace, Name: key.Name}})) + + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: key}) + + require.NoError(t, err) + assert.Empty(t, r.Registry.Scripts()) + }) + + t.Run("copies an evaluation to the status", func(t *testing.T) { + r, _ := setup(t, v1alpha1.CustomInsightPrioritySpec{Script: validScript}) + reconcile(t, r) + evaluator := insightpriority.NewEvaluator(r.Registry) + now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) + evaluator.Overrides(ctx, []insights.Root{{UID: "a", Kind: "Deployment", Namespace: "apps", Name: "a"}, {UID: "b", Kind: "Deployment", Namespace: "apps", Name: "b"}}, insightpriority.Cluster{}, now) + require.Len(t, r.Registry.Events(), 1, "the controller is told to look") + + cr := reconcile(t, r) + + require.NotNil(t, cr.Status.LastEvaluation) + assert.Equal(t, 2, cr.Status.LastEvaluation.Matched) + assert.Equal(t, 2, cr.Status.LastEvaluation.Overridden) + assert.True(t, cr.Status.LastEvaluation.Time.Equal(&metav1.Time{Time: now})) + assert.Empty(t, cr.Status.LastError) + assert.Equal(t, metav1.ConditionTrue, condition(cr, v1alpha1.ReadyConditionType).Status) + }) + + t.Run("reports a failing evaluation once as a warning", func(t *testing.T) { + r, recorder := setup(t, v1alpha1.CustomInsightPrioritySpec{Script: "def evaluate(input):\n return input['missing']\n"}) + reconcile(t, r) + insightpriority.NewEvaluator(r.Registry).Overrides(ctx, []insights.Root{{UID: "a", Kind: "Deployment", Name: "a"}}, insightpriority.Cluster{}, time.Now()) + + cr := reconcile(t, r) + cr = reconcile(t, r) + + ready := condition(cr, v1alpha1.ReadyConditionType) + assert.Equal(t, metav1.ConditionFalse, ready.Status) + assert.Equal(t, v1alpha1.EvaluationFailedConditionReason.String(), ready.Reason) + assert.Contains(t, cr.Status.LastError, "KeyError") + assert.Len(t, recorder.Events, 1, "a repeated reconcile does not repeat the event") + }) + + t.Run("flags a script that lost objects to another script", func(t *testing.T) { + r, _ := setup(t, v1alpha1.CustomInsightPrioritySpec{Script: validScript}) + reconcile(t, r) + winner := insightpriority.Script{Key: types.NamespacedName{Namespace: "plrl-deploy-operator", Name: "aaa"}, Precedence: 100, Source: validScript} + r.Registry.Set(winner) + insightpriority.NewEvaluator(r.Registry).Overrides(ctx, []insights.Root{{UID: "a", Kind: "Deployment", Name: "a"}}, insightpriority.Cluster{}, time.Now()) + + cr := reconcile(t, r) + + overlapping := condition(cr, v1alpha1.OverlappingConditionType) + require.NotNil(t, overlapping) + assert.Equal(t, metav1.ConditionTrue, overlapping.Status) + assert.Contains(t, overlapping.Message, "aaa") + }) +} diff --git a/go/deployment-operator/internal/metrics/metrics_prometheus.go b/go/deployment-operator/internal/metrics/metrics_prometheus.go index ca4a012120..b844ee6cef 100644 --- a/go/deployment-operator/internal/metrics/metrics_prometheus.go +++ b/go/deployment-operator/internal/metrics/metrics_prometheus.go @@ -31,6 +31,8 @@ type prometheusRecorder struct { controllerRestartCounter *prometheus.CounterVec + insightPriorityFailureCounter *prometheus.CounterVec + synchronizationEventCounter *prometheus.CounterVec } @@ -111,6 +113,10 @@ func (in *prometheusRecorder) ControllerRestart(name string) { in.controllerRestartCounter.WithLabelValues(name).Inc() } +func (in *prometheusRecorder) InsightPriorityFailure(name string) { + in.insightPriorityFailureCounter.WithLabelValues(name).Inc() +} + func (in *prometheusRecorder) init() Recorder { in.discoveryAPICacheRefreshCounter = promauto.NewCounter(prometheus.CounterOpts{ Name: DiscoveryAPICacheRefreshMetricName, @@ -157,6 +163,11 @@ func (in *prometheusRecorder) init() Recorder { Help: ControllerRestartsMetricDescription, }, []string{MetricLabelControllerName}) + in.insightPriorityFailureCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: InsightPriorityFailureMetricName, + Help: InsightPriorityFailureMetricDescription, + }, []string{MetricLabelInsightPriorityName}) + in.synchronizationEventCounter = promauto.NewCounterVec(prometheus.CounterOpts{ Name: SynchronizerEventMetricName, Help: SynchronizerEventMetricDescription, diff --git a/go/deployment-operator/internal/metrics/metrics_types.go b/go/deployment-operator/internal/metrics/metrics_types.go index 5a9317d1c6..b36aac194b 100644 --- a/go/deployment-operator/internal/metrics/metrics_types.go +++ b/go/deployment-operator/internal/metrics/metrics_types.go @@ -43,6 +43,10 @@ const ( ControllerRestartsMetricName = "agent_controller_restarts_total" ControllerRestartsMetricDescription = "The total number of controller restarts" + InsightPriorityFailureMetricName = "agent_insight_priority_failures_total" + InsightPriorityFailureMetricDescription = "The total number of failed evaluations of CustomInsightPriority scripts" + MetricLabelInsightPriorityName = "custom_insight_priority" + SynchronizerEventMetricName = "agent_synchronizer_event_total" SynchronizerEventMetricDescription = "The total number of synchronizer events" @@ -92,5 +96,6 @@ type Recorder interface { ResourceCacheHit(serviceID string) ResourceCacheMiss(serviceID string) ControllerRestart(name string) + InsightPriorityFailure(name string) SynchronizationEvent(event watch.Event) } diff --git a/go/deployment-operator/pkg/common/health.go b/go/deployment-operator/pkg/common/health.go index 2dd5c6381b..535c95a356 100644 --- a/go/deployment-operator/pkg/common/health.go +++ b/go/deployment-operator/pkg/common/health.go @@ -241,6 +241,8 @@ func asString(value interface{}) string { } func checkConditions(conditions []hpaCondition, progressingStatus *HealthStatus) (*HealthStatus, error) { + // Degraded wins over healthy no matter the order. The HPA controller lists AbleToScale + // first and it stays True while ScalingActive is False with FailedGetResourceMetric. for _, condition := range conditions { if isDegraded(&condition) { return &HealthStatus{ @@ -248,7 +250,9 @@ func checkConditions(conditions []hpaCondition, progressingStatus *HealthStatus) Message: condition.Message, }, nil } + } + for _, condition := range conditions { if isHealthy(&condition) { return &HealthStatus{ Status: HealthStatusHealthy, diff --git a/go/deployment-operator/pkg/common/health_test.go b/go/deployment-operator/pkg/common/health_test.go index 44caf49ec5..3e9075957d 100644 --- a/go/deployment-operator/pkg/common/health_test.go +++ b/go/deployment-operator/pkg/common/health_test.go @@ -156,5 +156,46 @@ var _ = Describe("Health Test", Ordered, func() { } }) + DescribeTable("should get HPA status from conditions", + func(conditions []interface{}, expected common.HealthStatus) { + for _, apiVersion := range []string{"autoscaling/v2", "autoscaling/v2beta2"} { + obj := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": apiVersion, + "kind": common.HorizontalPodAutoscalerKind, + "status": map[string]interface{}{"conditions": conditions}, + }, + } + + status, err := common.GetResourceHealth(obj) + Expect(err).NotTo(HaveOccurred()) + Expect(status).To(Not(BeNil())) + Expect(*status).To(Equal(expected)) + } + }, + Entry("degraded when metrics can not be read although AbleToScale is true", + []interface{}{ + map[string]interface{}{"type": "AbleToScale", "status": "True", "reason": "SucceededGetScale", "message": "scaled"}, + map[string]interface{}{"type": "ScalingActive", "status": "False", "reason": "FailedGetResourceMetric", "message": "no metrics"}, + }, + common.HealthStatus{Status: common.HealthStatusDegraded, Message: "no metrics"}), + Entry("degraded when the scale target can not be read", + []interface{}{ + map[string]interface{}{"type": "AbleToScale", "status": "False", "reason": "FailedGetScale", "message": "not found"}, + }, + common.HealthStatus{Status: common.HealthStatusDegraded, Message: "not found"}), + Entry("healthy when only healthy conditions are present", + []interface{}{ + map[string]interface{}{"type": "AbleToScale", "status": "True", "reason": "SucceededGetScale", "message": "scaled"}, + map[string]interface{}{"type": "ScalingActive", "status": "True", "reason": "ValidMetricFound", "message": "ok"}, + }, + common.HealthStatus{Status: common.HealthStatusHealthy, Message: "scaled"}), + Entry("progressing when no condition is known yet", + []interface{}{ + map[string]interface{}{"type": "ScalingActive", "status": "True", "reason": "ValidMetricFound", "message": "ok"}, + }, + common.HealthStatus{Status: common.HealthStatusProgressing, Message: "Waiting to Autoscale"}), + ) + }) }) diff --git a/go/deployment-operator/pkg/common/status.go b/go/deployment-operator/pkg/common/status.go index 3a1508ce69..6347c3d202 100644 --- a/go/deployment-operator/pkg/common/status.go +++ b/go/deployment-operator/pkg/common/status.go @@ -50,27 +50,30 @@ type Progress struct { } func ToStatus(obj *unstructured.Unstructured) *console.ComponentState { + state, _ := ToStatusWithMessage(obj) + return state +} + +// ToStatusWithMessage returns the component state together with the message of the health check that produced it. +func ToStatusWithMessage(obj *unstructured.Unstructured) (*console.ComponentState, string) { h, err := GetResourceHealth(obj) if err != nil { klog.ErrorS(err, "failed to get resource health status", "name", obj.GetName(), "namespace", obj.GetNamespace()) } if h == nil { - return nil - } - - if h.Status == HealthStatusDegraded { - return lo.ToPtr(console.ComponentStateFailed) + return nil, "" } - if h.Status == HealthStatusHealthy { - return lo.ToPtr(console.ComponentStateRunning) + switch h.Status { + case HealthStatusDegraded: + return lo.ToPtr(console.ComponentStateFailed), h.Message + case HealthStatusHealthy: + return lo.ToPtr(console.ComponentStateRunning), h.Message + case HealthStatusPaused: + return lo.ToPtr(console.ComponentStatePaused), h.Message + default: + return lo.ToPtr(console.ComponentStatePending), h.Message } - - if h.Status == HealthStatusPaused { - return lo.ToPtr(console.ComponentStatePaused) - } - - return lo.ToPtr(console.ComponentStatePending) } func ToComponentAttributes(obj *unstructured.Unstructured) *console.ComponentAttributes { diff --git a/go/deployment-operator/pkg/insightpriority/evaluator.go b/go/deployment-operator/pkg/insightpriority/evaluator.go new file mode 100644 index 0000000000..3b8a917814 --- /dev/null +++ b/go/deployment-operator/pkg/insightpriority/evaluator.go @@ -0,0 +1,223 @@ +package insightpriority + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "time" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + + "github.com/pluralsh/console/go/deployment-operator/internal/metrics" + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/insights" +) + +// Cluster is what scripts are told about the cluster as a whole. +type Cluster struct { + Nodes int64 + Namespaces int64 +} + +// RunFunc calls the evaluate function of a script with an input and returns what it returned, as JSON. +type RunFunc func(ctx context.Context, script string, input any) (json.RawMessage, error) + +// Evaluator runs the registered scripts over the failing roots of a ping. +type Evaluator struct { + registry *Registry + run RunFunc +} + +func NewEvaluator(registry *Registry) *Evaluator { + return &Evaluator{registry: registry, run: Run} +} + +// Overrides gives every failing root to the script that owns it, and returns the priorities the scripts decided. +// +// A root is owned by at most one script, the matching one with the highest precedence, scripts are not chained and a +// script never sees a root that is decided by its annotation. A script that fails only loses its own roots to the +// built-in priority. The result is the same for the same input, nothing time or state dependent reaches a script. +func (e *Evaluator) Overrides(ctx context.Context, roots []insights.Root, cluster Cluster, now time.Time) map[string]insights.Override { + scripts := e.registry.Scripts() + if len(scripts) == 0 { + return nil + } + + owned := make(map[types.NamespacedName][]insights.Entry, len(scripts)) + results := make(map[types.NamespacedName]Result, len(scripts)) + lostTo := make(map[types.NamespacedName]map[types.NamespacedName]struct{}) + + failing := make([]map[string]any, 0, len(roots)) + for _, root := range roots { + entry := insights.Resolve(root, nil, now) + failing = append(failing, map[string]any{ + "kind": root.Kind, + "namespace": root.Namespace, + "name": root.Name, + "builtin_priority": entry.Priority, + }) + + if root.PriorityAnnotation != "" { + continue + } + + var winner *Script + for i := range scripts { + if !scripts[i].Matches(root) { + continue + } + + if winner == nil { + winner = &scripts[i] + owned[winner.Key] = append(owned[winner.Key], entry) + continue + } + + if lostTo[scripts[i].Key] == nil { + lostTo[scripts[i].Key] = make(map[types.NamespacedName]struct{}) + } + lostTo[scripts[i].Key][winner.Key] = struct{}{} + } + } + + overrides := make(map[string]insights.Override) + for _, script := range scripts { + result := e.evaluate(ctx, script, owned[script.Key], failing, cluster, now, overrides) + result.Time = now + result.Generation = script.Generation + for winner := range lostTo[script.Key] { + result.LostTo = append(result.LostTo, winner) + } + slices.SortFunc(result.LostTo, func(a, b types.NamespacedName) int { return strings.Compare(a.String(), b.String()) }) + results[script.Key] = result + } + + e.registry.record(results) + return overrides +} + +// evaluate runs one script over its entries and adds what it decided to overrides. When the script fails nothing is added. +func (e *Evaluator) evaluate(ctx context.Context, script Script, entries []insights.Entry, failing []map[string]any, cluster Cluster, now time.Time, overrides map[string]insights.Override) Result { + result := Result{Matched: len(entries)} + if len(entries) == 0 { + return result + } + + start := time.Now() + decided, err := e.decide(ctx, script, entries, failing, cluster, now) + result.Duration = time.Since(start) + if err != nil { + result.Err = err.Error() + klog.ErrorS(err, "insight priority script failed, using built-in priorities", "script", script.Key.String()) + metrics.Record().InsightPriorityFailure(script.Key.String()) + return result + } + + for uid, override := range decided { + overrides[uid] = override + if override.Priority == insights.PriorityIgnore { + result.Ignored++ + } else { + result.Overridden++ + } + } + return result +} + +type scriptOutput struct { + Priority string `json:"priority"` + Reason string `json:"reason"` +} + +func (e *Evaluator) decide(ctx context.Context, script Script, entries []insights.Entry, failing []map[string]any, cluster Cluster, now time.Time) (map[string]insights.Override, error) { + rows := make([]map[string]any, 0, len(entries)) + known := make(map[string]struct{}, len(entries)) + for _, entry := range entries { + rows = append(rows, row(entry)) + known[entry.UID] = struct{}{} + } + + raw, err := e.run(ctx, script.Source, map[string]any{ + "now": now.UTC().Format(time.RFC3339), + "cluster": map[string]any{ + "nodes": cluster.Nodes, + "namespaces": cluster.Namespaces, + "failing_roots": failing, + }, + "rows": rows, + }) + if err != nil { + return nil, fmt.Errorf("evaluation failed: %w", err) + } + + var output map[string]scriptOutput + if err := json.Unmarshal(raw, &output); err != nil { + return nil, fmt.Errorf("evaluate must return a dictionary from uid to {\"priority\": ..., \"reason\": ...}") + } + + decided := make(map[string]insights.Override, len(output)) + for uid, o := range output { + if _, ok := known[uid]; !ok { + return nil, errors.New("evaluate returned a uid that is not one of the rows it was given") + } + + priority := insights.Priority(strings.ToUpper(strings.TrimSpace(o.Priority))) + switch priority { + case insights.PriorityLow, insights.PriorityMedium, insights.PriorityHigh, insights.PriorityCritical, insights.PriorityIgnore: + default: + return nil, errors.New("evaluate returned an invalid priority, expected LOW, MEDIUM, HIGH, CRITICAL or IGNORE") + } + + decided[uid] = insights.Override{Priority: priority, Reason: o.Reason} + } + + return decided, nil +} + +// row is the view of a root a script gets. It carries no manifests, events, logs or annotations: they are not in the +// store, and fetching them would make a ping slow and its outcome depend on more than the input. +func row(e insights.Entry) map[string]any { + descendants := make(map[string]any, len(e.Descendants)) + for kind, d := range e.Descendants { + descendants[kind] = map[string]any{"failed": d.Failed, "pending": d.Pending} + } + + labels := e.Labels + if labels == nil { + labels = map[string]string{} + } + + return map[string]any{ + "uid": e.UID, + "group": e.Group, + "version": e.Version, + "kind": e.Kind, + "namespace": e.Namespace, + "name": e.Name, + "labels": labels, + "managed": e.ServiceID != "", + "service_id": e.ServiceID, + "priority_class": e.PriorityClass, + "state": e.State, + "health_message": e.HealthMessage, + "desired_replicas": e.DesiredReplicas, + "ready_replicas": e.ReadyReplicas, + "descendants": descendants, + "created_at": formatTime(e.CreatedAt), + "failing_since": formatTime(e.FailingSince), + "criticality": e.Criticality, + "builtin_score": e.Score, + "builtin_priority": e.Priority, + "builtin_reason": e.Reason, + } +} + +func formatTime(t time.Time) string { + if t.IsZero() { + return "" + } + return t.UTC().Format(time.RFC3339) +} diff --git a/go/deployment-operator/pkg/insightpriority/evaluator_test.go b/go/deployment-operator/pkg/insightpriority/evaluator_test.go new file mode 100644 index 0000000000..fe64d2457a --- /dev/null +++ b/go/deployment-operator/pkg/insightpriority/evaluator_test.go @@ -0,0 +1,386 @@ +package insightpriority_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/pluralsh/console/go/deployment-operator/api/v1alpha1" + "github.com/pluralsh/console/go/deployment-operator/pkg/insightpriority" + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/insights" +) + +var now = time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) + +func newScript(t *testing.T, name string, precedence int, target v1alpha1.InsightTarget, source string) insightpriority.Script { + t.Helper() + script, err := insightpriority.NewScript(&v1alpha1.CustomInsightPriority{ + ObjectMeta: metav1.ObjectMeta{Namespace: "plrl-deploy-operator", Name: name}, + Spec: v1alpha1.CustomInsightPrioritySpec{Target: target, Precedence: precedence, Script: source}, + }) + require.NoError(t, err) + return script +} + +func root(kind, namespace, name string, labels map[string]string) insights.Root { + return insights.Root{ + UID: name, Group: "apps", Version: "v1", Kind: kind, Namespace: namespace, Name: name, Labels: labels, + Criticality: insights.ScoreLow, CriticalityReason: "kind default", + } +} + +// setup returns an evaluator whose scripts are given by the fake run function, keyed by script source. +func setup(t *testing.T, scripts map[string]func(input map[string]any) (any, error), registered ...insightpriority.Script) (*insightpriority.Evaluator, *insightpriority.Registry) { + t.Helper() + registry := insightpriority.NewRegistry() + for _, s := range registered { + registry.Set(s) + } + evaluator := insightpriority.NewEvaluator(registry) + insightpriority.SetRunFunc(evaluator, func(_ context.Context, source string, input any) (json.RawMessage, error) { + raw, err := json.Marshal(input) + require.NoError(t, err) + var decoded map[string]any + require.NoError(t, json.Unmarshal(raw, &decoded)) + + out, err := scripts[source](decoded) + if err != nil { + return nil, err + } + return json.Marshal(out) + }) + return evaluator, registry +} + +func key(name string) types.NamespacedName { + return types.NamespacedName{Namespace: "plrl-deploy-operator", Name: name} +} + +func rowUIDs(input map[string]any) []string { + var uids []string + for _, r := range input["rows"].([]any) { + uids = append(uids, r.(map[string]any)["uid"].(string)) + } + return uids +} + +func TestEvaluatorOverrides(t *testing.T) { + critical := map[string]any{"priority": "CRITICAL", "reason": "payments"} + + t.Run("returns nothing without scripts", func(t *testing.T) { + evaluator, _ := setup(t, nil) + + assert.Nil(t, evaluator.Overrides(context.Background(), []insights.Root{root("Deployment", "apps", "api", nil)}, insightpriority.Cluster{}, now)) + }) + + t.Run("a script only sees the roots it targets", func(t *testing.T) { + var seen []string + script := newScript(t, "payments", 0, v1alpha1.InsightTarget{ + Kinds: []v1alpha1.InsightTargetKind{{Group: "apps", Kind: "StatefulSet"}}, + Namespaces: []string{"payments"}, + }, "payments") + evaluator, _ := setup(t, map[string]func(map[string]any) (any, error){ + "payments": func(input map[string]any) (any, error) { + seen = rowUIDs(input) + return map[string]any{"ledger": critical}, nil + }, + }, script) + + overrides := evaluator.Overrides(context.Background(), []insights.Root{ + root("StatefulSet", "payments", "ledger", nil), + root("StatefulSet", "other", "elsewhere", nil), + root("Deployment", "payments", "api", nil), + }, insightpriority.Cluster{}, now) + + assert.Equal(t, []string{"ledger"}, seen) + assert.Equal(t, map[string]insights.Override{"ledger": {Priority: insights.PriorityCritical, Reason: "payments"}}, overrides) + }) + + t.Run("targets by label selector, and an empty target takes everything", func(t *testing.T) { + var seen []string + labelled := newScript(t, "core", 10, v1alpha1.InsightTarget{LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"tier": "core"}}}, "core") + all := newScript(t, "catch-all", 0, v1alpha1.InsightTarget{}, "catch-all") + evaluator, _ := setup(t, map[string]func(map[string]any) (any, error){ + "core": func(map[string]any) (any, error) { return nil, nil }, + "catch-all": func(input map[string]any) (any, error) { seen = rowUIDs(input); return nil, nil }, + }, labelled, all) + + evaluator.Overrides(context.Background(), []insights.Root{ + root("Deployment", "apps", "api", map[string]string{"tier": "core"}), + root("Deployment", "apps", "batch", nil), + }, insightpriority.Cluster{}, now) + + assert.Equal(t, []string{"batch"}, seen, "the labelled root is owned by the higher precedence script") + }) + + t.Run("the highest precedence owns a root, ties go to the name, and the loser is told", func(t *testing.T) { + var seenA, seenB []string + a := newScript(t, "a", 5, v1alpha1.InsightTarget{}, "a") + b := newScript(t, "b", 5, v1alpha1.InsightTarget{}, "b") + evaluator, registry := setup(t, map[string]func(map[string]any) (any, error){ + "a": func(input map[string]any) (any, error) { seenA = rowUIDs(input); return nil, nil }, + "b": func(input map[string]any) (any, error) { seenB = rowUIDs(input); return nil, nil }, + }, b, a) + + evaluator.Overrides(context.Background(), []insights.Root{root("Deployment", "apps", "api", nil)}, insightpriority.Cluster{}, now) + + assert.Equal(t, []string{"api"}, seenA) + assert.Empty(t, seenB, "scripts are not chained") + result, ok := registry.Result(key("b")) + require.True(t, ok) + assert.Equal(t, []types.NamespacedName{key("a")}, result.LostTo) + }) + + t.Run("annotated roots never reach a script", func(t *testing.T) { + var seen []string + script := newScript(t, "all", 0, v1alpha1.InsightTarget{}, "all") + evaluator, _ := setup(t, map[string]func(map[string]any) (any, error){ + "all": func(input map[string]any) (any, error) { seen = rowUIDs(input); return nil, nil }, + }, script) + pinned := root("Deployment", "apps", "pinned", nil) + pinned.PriorityAnnotation = "LOW" + + evaluator.Overrides(context.Background(), []insights.Root{pinned, root("Deployment", "apps", "api", nil)}, insightpriority.Cluster{}, now) + + assert.Equal(t, []string{"api"}, seen) + }) + + t.Run("rows left out keep the built-in priority and IGNORE is counted", func(t *testing.T) { + script := newScript(t, "all", 0, v1alpha1.InsightTarget{}, "all") + evaluator, registry := setup(t, map[string]func(map[string]any) (any, error){ + "all": func(map[string]any) (any, error) { + return map[string]any{"a": critical, "b": map[string]any{"priority": "ignore"}}, nil + }, + }, script) + + overrides := evaluator.Overrides(context.Background(), []insights.Root{ + root("Deployment", "apps", "a", nil), root("Deployment", "apps", "b", nil), root("Deployment", "apps", "c", nil), + }, insightpriority.Cluster{}, now) + + assert.Len(t, overrides, 2) + assert.Equal(t, insights.PriorityIgnore, overrides["b"].Priority) + result, _ := registry.Result(key("all")) + assert.Equal(t, 3, result.Matched) + assert.Equal(t, 1, result.Overridden) + assert.Equal(t, 1, result.Ignored) + assert.Empty(t, result.Err) + }) + + t.Run("a failing script only loses its own roots", func(t *testing.T) { + failing := newScript(t, "failing", 10, v1alpha1.InsightTarget{Namespaces: []string{"broken"}}, "failing") + working := newScript(t, "working", 0, v1alpha1.InsightTarget{Namespaces: []string{"fine"}}, "working") + evaluator, registry := setup(t, map[string]func(map[string]any) (any, error){ + "failing": func(map[string]any) (any, error) { return nil, errors.New("KeyError") }, + "working": func(map[string]any) (any, error) { return map[string]any{"ok": critical}, nil }, + }, failing, working) + + overrides := evaluator.Overrides(context.Background(), []insights.Root{ + root("Deployment", "broken", "bad", nil), root("Deployment", "fine", "ok", nil), + }, insightpriority.Cluster{}, now) + + assert.Equal(t, map[string]insights.Override{"ok": {Priority: insights.PriorityCritical, Reason: "payments"}}, overrides) + result, _ := registry.Result(key("failing")) + assert.Contains(t, result.Err, "KeyError") + result, _ = registry.Result(key("working")) + assert.Empty(t, result.Err) + }) + + t.Run("an unknown uid or priority is an error that discards the whole script", func(t *testing.T) { + for name, output := range map[string]map[string]any{ + "unknown uid": {"a": critical, "nope": critical}, + "invalid priority": {"a": critical, "b": map[string]any{"priority": "URGENT"}}, + } { + t.Run(name, func(t *testing.T) { + script := newScript(t, "all", 0, v1alpha1.InsightTarget{}, "all") + evaluator, registry := setup(t, map[string]func(map[string]any) (any, error){ + "all": func(map[string]any) (any, error) { return output, nil }, + }, script) + + overrides := evaluator.Overrides(context.Background(), []insights.Root{root("Deployment", "apps", "a", nil), root("Deployment", "apps", "b", nil)}, insightpriority.Cluster{}, now) + + assert.Empty(t, overrides) + result, _ := registry.Result(key("all")) + assert.NotEmpty(t, result.Err) + assert.NotContains(t, result.Err, "nope", "what the script returned must not end up in a status that namespace users can read") + assert.NotContains(t, result.Err, "URGENT") + }) + } + }) + + t.Run("an output that is not a dictionary is an error, null is not", func(t *testing.T) { + script := newScript(t, "all", 0, v1alpha1.InsightTarget{}, "all") + evaluator, registry := setup(t, map[string]func(map[string]any) (any, error){ + "all": func(map[string]any) (any, error) { return []string{"a"}, nil }, + }, script) + + evaluator.Overrides(context.Background(), []insights.Root{root("Deployment", "apps", "a", nil)}, insightpriority.Cluster{}, now) + + result, _ := registry.Result(key("all")) + assert.NotEmpty(t, result.Err) + }) + + t.Run("the input carries the cluster, the failing roots and the built-in verdict of the row", func(t *testing.T) { + var input map[string]any + script := newScript(t, "all", 0, v1alpha1.InsightTarget{}, "all") + evaluator, _ := setup(t, map[string]func(map[string]any) (any, error){ + "all": func(in map[string]any) (any, error) { input = in; return nil, nil }, + }, script) + api := root("Deployment", "apps", "api", map[string]string{"app": "api"}) + api.DesiredReplicas, api.ReadyReplicas = new(int64(3)), new(int64(1)) + api.ServiceID = "svc-1" + api.Descendants = map[string]insights.Descendants{"Pod": {Failed: 2}} + + evaluator.Overrides(context.Background(), []insights.Root{api, root("DaemonSet", "kube-system", "aws-node", nil)}, insightpriority.Cluster{Nodes: 12, Namespaces: 41}, now) + + assert.Equal(t, "2026-10-09T12:00:00Z", input["now"]) + cluster := input["cluster"].(map[string]any) + assert.EqualValues(t, 12, cluster["nodes"]) + assert.Len(t, cluster["failing_roots"], 2) + + rows := input["rows"].([]any) + require.Len(t, rows, 2) + row := rows[0].(map[string]any) + assert.Equal(t, "api", row["name"]) + assert.Equal(t, true, row["managed"]) + assert.EqualValues(t, 3, row["desired_replicas"]) + assert.EqualValues(t, 1, row["ready_replicas"]) + assert.Equal(t, "LOW", row["builtin_priority"]) + assert.Equal(t, map[string]any{"app": "api"}, row["labels"]) + assert.Equal(t, map[string]any{"Pod": map[string]any{"failed": float64(2), "pending": float64(0)}}, row["descendants"]) + assert.NotContains(t, row, "annotations") + }) + + t.Run("the status is only touched when the outcome changes", func(t *testing.T) { + script := newScript(t, "all", 0, v1alpha1.InsightTarget{}, "all") + evaluator, registry := setup(t, map[string]func(map[string]any) (any, error){ + "all": func(map[string]any) (any, error) { return map[string]any{"a": critical}, nil }, + }, script) + roots := []insights.Root{root("Deployment", "apps", "a", nil)} + + evaluator.Overrides(context.Background(), roots, insightpriority.Cluster{}, now) + require.Len(t, registry.Events(), 1) + <-registry.Events() + + evaluator.Overrides(context.Background(), roots, insightpriority.Cluster{}, now.Add(2*time.Minute)) + assert.Empty(t, registry.Events(), "same outcome, nothing to report") + result, _ := registry.Result(key("all")) + assert.Equal(t, now, result.Time, "the time stays at the last change") + + evaluator.Overrides(context.Background(), append(roots, root("Deployment", "apps", "b", nil)), insightpriority.Cluster{}, now.Add(4*time.Minute)) + assert.Len(t, registry.Events(), 1) + }) +} + +func TestRegistryResults(t *testing.T) { + cluster := insightpriority.Cluster{} + roots := []insights.Root{root("Deployment", "apps", "a", nil)} + + t.Run("a recovered cluster clears the previous result", func(t *testing.T) { + script := newScript(t, "failing", 0, v1alpha1.InsightTarget{}, "failing") + evaluator, registry := setup(t, map[string]func(map[string]any) (any, error){ + "failing": func(map[string]any) (any, error) { return nil, errors.New("boom") }, + }, script) + + evaluator.Overrides(context.Background(), roots, cluster, now) + result, _ := registry.Result(key("failing")) + require.NotEmpty(t, result.Err) + + evaluator.Overrides(context.Background(), nil, cluster, now.Add(time.Minute)) + + result, ok := registry.Result(key("failing")) + require.True(t, ok) + assert.Empty(t, result.Err) + assert.Zero(t, result.Matched) + }) + + t.Run("a result of a replaced script is dropped", func(t *testing.T) { + script := newScript(t, "edited", 0, v1alpha1.InsightTarget{}, "edited") + script.Generation = 1 + var registry *insightpriority.Registry + evaluator, registry := setup(t, map[string]func(map[string]any) (any, error){ + "edited": func(map[string]any) (any, error) { + // the script is edited while it is running + replaced := script + replaced.Generation = 2 + registry.Set(replaced) + return map[string]any{}, nil + }, + }, script) + + evaluator.Overrides(context.Background(), roots, cluster, now) + + _, ok := registry.Result(key("edited")) + assert.False(t, ok, "the old generation must not show up as the result of the new one") + assert.Empty(t, registry.Events()) + }) + + t.Run("an event that did not fit is sent in a later round", func(t *testing.T) { + const scripts = 130 // more than the events channel holds + registered := make([]insightpriority.Script, 0, scripts) + for i := range scripts { + registered = append(registered, newScript(t, fmt.Sprintf("s%03d", i), 0, v1alpha1.InsightTarget{}, "all")) + } + evaluator, registry := setup(t, map[string]func(map[string]any) (any, error){ + "all": func(map[string]any) (any, error) { return map[string]any{}, nil }, + }, registered...) + + evaluator.Overrides(context.Background(), roots, cluster, now) + sent := len(registry.Events()) + require.Less(t, sent, scripts) + for range sent { + <-registry.Events() + } + + evaluator.Overrides(context.Background(), roots, cluster, now.Add(time.Minute)) + + assert.Equal(t, scripts-sent, len(registry.Events()), "the outcomes did not change, only the unsent ones are reported") + }) +} + +func TestTargets(t *testing.T) { + r := root("Deployment", "apps", "api", map[string]string{"tier": "core", "env": "prod"}) + + for name, tc := range map[string]struct { + target v1alpha1.InsightTarget + want bool + }{ + "empty target": {v1alpha1.InsightTarget{}, true}, + "kind": {v1alpha1.InsightTarget{Kinds: []v1alpha1.InsightTargetKind{{Group: "apps", Kind: "Deployment"}}}, true}, + "kind with version": {v1alpha1.InsightTarget{Kinds: []v1alpha1.InsightTargetKind{{Group: "apps", Version: "v1", Kind: "Deployment"}}}, true}, + "other version": {v1alpha1.InsightTarget{Kinds: []v1alpha1.InsightTargetKind{{Group: "apps", Version: "v2", Kind: "Deployment"}}}, false}, + "other group": {v1alpha1.InsightTarget{Kinds: []v1alpha1.InsightTargetKind{{Group: "", Kind: "Deployment"}}}, false}, + "one of several kinds": {v1alpha1.InsightTarget{Kinds: []v1alpha1.InsightTargetKind{{Group: "apps", Kind: "StatefulSet"}, {Group: "apps", Kind: "Deployment"}}}, true}, + "namespace": {v1alpha1.InsightTarget{Namespaces: []string{"x", "apps"}}, true}, + "other namespace": {v1alpha1.InsightTarget{Namespaces: []string{"x"}}, false}, + "selector": {v1alpha1.InsightTarget{LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"tier": "core"}}}, true}, + "selector expression": {v1alpha1.InsightTarget{LabelSelector: &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{{Key: "env", Operator: metav1.LabelSelectorOpIn, Values: []string{"dev"}}}}}, false}, + "every field has to match": {v1alpha1.InsightTarget{Namespaces: []string{"apps"}, LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"tier": "edge"}}}, false}, + "cluster scoped only if open": {v1alpha1.InsightTarget{Namespaces: []string{"apps"}}, true}, + } { + t.Run(name, func(t *testing.T) { + script := newScript(t, "t", 0, tc.target, "") + assert.Equal(t, tc.want, script.Matches(r)) + }) + } + + t.Run("an invalid selector is rejected", func(t *testing.T) { + _, err := insightpriority.NewScript(&v1alpha1.CustomInsightPriority{Spec: v1alpha1.CustomInsightPrioritySpec{ + Target: v1alpha1.InsightTarget{LabelSelector: &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{{Key: "a", Operator: "Nope"}}}}, + }}) + require.Error(t, err) + }) + + t.Run("cluster scoped objects only match an open namespace target", func(t *testing.T) { + node := root("Node", "", "node-1", nil) + assert.True(t, newScript(t, "t", 0, v1alpha1.InsightTarget{}, "").Matches(node)) + assert.False(t, newScript(t, "t", 0, v1alpha1.InsightTarget{Namespaces: []string{"apps"}}, "").Matches(node)) + }) +} diff --git a/go/deployment-operator/pkg/insightpriority/export_test.go b/go/deployment-operator/pkg/insightpriority/export_test.go new file mode 100644 index 0000000000..882750a366 --- /dev/null +++ b/go/deployment-operator/pkg/insightpriority/export_test.go @@ -0,0 +1,6 @@ +package insightpriority + +// SetRunFunc replaces how scripts are run, so evaluator tests do not need the Python runtime. +func SetRunFunc(e *Evaluator, run RunFunc) { + e.run = run +} diff --git a/go/deployment-operator/pkg/insightpriority/integration_test.go b/go/deployment-operator/pkg/insightpriority/integration_test.go new file mode 100644 index 0000000000..c7c5828407 --- /dev/null +++ b/go/deployment-operator/pkg/insightpriority/integration_test.go @@ -0,0 +1,91 @@ +package insightpriority_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/pluralsh/console/go/deployment-operator/api/v1alpha1" + "github.com/pluralsh/console/go/deployment-operator/pkg/insightpriority" + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/api" + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/insights" + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/store" +) + +func failingWorkload(kind, namespace, name string, replicas int64, labels map[string]string) unstructured.Unstructured { + obj := unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "apps/v1", "kind": kind, + "metadata": map[string]any{"name": name, "namespace": namespace, "uid": name, "generation": int64(1)}, + "spec": map[string]any{"replicas": replicas}, + "status": map[string]any{ + "observedGeneration": int64(1), "readyReplicas": int64(0), + "conditions": []any{map[string]any{"type": "Progressing", "status": "False", "reason": "ProgressDeadlineExceeded", "lastTransitionTime": time.Now().Add(-time.Hour).UTC().Format(time.RFC3339)}}, + }, + }} + obj.SetLabels(labels) + return obj +} + +// A real store, the real Python runtime and the real rollup, as a ping runs them. +func TestScriptsDecideThePriorityOfRealRoots(t *testing.T) { + ctx := context.Background() + s, err := store.NewDatabaseStore(ctx, store.WithStorage(api.StorageFile)) + require.NoError(t, err) + t.Cleanup(func() { _ = s.Shutdown() }) + + require.NoError(t, s.SaveComponents([]unstructured.Unstructured{ + failingWorkload("Deployment", "payments", "checkout", 3, map[string]string{"tier": "core"}), + failingWorkload("Deployment", "payments", "batch", 1, map[string]string{"tier": "batch"}), + failingWorkload("Deployment", "sandbox", "playground", 1, nil), + })) + + registry := insightpriority.NewRegistry() + script, err := insightpriority.NewScript(&v1alpha1.CustomInsightPriority{ + ObjectMeta: metav1.ObjectMeta{Namespace: "plrl-deploy-operator", Name: "payments"}, + Spec: v1alpha1.CustomInsightPrioritySpec{ + Target: v1alpha1.InsightTarget{Namespaces: []string{"payments", "sandbox"}}, + Script: ` +def evaluate(input): + out = {} + for r in input["rows"]: + if r["namespace"] == "sandbox": + out[r["uid"]] = {"priority": "IGNORE", "reason": "sandbox"} + elif r["labels"].get("tier") == "core" and r["ready_replicas"] == 0: + out[r["uid"]] = {"priority": "CRITICAL", "reason": "core service is down"} + return out +`, + }, + }) + require.NoError(t, err) + require.NoError(t, insightpriority.Compile(ctx, script.Source)) + registry.Set(script) + + roots, err := s.GetFailingRoots() + require.NoError(t, err) + require.Len(t, roots, 3) + + now := time.Now() + withScripts := insights.Summarize(roots, insightpriority.NewEvaluator(registry).Overrides(ctx, roots, insightpriority.Cluster{}, now), now) + builtin := insights.Summarize(roots, nil, now) + + priorities := map[string]insights.Priority{} + for _, e := range withScripts.Entries { + priorities[e.Name] = e.Priority + } + assert.Equal(t, map[string]insights.Priority{"checkout": insights.PriorityCritical, "batch": insights.PriorityMedium}, priorities, + "sandbox is ignored and batch keeps the built-in priority, a full outage of a LOW workload") + assert.Less(t, withScripts.HealthScore, builtin.HealthScore, "the critical override costs more than the ignored sandbox saves") + assert.Equal(t, int64(100-50-8), withScripts.HealthScore) + + result, ok := registry.Result(script.Key) + require.True(t, ok) + assert.Empty(t, result.Err) + assert.Equal(t, 3, result.Matched) + assert.Equal(t, 1, result.Overridden) + assert.Equal(t, 1, result.Ignored) +} diff --git a/go/deployment-operator/pkg/insightpriority/registry.go b/go/deployment-operator/pkg/insightpriority/registry.go new file mode 100644 index 0000000000..600b7ab029 --- /dev/null +++ b/go/deployment-operator/pkg/insightpriority/registry.go @@ -0,0 +1,149 @@ +package insightpriority + +import ( + "slices" + "sync" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/event" + + "github.com/pluralsh/console/go/deployment-operator/api/v1alpha1" +) + +// Result is the outcome of the last evaluation of a script. +type Result struct { + // Generation of the script that produced the result, a result of another generation is not recorded. + Generation int64 + Time time.Time + Matched int + Overridden int + Ignored int + Duration time.Duration + // Err is why the script failed, its objects then keep their built-in priority. + Err string + // LostTo are the scripts that took objects away from this one because of their precedence. + LostTo []types.NamespacedName +} + +// changed reports whether the result differs from the previous one in anything worth a status update. +func (r Result) changed(previous Result) bool { + return r.Matched != previous.Matched || r.Overridden != previous.Overridden || r.Ignored != previous.Ignored || + r.Err != previous.Err || !slices.Equal(r.LostTo, previous.LostTo) +} + +// Registry holds the scripts that compiled, and the results of their evaluations. +// It is safe for concurrent use. +type Registry struct { + mu sync.RWMutex + scripts map[types.NamespacedName]Script + results map[types.NamespacedName]Result + // unsent are the scripts whose result changed and whose event did not fit in the channel, they are sent again. + unsent map[types.NamespacedName]struct{} + events chan event.GenericEvent +} + +func NewRegistry() *Registry { + return &Registry{ + scripts: make(map[types.NamespacedName]Script), + results: make(map[types.NamespacedName]Result), + unsent: make(map[types.NamespacedName]struct{}), + events: make(chan event.GenericEvent, 128), + } +} + +// Events emits the scripts whose result changed, to be reconciled so their status follows. +func (r *Registry) Events() <-chan event.GenericEvent { + return r.events +} + +// Set registers a script, replacing the one with the same key. The result of a previous version of the script is dropped. +func (r *Registry) Set(script Script) { + r.mu.Lock() + defer r.mu.Unlock() + if previous, ok := r.scripts[script.Key]; ok && previous.Generation != script.Generation { + delete(r.results, script.Key) + } + r.scripts[script.Key] = script +} + +// Remove forgets a script and its result. +func (r *Registry) Remove(key types.NamespacedName) { + r.mu.Lock() + defer r.mu.Unlock() + delete(r.scripts, key) + delete(r.results, key) + delete(r.unsent, key) +} + +// Scripts returns the registered scripts, the ones that own contested objects first. +func (r *Registry) Scripts() []Script { + r.mu.RLock() + scripts := make([]Script, 0, len(r.scripts)) + for _, s := range r.scripts { + scripts = append(scripts, s) + } + r.mu.RUnlock() + + slices.SortFunc(scripts, func(a, b Script) int { + switch { + case before(a, b): + return -1 + case before(b, a): + return 1 + default: + return 0 + } + }) + return scripts +} + +// Result returns the result of the last evaluation of a script. +func (r *Registry) Result(key types.NamespacedName) (Result, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + result, ok := r.results[key] + return result, ok +} + +// record stores the results of an evaluation round and tells the controller about the ones that changed. +// A result of a script that was replaced while it was evaluated is dropped, the next round evaluates the new one. +func (r *Registry) record(results map[types.NamespacedName]Result) { + r.mu.Lock() + defer r.mu.Unlock() + + for key, result := range results { + script, registered := r.scripts[key] + if !registered || script.Generation != result.Generation { + continue + } + + previous, seen := r.results[key] + if seen && !result.changed(previous) { + result.Time = previous.Time + r.results[key] = result + continue + } + + r.results[key] = result + r.unsent[key] = struct{}{} + } + + r.flush() +} + +// flush sends the events that are pending, the ones that do not fit stay pending for the next round. +// The controller reads the latest result from the registry, so one event per script is all it needs. +func (r *Registry) flush() { + for key := range r.unsent { + select { + case r.events <- event.GenericEvent{Object: &v1alpha1.CustomInsightPriority{ + ObjectMeta: metav1.ObjectMeta{Namespace: key.Namespace, Name: key.Name}, + }}: + delete(r.unsent, key) + default: + return // the controller is behind, the channel is full for everything else too + } + } +} diff --git a/go/deployment-operator/pkg/insightpriority/script.go b/go/deployment-operator/pkg/insightpriority/script.go new file mode 100644 index 0000000000..5b53262775 --- /dev/null +++ b/go/deployment-operator/pkg/insightpriority/script.go @@ -0,0 +1,159 @@ +// Package insightpriority evaluates the CustomInsightPriority scripts that override the built-in priority of failing +// objects in the cluster insights and the cluster health score. +// +// Scripts are sandboxed Python run by Monty. Unlike the Helm templating runtime they get no host callbacks, no OS access +// and no clock: everything a script can see is in its input, so the same input always gives the same priorities. +package insightpriority + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + "unicode/utf8" + + monty "github.com/ewhauser/gomonty" +) + +const ( + // scriptName is shown in the errors of scripts. + scriptName = "insight-priority.py" + + // EvaluationTimeout bounds one evaluation of a script. + EvaluationTimeout = 2 * time.Second + + maxMemoryBytes = 64 << 20 + maxRecursion = 100 + + // chunkSize keeps every line of the input literal short, Monty reports columns as 16 bit numbers and a single + // line that is too long makes it panic. + chunkSize = 2000 +) + +// ErrInvalidScript means the script can not be compiled, or does not define evaluate. +var ErrInvalidScript = errors.New("invalid script") + +func limits() *monty.ResourceLimits { + return &monty.ResourceLimits{ + MaxDuration: EvaluationTimeout, + MaxMemory: maxMemoryBytes, + MaxRecursionDepth: maxRecursion, + } +} + +// Compile loads the script and checks that it defines evaluate. It does not call it. +func Compile(ctx context.Context, script string) (err error) { + defer recoverFFI(&err) + + ctx, cancel := context.WithTimeout(ctx, EvaluationTimeout) + defer cancel() + + repl, err := monty.NewRepl(monty.ReplOptions{ScriptName: scriptName, Limits: limits()}) + if err != nil { + return fmt.Errorf("create Python REPL: %w", err) + } + + if _, err := repl.FeedRun(ctx, script, monty.FeedOptions{}); err != nil { + return fmt.Errorf("%w: %s", ErrInvalidScript, describe(ctx, err)) + } + + defined, err := repl.FeedRun(ctx, "type(evaluate).__name__", monty.FeedOptions{}) + if err != nil || defined.Raw() != "function" { + return fmt.Errorf("%w: the script must define evaluate(input)", ErrInvalidScript) + } + + return nil +} + +// Run calls evaluate with the input and returns what it returned, as JSON. +func Run(ctx context.Context, script string, input any) (_ json.RawMessage, err error) { + defer recoverFFI(&err) + + ctx, cancel := context.WithTimeout(ctx, EvaluationTimeout) + defer cancel() + + encoded, err := json.Marshal(input) + if err != nil { + return nil, fmt.Errorf("serialize input: %w", err) + } + + var chunks strings.Builder + for len(encoded) > 0 { + n := min(len(encoded), chunkSize) + for n < len(encoded) && !utf8.RuneStart(encoded[n]) { + n-- // never cut a character in half + } + quoted, _ := json.Marshal(string(encoded[:n])) + chunks.Write(quoted) + chunks.WriteString(",\n") + encoded = encoded[n:] + } + + repl, err := monty.NewRepl(monty.ReplOptions{ScriptName: scriptName, Limits: limits()}) + if err != nil { + return nil, fmt.Errorf("create Python REPL: %w", err) + } + + preamble := "import json as __json\n__input = __json.loads(''.join([\n" + chunks.String() + "]))\n" + for _, source := range []string{preamble, script} { + if _, err := repl.FeedRun(ctx, source, monty.FeedOptions{}); err != nil { + return nil, errors.New(describe(ctx, err)) + } + } + + result, err := repl.FeedRun(ctx, "__json.dumps(evaluate(__input))", monty.FeedOptions{}) + if err != nil { + return nil, errors.New(describe(ctx, err)) + } + + raw, ok := result.Raw().(string) + if !ok { + return nil, errors.New("evaluate must return a dictionary") + } + + return json.RawMessage(raw), nil +} + +// recoverFFI turns a panic of the native runtime into an error, a broken script must never take the agent down. +func recoverFFI(err *error) { + if r := recover(); r != nil { + *err = fmt.Errorf("python runtime failed: %v", r) + } +} + +// describe renders an execution error without the input values that may be echoed in it. +func describe(ctx context.Context, err error) string { + if ctxErr := ctx.Err(); ctxErr != nil { + if errors.Is(ctxErr, context.DeadlineExceeded) { + return fmt.Sprintf("timed out after %s", EvaluationTimeout) + } + return ctxErr.Error() + } + + var syntax *monty.SyntaxError + if errors.As(err, &syntax) { + return "SyntaxError: " + firstLine(syntax.Error()) + } + + var runtimeErr *monty.RuntimeError + if errors.As(err, &runtimeErr) { + // Only the exception type: the message is script controlled and could carry the input of other namespaces + // into a status that namespace users can read. + message, _, _ := strings.Cut(firstLine(runtimeErr.Error()), ":") + message = strings.TrimSpace(message) + if len(runtimeErr.Frames) > 0 { + frame := runtimeErr.Frames[0] + return fmt.Sprintf("%s (line %d)", message, frame.Line) + } + return message + } + + return firstLine(err.Error()) +} + +func firstLine(s string) string { + line, _, _ := strings.Cut(s, "\n") + return line +} diff --git a/go/deployment-operator/pkg/insightpriority/script_test.go b/go/deployment-operator/pkg/insightpriority/script_test.go new file mode 100644 index 0000000000..fb06538901 --- /dev/null +++ b/go/deployment-operator/pkg/insightpriority/script_test.go @@ -0,0 +1,138 @@ +package insightpriority_test + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/pluralsh/console/go/deployment-operator/pkg/insightpriority" +) + +func TestCompile(t *testing.T) { + ctx := context.Background() + + t.Run("accepts a script that defines evaluate", func(t *testing.T) { + require.NoError(t, insightpriority.Compile(ctx, "def evaluate(input):\n return {}\n")) + }) + + t.Run("rejects a syntax error", func(t *testing.T) { + err := insightpriority.Compile(ctx, "def evaluate(input:\n return {}\n") + require.ErrorIs(t, err, insightpriority.ErrInvalidScript) + }) + + t.Run("rejects a script without evaluate", func(t *testing.T) { + err := insightpriority.Compile(ctx, "x = 1\n") + require.ErrorIs(t, err, insightpriority.ErrInvalidScript) + }) + + t.Run("rejects a script that fails while loading", func(t *testing.T) { + err := insightpriority.Compile(ctx, "raise ValueError('boom')\ndef evaluate(input):\n return {}\n") + require.ErrorIs(t, err, insightpriority.ErrInvalidScript) + }) +} + +func TestRun(t *testing.T) { + ctx := context.Background() + input := map[string]any{"rows": []any{map[string]any{"uid": "a", "ready": 0}, map[string]any{"uid": "b", "ready": 2}}} + + t.Run("calls evaluate with the input", func(t *testing.T) { + raw, err := insightpriority.Run(ctx, ` +def evaluate(input): + return {r["uid"]: {"priority": "CRITICAL"} for r in input["rows"] if r["ready"] == 0} +`, input) + + require.NoError(t, err) + assert.JSONEq(t, `{"a": {"priority": "CRITICAL"}}`, string(raw)) + }) + + t.Run("accepts an input that is much longer than a line", func(t *testing.T) { + rows := make([]any, 600) + for i := range rows { + rows[i] = map[string]any{"uid": fmt.Sprintf("uid-%d", i), "message": strings.Repeat("x", 100)} + } + + raw, err := insightpriority.Run(ctx, "def evaluate(input):\n return {'count': len(input['rows'])}\n", map[string]any{"rows": rows}) + + require.NoError(t, err) + assert.JSONEq(t, `{"count": 600}`, string(raw)) + }) + + t.Run("keeps non-ASCII text intact when the input is split into lines", func(t *testing.T) { + message := strings.Repeat("Å‚", 1500) // 2 bytes each, so a chunk boundary falls inside a character + raw, err := insightpriority.Run(ctx, "def evaluate(input):\n return {'len': len(input['m'])}\n", map[string]any{"m": message}) + + require.NoError(t, err) + assert.JSONEq(t, `{"len": 1500}`, string(raw)) + }) + + t.Run("does not repeat the message of an exception", func(t *testing.T) { + _, err := insightpriority.Run(ctx, "def evaluate(input):\n raise ValueError('secret-from-another-namespace')\n", input) + + require.Error(t, err) + assert.Contains(t, err.Error(), "ValueError") + assert.NotContains(t, err.Error(), "secret-from-another-namespace") + }) + + t.Run("is deterministic", func(t *testing.T) { + script := "def evaluate(input):\n return {r['uid']: {'priority': 'LOW'} for r in input['rows']}\n" + first, err := insightpriority.Run(ctx, script, input) + require.NoError(t, err) + + for range 5 { + again, err := insightpriority.Run(ctx, script, input) + require.NoError(t, err) + assert.JSONEq(t, string(first), string(again)) + } + }) + + t.Run("returns null for a script that returns nothing", func(t *testing.T) { + raw, err := insightpriority.Run(ctx, "def evaluate(input):\n return None\n", input) + + require.NoError(t, err) + assert.Equal(t, "null", string(raw)) + }) + + t.Run("reports an exception without the input", func(t *testing.T) { + _, err := insightpriority.Run(ctx, "def evaluate(input):\n return input['rows'][0]['missing']\n", input) + + require.Error(t, err) + assert.Contains(t, err.Error(), "KeyError") + assert.NotContains(t, err.Error(), `"uid"`) + }) + + t.Run("stops a script that never ends", func(t *testing.T) { + cancelled, cancel := context.WithTimeout(ctx, 300_000_000) + defer cancel() + + _, err := insightpriority.Run(cancelled, "def evaluate(input):\n while True:\n pass\n", input) + + require.Error(t, err) + }) + + t.Run("has no access to the host", func(t *testing.T) { + for name, script := range map[string]string{ + "open": "def evaluate(input):\n return open('/etc/passwd').read()\n", + "os": "import os\ndef evaluate(input):\n return os.environ\n", + "socket": "import socket\ndef evaluate(input):\n return {}\n", + } { + t.Run(name, func(t *testing.T) { + _, err := insightpriority.Run(ctx, script, input) + require.Error(t, err) + }) + } + }) +} + +func TestRunJSONShape(t *testing.T) { + raw, err := insightpriority.Run(context.Background(), "def evaluate(input):\n return {'a': {'priority': 'HIGH', 'reason': 'because'}}\n", map[string]any{}) + require.NoError(t, err) + + var decoded map[string]map[string]string + require.NoError(t, json.Unmarshal(raw, &decoded)) + assert.Equal(t, "because", decoded["a"]["reason"]) +} diff --git a/go/deployment-operator/pkg/insightpriority/target.go b/go/deployment-operator/pkg/insightpriority/target.go new file mode 100644 index 0000000000..d98abd7650 --- /dev/null +++ b/go/deployment-operator/pkg/insightpriority/target.go @@ -0,0 +1,86 @@ +package insightpriority + +import ( + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + + "github.com/pluralsh/console/go/deployment-operator/api/v1alpha1" + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/insights" +) + +// Script is a validated CustomInsightPriority. +type Script struct { + Key types.NamespacedName + Precedence int + Source string + // Generation of the CustomInsightPriority, a result of an older generation is stale. + Generation int64 + + kinds []v1alpha1.InsightTargetKind + namespaces map[string]struct{} + selector labels.Selector +} + +// NewScript validates the target of a CustomInsightPriority. The script itself is checked by Compile. +func NewScript(cr *v1alpha1.CustomInsightPriority) (Script, error) { + script := Script{ + Key: types.NamespacedName{Namespace: cr.Namespace, Name: cr.Name}, + Precedence: cr.Spec.Precedence, + Source: cr.Spec.Script, + Generation: cr.Generation, + kinds: cr.Spec.Target.Kinds, + namespaces: make(map[string]struct{}, len(cr.Spec.Target.Namespaces)), + } + + for _, namespace := range cr.Spec.Target.Namespaces { + script.namespaces[namespace] = struct{}{} + } + + if cr.Spec.Target.LabelSelector != nil { + selector, err := metav1.LabelSelectorAsSelector(cr.Spec.Target.LabelSelector) + if err != nil { + return Script{}, fmt.Errorf("invalid label selector: %w", err) + } + script.selector = selector + } + + return script, nil +} + +// Matches reports whether the script is responsible for the root: it has to match every field of the target that is set. +func (s Script) Matches(r insights.Root) bool { + if len(s.kinds) > 0 && !s.matchesKind(r) { + return false + } + + if len(s.namespaces) > 0 { + if _, ok := s.namespaces[r.Namespace]; !ok { + return false + } + } + + return s.selector == nil || s.selector.Matches(labels.Set(r.Labels)) +} + +func (s Script) matchesKind(r insights.Root) bool { + for _, k := range s.kinds { + if k.Kind == r.Kind && k.Group == r.Group && (k.Version == "" || k.Version == r.Version) { + return true + } + } + return false +} + +// before orders scripts by who owns an object several of them match: the highest precedence, then the name. +func before(a, b Script) bool { + if a.Precedence != b.Precedence { + return a.Precedence > b.Precedence + } + if a.Key.Namespace != b.Key.Namespace { + return a.Key.Namespace < b.Key.Namespace + } + return a.Key.Name < b.Key.Name +} diff --git a/go/deployment-operator/pkg/ping/build.go b/go/deployment-operator/pkg/ping/build.go index 6fbe69d51f..bdb682034b 100644 --- a/go/deployment-operator/pkg/ping/build.go +++ b/go/deployment-operator/pkg/ping/build.go @@ -1,30 +1,44 @@ package ping import ( + "context" + "fmt" "strings" + "time" console "github.com/pluralsh/console/go/client" "github.com/samber/lo" "k8s.io/apimachinery/pkg/version" "k8s.io/klog/v2" + "github.com/pluralsh/console/go/deployment-operator/pkg/insightpriority" "github.com/pluralsh/console/go/deployment-operator/pkg/scraper" + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/insights" ) -func (p *Pinger) pingAttributes(info *version.Info, pods []string, minKubeletVersion, openShiftVersion *string, podCount *int64) console.ClusterPing { - hs, err := p.store.GetHealthScore() +func (p *Pinger) pingAttributes(info *version.Info, pods []string, minKubeletVersion, openShiftVersion *string, podCount *int64) (console.ClusterPing, error) { + nodCount, namespaceCount, err := p.store.GetComponentCounts() if err != nil { - klog.ErrorS(err, "failed to get health score") + klog.ErrorS(err, "failed to get cluster component counts") } - ns, err := p.store.GetNodeStatistics() + // The health score and the insights are computed from the same failing roots, so they always agree. + now := time.Now() + roots, err := p.store.GetFailingRoots() if err != nil { - klog.ErrorS(err, "failed to get node statistics") + // An empty list would report a healthy cluster and make Console delete all its insights, so skip this ping. + return console.ClusterPing{}, fmt.Errorf("failed to get failing components: %w", err) + } + var overrides map[string]insights.Override + if p.priorities != nil { + overrides = p.priorities.Overrides(context.Background(), roots, insightpriority.Cluster{Nodes: nodCount, Namespaces: namespaceCount}, now) } + summary := insights.Summarize(roots, overrides, now) + hs := summary.HealthScore - nodCount, namespaceCount, err := p.store.GetComponentCounts() + ns, err := p.store.GetNodeStatistics() if err != nil { - klog.ErrorS(err, "failed to get cluster component counts") + klog.ErrorS(err, "failed to get node statistics") } vs := strings.Split(info.GitVersion, "-") @@ -47,12 +61,7 @@ func (p *Pinger) pingAttributes(info *version.Info, pods []string, minKubeletVer OpenshiftVersion: openShiftVersion, } - cInsights, err := p.store.GetComponentInsights() - if err != nil { - klog.ErrorS(err, "failed to get component insights") - } - - cp.InsightComponents = lo.ToSlicePtr(cInsights) + cp.InsightComponents = lo.ToSlicePtr(insights.Attributes(p.grace.apply(summary, p.store, now))) if metrics.CPUAvailableMillicores > 0 { cp.CPUTotal = lo.ToPtr(float64(metrics.CPUAvailableMillicores)) } @@ -66,5 +75,5 @@ func (p *Pinger) pingAttributes(info *version.Info, pods []string, minKubeletVer cp.MemoryUtil = lo.ToPtr(float64(metrics.MemoryUsedPercentage)) } - return cp + return cp, nil } diff --git a/go/deployment-operator/pkg/ping/cluster.go b/go/deployment-operator/pkg/ping/cluster.go index 745bcc0f1c..e69a026332 100644 --- a/go/deployment-operator/pkg/ping/cluster.go +++ b/go/deployment-operator/pkg/ping/cluster.go @@ -6,33 +6,45 @@ import ( "sort" "time" + "github.com/Masterminds/semver/v3" + "github.com/samber/lo" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" + console "github.com/pluralsh/console/go/client" "github.com/pluralsh/console/go/deployment-operator/internal/helpers" + "github.com/pluralsh/console/go/deployment-operator/pkg/client" "github.com/pluralsh/console/go/deployment-operator/pkg/common" - - "github.com/Masterminds/semver/v3" - "github.com/pluralsh/console/go/polly/containers" - "github.com/samber/lo" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - internallog "github.com/pluralsh/console/go/deployment-operator/pkg/log" + "github.com/pluralsh/console/go/polly/containers" ) -func RunClusterPingerInBackgroundOrDie(ctx context.Context, pinger *Pinger, duration time.Duration) { +// RunClusterPingerInBackgroundOrDie pings the cluster every interval. The first ping waits for ready, which should return +// once the store holds the state of the cluster: a ping built from a partially loaded store would report healthy +// workloads as missing and drop their insights in Console. Nothing else is held back by it. +func RunClusterPingerInBackgroundOrDie(ctx context.Context, pinger *Pinger, duration time.Duration, ready func(context.Context) error) { interval := func() time.Duration { return clusterPingInterval(duration) } - _ = helpers.DynamicBackgroundPollUntilContextCancel(ctx, interval, false, func(_ context.Context) (done bool, err error) { - if err := pinger.PingCluster(); err != nil { - klog.ErrorS(err, "failed ping cluster") + go func() { + start := time.Now() + if err := ready(ctx); err != nil { + klog.V(internallog.LogLevelDefault).InfoS("cluster pinger stopped before the store was ready", "reason", err) + return } - return false, nil - }) + klog.V(internallog.LogLevelDefault).InfoS("store is ready, starting cluster pinger", "waited", time.Since(start)) - klog.V(internallog.LogLevelDefault).InfoS("started cluster pinger", "interval", duration) + _ = helpers.DynamicBackgroundPollUntilContextCancel(ctx, interval, false, func(_ context.Context) (done bool, err error) { + if err := pinger.PingCluster(); err != nil { + klog.ErrorS(err, "failed ping cluster") + } + return false, nil + }) + + klog.V(internallog.LogLevelDefault).InfoS("started cluster pinger", "interval", duration) + }() } func clusterPingInterval(defaultDuration time.Duration) time.Duration { @@ -71,14 +83,12 @@ func (p *Pinger) PingCluster() error { } } - attrs := p.pingAttributes(info, podNames, minKubeletVersion, openShiftVersion, podCount) - attrs.AvailabilityZones = stabilize(azs) - if err := p.consoleClient.PingCluster(attrs); err != nil { - attrs.Distro = nil - return p.consoleClient.PingCluster(attrs) // fallback to no distro to support old console servers + attrs, err := p.pingAttributes(info, podNames, minKubeletVersion, openShiftVersion, podCount) + if err != nil { + return err } - - return nil + attrs.AvailabilityZones = stabilize(azs) + return pingWithFallback(p.consoleClient, attrs) } // kubeNodeData tries to scrape a minimum kubelet version across all nodes in the cluster. @@ -126,3 +136,17 @@ func stabilize(azs containers.Set[string]) []*string { sort.Strings(azsList) return lo.ToSlicePtr(azsList) } + +// pingWithFallback pings the cluster and, if that fails, once more without the fields that old Console servers do not +// know. They reject the whole ping for an unknown field, which would also lose the health score. +func pingWithFallback(c client.Client, attrs console.ClusterPing) error { + if err := c.PingCluster(attrs); err != nil { + attrs.Distro = nil + for _, component := range attrs.InsightComponents { + component.ServiceID = nil + } + return c.PingCluster(attrs) + } + + return nil +} diff --git a/go/deployment-operator/pkg/ping/insights.go b/go/deployment-operator/pkg/ping/insights.go new file mode 100644 index 0000000000..ce1e316458 --- /dev/null +++ b/go/deployment-operator/pkg/ping/insights.go @@ -0,0 +1,87 @@ +package ping + +import ( + "slices" + "strings" + "sync" + "time" + + "k8s.io/apimachinery/pkg/types" + + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/insights" + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/store" +) + +// recentRoot is a failing root that was reported and has not been seen failing since. +type recentRoot struct { + entry insights.Entry + lastSeen time.Time +} + +// graceWindow keeps reporting a root for a while after it stops failing, with the priority it had when it was last seen. +// +// Console deletes the insight of a component that is missing from a ping, and generating a new one costs an LLM call, +// so a workload that flaps, or recovers and fails again, would regenerate its insight on every flap. The window only +// affects the list, the health score reflects recoveries immediately. Roots that are ignored or deleted leave at once. +type graceWindow struct { + mu sync.Mutex + period time.Duration + recent map[string]recentRoot +} + +func newGraceWindow(period time.Duration) *graceWindow { + return &graceWindow{period: period, recent: make(map[string]recentRoot)} +} + +// apply returns the entries to report: the failing ones followed by the recently recovered ones. +func (g *graceWindow) apply(summary insights.Summary, db store.Store, now time.Time) []insights.Entry { + g.mu.Lock() + defer g.mu.Unlock() + + if g.period <= 0 { + clear(g.recent) + return summary.Entries + } + + reported := make(map[string]struct{}, len(summary.Entries)) + for _, e := range summary.Entries { + reported[e.UID] = struct{}{} + g.recent[e.UID] = recentRoot{entry: e, lastSeen: now} + } + + entries := summary.Entries + var recovered []insights.Entry + for uid, r := range g.recent { + if _, ok := reported[uid]; ok { + continue + } + + if _, ignored := summary.Ignored[uid]; ignored || now.Sub(r.lastSeen) >= g.period || !visible(db, uid) { + delete(g.recent, uid) + continue + } + + recovered = append(recovered, r.entry) + } + + // map order is random, and the cap must not drop a different entry on every ping + slices.SortFunc(recovered, func(a, b insights.Entry) int { + if ra, rb := insights.Rank(a.Priority), insights.Rank(b.Priority); ra != rb { + return rb - ra + } + if a.Score != b.Score { + return b.Score - a.Score + } + return strings.Compare(a.UID, b.UID) + }) + entries = append(entries[:len(entries):len(entries)], recovered...) + + return entries +} + +// visible tells whether a recovered root may still be reported: it is stored and it was not muted since. +// summary.Ignored only knows the roots that are failing, so the annotation of a recovered one is read from the store. +func visible(db store.Store, uid string) bool { + annotation, found, err := db.GetPriorityAnnotation(types.UID(uid)) + return err == nil && found && insights.Priority(annotation) != insights.PriorityIgnore +} diff --git a/go/deployment-operator/pkg/ping/insights_test.go b/go/deployment-operator/pkg/ping/insights_test.go new file mode 100644 index 0000000000..d2ff606f74 --- /dev/null +++ b/go/deployment-operator/pkg/ping/insights_test.go @@ -0,0 +1,163 @@ +package ping + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/types" + + console "github.com/pluralsh/console/go/client" + "github.com/pluralsh/console/go/deployment-operator/pkg/client" + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/insights" + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/store" +) + +// existingStore reports the given UIDs as existing components, with the priority annotation they are given in muted. +type existingStore struct { + store.Store + uids map[string]struct{} + muted map[string]string +} + +func (s existingStore) GetPriorityAnnotation(uid types.UID) (string, bool, error) { + _, ok := s.uids[string(uid)] + return s.muted[string(uid)], ok, nil +} + +func summaryOf(roots ...insights.Root) insights.Summary { + return insights.Summarize(roots, nil, time.Now()) +} + +func failing(name string, criticality int) insights.Root { + return insights.Root{UID: name, Kind: "Deployment", Name: name, Criticality: criticality} +} + +func names(entries []insights.Entry) []string { + result := make([]string, 0, len(entries)) + for _, e := range entries { + result = append(result, e.Name) + } + return result +} + +func TestGraceWindow(t *testing.T) { + now := time.Now() + db := existingStore{uids: map[string]struct{}{"api": {}, "worker": {}}} + + t.Run("keeps a recovered root with its last priority until the period ends", func(t *testing.T) { + g := newGraceWindow(10 * time.Minute) + g.apply(summaryOf(failing("api", insights.ThresholdCritical)), db, now) + + entries := g.apply(summaryOf(), db, now.Add(5*time.Minute)) + + require.Equal(t, []string{"api"}, names(entries)) + assert.Equal(t, insights.PriorityCritical, entries[0].Priority) + assert.Empty(t, g.apply(summaryOf(), db, now.Add(10*time.Minute))) + }) + + t.Run("restarts the period while the root keeps failing", func(t *testing.T) { + g := newGraceWindow(10 * time.Minute) + g.apply(summaryOf(failing("api", insights.ScoreCore)), db, now) + g.apply(summaryOf(failing("api", insights.ScoreCore)), db, now.Add(8*time.Minute)) + + assert.Equal(t, []string{"api"}, names(g.apply(summaryOf(), db, now.Add(15*time.Minute)))) + }) + + t.Run("reports a failing root once", func(t *testing.T) { + g := newGraceWindow(10 * time.Minute) + + entries := g.apply(summaryOf(failing("api", insights.ScoreCore)), db, now) + entries = g.apply(summaryOf(failing("api", insights.ScoreCore)), db, now.Add(time.Minute)) + + assert.Equal(t, []string{"api"}, names(entries)) + }) + + t.Run("drops a deleted root at once", func(t *testing.T) { + g := newGraceWindow(10 * time.Minute) + g.apply(summaryOf(failing("gone", insights.ScoreCore)), db, now) + + assert.Empty(t, g.apply(summaryOf(), db, now.Add(time.Minute))) + }) + + t.Run("drops a root that became ignored at once", func(t *testing.T) { + g := newGraceWindow(10 * time.Minute) + g.apply(summaryOf(failing("api", insights.ScoreCore)), db, now) + + ignored := failing("api", insights.ScoreCore) + ignored.PriorityAnnotation = insights.Ignore + + assert.Empty(t, g.apply(summaryOf(ignored), db, now.Add(time.Minute))) + assert.Empty(t, g.apply(summaryOf(), db, now.Add(2*time.Minute)), "and it stays dropped") + }) + + t.Run("drops a recovered root that was muted since", func(t *testing.T) { + muted := existingStore{uids: map[string]struct{}{"api": {}}, muted: map[string]string{"api": insights.Ignore}} + g := newGraceWindow(10 * time.Minute) + g.apply(summaryOf(failing("api", insights.ScoreCore)), muted, now) + + assert.Empty(t, g.apply(summaryOf(), muted, now.Add(time.Minute))) + }) + + t.Run("a zero period disables the window", func(t *testing.T) { + g := newGraceWindow(0) + g.apply(summaryOf(failing("api", insights.ScoreCore)), db, now) + + assert.Empty(t, g.apply(summaryOf(), db, now.Add(time.Second))) + }) +} + +type recordingConsole struct { + client.Client + pings []console.ClusterPing + // failFirst makes the first ping fail like an old Console server rejecting unknown fields. + failFirst bool +} + +func (r *recordingConsole) PingCluster(attrs console.ClusterPing) error { + serviceIDs := make([]*console.ClusterInsightComponentAttributes, len(attrs.InsightComponents)) + for i, c := range attrs.InsightComponents { + copied := *c + serviceIDs[i] = &copied + } + attrs.InsightComponents = serviceIDs + r.pings = append(r.pings, attrs) + if r.failFirst && len(r.pings) == 1 { + return errors.New("unknown field serviceId") + } + return nil +} + +func TestPingWithFallback(t *testing.T) { + attrs := func() console.ClusterPing { + return console.ClusterPing{ + Distro: new(console.ClusterDistroEks), + InsightComponents: []*console.ClusterInsightComponentAttributes{ + {Kind: "Deployment", Name: "api", ServiceID: new("svc-1")}, + }, + } + } + + t.Run("sends everything to a Console that understands it", func(t *testing.T) { + c := &recordingConsole{} + + require.NoError(t, pingWithFallback(c, attrs())) + + require.Len(t, c.pings, 1) + assert.Equal(t, "svc-1", *c.pings[0].InsightComponents[0].ServiceID) + }) + + t.Run("retries without the service ids and the distro when the ping is rejected", func(t *testing.T) { + c := &recordingConsole{failFirst: true} + + require.NoError(t, pingWithFallback(c, attrs())) + + require.Len(t, c.pings, 2) + assert.NotNil(t, c.pings[0].InsightComponents[0].ServiceID) + assert.Nil(t, c.pings[1].InsightComponents[0].ServiceID) + assert.Nil(t, c.pings[1].Distro) + assert.Equal(t, "api", c.pings[1].InsightComponents[0].Name, "the health score and insights are still sent") + }) +} diff --git a/go/deployment-operator/pkg/ping/pinger.go b/go/deployment-operator/pkg/ping/pinger.go index 7a35840c7d..c30be0d5e7 100644 --- a/go/deployment-operator/pkg/ping/pinger.go +++ b/go/deployment-operator/pkg/ping/pinger.go @@ -2,6 +2,7 @@ package ping import ( "fmt" + "time" apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/client-go/kubernetes" @@ -12,10 +13,14 @@ import ( "github.com/pluralsh/console/go/deployment-operator/internal/utils" discoverycache "github.com/pluralsh/console/go/deployment-operator/pkg/cache/discovery" "github.com/pluralsh/console/go/deployment-operator/pkg/client" + "github.com/pluralsh/console/go/deployment-operator/pkg/insightpriority" "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/store" "github.com/pluralsh/console/go/polly/containers" ) +// defaultInsightGracePeriod is how long a recovered component is still reported unless configured otherwise. +const defaultInsightGracePeriod = 10 * time.Minute + type Pinger struct { consoleClient client.Client discoveryCache discoverycache.Cache @@ -25,6 +30,25 @@ type Pinger struct { apiExtClient *apiextensionsclient.Clientset store store.Store supportedAddons containers.Set[string] + grace *graceWindow + priorities *insightpriority.Evaluator +} + +// Option configures a Pinger. +type Option func(*Pinger) + +// WithInsightPriorities lets CustomInsightPriority scripts decide the priority of failing components. +func WithInsightPriorities(evaluator *insightpriority.Evaluator) Option { + return func(p *Pinger) { + p.priorities = evaluator + } +} + +// WithInsightGracePeriod sets how long a recovered component is still reported as an insight, zero disables it. +func WithInsightGracePeriod(period time.Duration) Option { + return func(p *Pinger) { + p.grace = newGraceWindow(period) + } } var supported = []string{ @@ -61,15 +85,15 @@ var supported = []string{ "vector", } -func NewOrDie(console client.Client, config *rest.Config, k8sClient ctrclient.Client, discoveryCache discoverycache.Cache, store store.Store) *Pinger { - pinger, err := New(console, config, k8sClient, discoveryCache, store) +func NewOrDie(console client.Client, config *rest.Config, k8sClient ctrclient.Client, discoveryCache discoverycache.Cache, store store.Store, options ...Option) *Pinger { + pinger, err := New(console, config, k8sClient, discoveryCache, store, options...) if err != nil { panic(fmt.Errorf("failed to create Pinger: %w", err)) } return pinger } -func New(console client.Client, config *rest.Config, k8sClient ctrclient.Client, discoveryCache discoverycache.Cache, store store.Store) (*Pinger, error) { +func New(console client.Client, config *rest.Config, k8sClient ctrclient.Client, discoveryCache discoverycache.Cache, store store.Store, options ...Option) (*Pinger, error) { f := utils.NewFactory(config) cs, err := f.KubernetesClientSet() if err != nil { @@ -93,7 +117,7 @@ func New(console client.Client, config *rest.Config, k8sClient ctrclient.Client, } } - return &Pinger{ + pinger := &Pinger{ consoleClient: console, factory: f, k8sClient: k8sClient, @@ -102,5 +126,11 @@ func New(console client.Client, config *rest.Config, k8sClient ctrclient.Client, discoveryCache: discoveryCache, store: store, supportedAddons: supportedAddons, - }, nil + grace: newGraceWindow(defaultInsightGracePeriod), + } + for _, option := range options { + option(pinger) + } + + return pinger, nil } diff --git a/go/deployment-operator/pkg/streamline/insights/attributes.go b/go/deployment-operator/pkg/streamline/insights/attributes.go new file mode 100644 index 0000000000..ba3ee03587 --- /dev/null +++ b/go/deployment-operator/pkg/streamline/insights/attributes.go @@ -0,0 +1,27 @@ +package insights + +import ( + "github.com/pluralsh/console/go/client" + "github.com/samber/lo" +) + +// Attributes converts the entries to the insight components reported to Console. It returns nil when there are none. +func Attributes(entries []Entry) []client.ClusterInsightComponentAttributes { + if len(entries) == 0 { + return nil + } + + result := make([]client.ClusterInsightComponentAttributes, 0, len(entries)) + for _, e := range entries { + result = append(result, client.ClusterInsightComponentAttributes{ + Group: new(e.Group), + Version: e.Version, + Kind: e.Kind, + Namespace: lo.EmptyableToPtr(e.Namespace), + Name: e.Name, + Priority: new(client.InsightComponentPriority(e.Priority)), + ServiceID: lo.EmptyableToPtr(e.ServiceID), + }) + } + return result +} diff --git a/go/deployment-operator/pkg/streamline/insights/catalog.go b/go/deployment-operator/pkg/streamline/insights/catalog.go new file mode 100644 index 0000000000..81a8aeacbd --- /dev/null +++ b/go/deployment-operator/pkg/streamline/insights/catalog.go @@ -0,0 +1,99 @@ +package insights + +// Priority scores. A root failure is scored as criticality + severity and bucketed with the +// thresholds below. +// +// There are two classes of objects. Core components of the cluster start at ScoreCore, which is +// HIGH: a partial failure stays HIGH and only a failure that takes most of the component down +// (SeverityMax lifts it past ThresholdCritical) is CRITICAL. Everything else is an app, it starts +// at ScoreLow or ScoreMedium and is capped below ThresholdHigh, so a third-party workload can +// never be more than MEDIUM on its own. Users raise their own apps with the priority annotation +// or a CustomInsightPriority script, which do not go through that cap. +const ( + ScoreLow = 10 + ScoreMedium = 35 + ScoreCore = 60 + + // ThresholdMedium, ThresholdHigh and ThresholdCritical are the lowest scores of their buckets. + ThresholdMedium = 25 + ThresholdHigh = 50 + ThresholdCritical = 75 + + // SeverityMax is the most the replica ratio can add to the criticality. + SeverityMax = 20 +) + +// catalogEntry is a well-known cluster component whose failure has a cluster-wide blast radius. +type catalogEntry struct { + name string + score int + // keys are lowercase names without separators. A name matches when any contiguous run of its + // tokens joined without separators equals a key. + keys []string +} + +var catalog = []catalogEntry{ + {name: "dns", score: ScoreCore, keys: []string{"coredns", "kubedns"}}, + {name: "kube-proxy", score: ScoreCore, keys: []string{"kubeproxy"}}, + {name: "cni", score: ScoreCore, keys: []string{ + "awsnode", "amazonvpccni", "awscni", "cilium", "calico", "calicotypha", "flannel", "kubeflannel", "canal", "antrea", "weavenet", + }}, + {name: "csi", score: ScoreCore, keys: []string{ + "csinode", "csicontroller", "ebscsi", "efscsi", "gcepdcsi", "azurediskcsi", "azurefilecsi", + }}, + {name: "service mesh", score: ScoreCore, keys: []string{"istiod", "istio", "linkerd"}}, + {name: "cert-manager", score: ScoreCore, keys: []string{"certmanager"}}, + {name: "ingress controller", score: ScoreCore, keys: []string{ + "ingressnginx", "nginxingress", "traefik", "haproxyingress", "contour", "emissaryingress", "ambassador", "envoygateway", + }}, + {name: "metrics-server", score: ScoreCore, keys: []string{"metricsserver"}}, + {name: "autoscaler", score: ScoreCore, keys: []string{"clusterautoscaler", "karpenter"}}, +} + +// namespaceTiers hold the namespaces of core components by their separator-less, lowercase token runs. +var namespaceTiers = map[string]int{ + "kubesystem": ScoreCore, + "istiosystem": ScoreCore, + "linkerd": ScoreCore, + "certmanager": ScoreCore, + "ingressnginx": ScoreCore, + "traefik": ScoreCore, + "calicosystem": ScoreCore, + "tigeraoperator": ScoreCore, + "cilium": ScoreCore, + + // Plural itself: the Console and the agent that reports every cluster + "plrlconsole": ScoreCore, + "plrldeployoperator": ScoreCore, +} + +// kindScores are the criticality of a failing root when nothing more specific applies. Only a node +// is core by kind, every workload and its Ingress or Certificate is an app unless it is classified +// as core by name, namespace or priority class. +var kindScores = map[string]int{ + kindStatefulSet: ScoreLow, + kindDaemonSet: ScoreLow, + kindDeployment: ScoreLow, + "Ingress": ScoreMedium, + "Certificate": ScoreMedium, + "Node": ScoreCore, + "PersistentVolumeClaim": ScoreMedium, + "PersistentVolume": ScoreMedium, +} + +var catalogIndex map[string]catalogHit + +type catalogHit struct { + entry string + score int + key string +} + +func init() { + catalogIndex = make(map[string]catalogHit) + for _, e := range catalog { + for _, k := range e.keys { + catalogIndex[k] = catalogHit{entry: e.name, score: e.score, key: k} + } + } +} diff --git a/go/deployment-operator/pkg/streamline/insights/classify.go b/go/deployment-operator/pkg/streamline/insights/classify.go new file mode 100644 index 0000000000..7b77e492c0 --- /dev/null +++ b/go/deployment-operator/pkg/streamline/insights/classify.go @@ -0,0 +1,218 @@ +package insights + +import ( + "strings" + "time" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +const ( + // AnnotationPriority pins the insight priority of an object. IGNORE removes it from insights and the health score. + AnnotationPriority = "deployments.plural.sh/insight-priority" + + // Ignore is the annotation value that excludes an object. + Ignore = "IGNORE" +) + +const ( + kindDeployment = "Deployment" + kindStatefulSet = "StatefulSet" + kindDaemonSet = "DaemonSet" +) + +var annotationValues = map[string]struct{}{"LOW": {}, "MEDIUM": {}, "HIGH": {}, "CRITICAL": {}, Ignore: {}} + +// Criticality is how important a component is to the cluster, independent of how badly it is failing. +type Criticality struct { + Score int + Reason string +} + +// Classify scores the criticality of a component. The first source that applies wins: +// system-* priority class, catalog, system namespace tier, kind default. +func Classify(kind, namespace, name, priorityClass string) Criticality { + if strings.HasPrefix(priorityClass, "system-") { + return Criticality{Score: ScoreCore, Reason: "priority class " + priorityClass} + } + + if isWorkload(kind) { + if _, hit, ok := longestRun(name, catalogIndex); ok { + return Criticality{Score: hit.score, Reason: "catalog: " + hit.entry} + } + } + + if _, score, ok := longestRun(namespace, namespaceTiers); ok { + return Criticality{Score: score, Reason: "namespace: " + namespace} + } + + if score, ok := kindScores[kind]; ok { + return Criticality{Score: score, Reason: "kind default: " + kind} + } + return Criticality{Score: ScoreLow, Reason: "default"} +} + +func isWorkload(kind string) bool { + switch kind { + case kindDeployment, kindStatefulSet, kindDaemonSet: + return true + } + return false +} + +// IsRootKind reports whether objects of the kind can stand for a failure of the objects they own. +func IsRootKind(kind string) bool { + switch kind { + case kindDeployment, kindStatefulSet, kindDaemonSet, "Ingress", "Certificate", "CronJob": + return true + } + return false +} + +// Facts are the insight related attributes derived from an object at write time. +type Facts struct { + Criticality int + CriticalityReason string + PriorityClass string + PriorityAnnotation string + DesiredReplicas *int64 + ReadyReplicas *int64 + HealthMessage string + // FailingSince is a unix time, zero when the object is not failing. + FailingSince int64 + // LastSuccessfulTime is the unix time of the last successful run of a CronJob, zero when it has none or the + // object is not a CronJob. A retained failed job that is older than it is not a failure of the CronJob. + LastSuccessfulTime int64 +} + +// Extract derives Facts from an object. Criticality is only classified for root kinds and objects that +// are not running, as only those can end up reported. A failing object takes its failing time from +// its conditions when they carry one, and from now otherwise. +func Extract(obj *unstructured.Unstructured, running, failed bool, message string, now time.Time) Facts { + kind := obj.GetKind() + facts := Facts{} + + if failed { + facts.FailingSince = failingSince(obj, now) + facts.HealthMessage = message + } + + // Any object can end up as the root of a failure, so the annotation is always kept. + facts.PriorityAnnotation = annotation(obj) + if running && !IsRootKind(kind) { + if kind != "Pod" { // pods can not own other objects, the others might end up as the root of a failure + facts.PriorityClass = priorityClass(obj) + } + return facts + } + + facts.PriorityClass = priorityClass(obj) + facts.DesiredReplicas, facts.ReadyReplicas = replicas(obj) + facts.LastSuccessfulTime = lastSuccessfulTime(obj) + + c := Classify(kind, obj.GetNamespace(), obj.GetName(), facts.PriorityClass) + facts.Criticality, facts.CriticalityReason = c.Score, c.Reason + return facts +} + +// Annotation returns the normalized insight priority annotation of an object, or an empty string. +func Annotation(obj *unstructured.Unstructured) string { + return annotation(obj) +} + +func annotation(obj *unstructured.Unstructured) string { + // GetAnnotations copies the whole map, and this runs for every object that is written + raw, _, _ := unstructured.NestedString(obj.Object, "metadata", "annotations", AnnotationPriority) + value := strings.ToUpper(strings.TrimSpace(raw)) + if _, ok := annotationValues[value]; !ok { + return "" + } + return value +} + +var priorityClassPaths = [][]string{ + {"spec", "template", "spec", "priorityClassName"}, + {"spec", "jobTemplate", "spec", "template", "spec", "priorityClassName"}, + {"spec", "priorityClassName"}, +} + +func priorityClass(obj *unstructured.Unstructured) string { + for _, path := range priorityClassPaths { + if v, found, _ := unstructured.NestedString(obj.Object, path...); found && v != "" { + return v + } + } + return "" +} + +// lastSuccessfulTime returns the unix time of the last successful run of a CronJob, zero when unknown. +func lastSuccessfulTime(obj *unstructured.Unstructured) int64 { + if obj.GetKind() != "CronJob" { + return 0 + } + raw, _, _ := unstructured.NestedString(obj.Object, "status", "lastSuccessfulTime") + t, err := time.Parse(time.RFC3339, raw) + if err != nil { + return 0 + } + return t.Unix() +} + +// replicas returns the desired and ready replica counts of workloads, nil when unknown. +func replicas(obj *unstructured.Unstructured) (desired, ready *int64) { + switch obj.GetKind() { + case kindDeployment, kindStatefulSet: + d := int64(1) // the API default + if v, found, _ := unstructured.NestedInt64(obj.Object, "spec", "replicas"); found { + d = v + } + r, _, _ := unstructured.NestedInt64(obj.Object, "status", "readyReplicas") + return new(d), new(r) + case kindDaemonSet: + d, found, _ := unstructured.NestedInt64(obj.Object, "status", "desiredNumberScheduled") + if !found { + return nil, nil + } + r, _, _ := unstructured.NestedInt64(obj.Object, "status", "numberReady") + return new(d), new(r) + } + return nil, nil +} + +// failingConditions are the condition types which, with the matching status, mean the object is not working. +var failingConditions = map[string]string{ + "Ready": "False", + "Available": "False", + "Progressing": "False", + "Failed": "True", + "Degraded": "True", + "Stalled": "True", +} + +func failingSince(obj *unstructured.Unstructured, now time.Time) int64 { + conditions, _, _ := unstructured.NestedSlice(obj.Object, "status", "conditions") + var earliest time.Time + for _, c := range conditions { + cond, ok := c.(map[string]any) + if !ok { + continue + } + condType, _ := cond["type"].(string) + status, _ := cond["status"].(string) + if want, ok := failingConditions[condType]; !ok || want != status { + continue + } + raw, _ := cond["lastTransitionTime"].(string) + t, err := time.Parse(time.RFC3339, raw) + if err != nil || t.After(now) { + continue + } + if earliest.IsZero() || t.Before(earliest) { + earliest = t + } + } + if earliest.IsZero() { + return now.Unix() + } + return earliest.Unix() +} diff --git a/go/deployment-operator/pkg/streamline/insights/classify_test.go b/go/deployment-operator/pkg/streamline/insights/classify_test.go new file mode 100644 index 0000000000..3d1edc0a59 --- /dev/null +++ b/go/deployment-operator/pkg/streamline/insights/classify_test.go @@ -0,0 +1,171 @@ +package insights_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/insights" +) + +func TestClassify(t *testing.T) { + tests := []struct { + name string + kind string + namespace string + workload string + priorityClass string + want int + }{ + {name: "coredns", kind: "Deployment", workload: "coredns", want: insights.ScoreCore}, + {name: "core-dns", kind: "Deployment", workload: "core-dns", want: insights.ScoreCore}, + {name: "kube-dns alias", kind: "Deployment", workload: "kube-dns", want: insights.ScoreCore}, + {name: "aws-node", kind: "DaemonSet", namespace: "kube-system", workload: "aws-node", want: insights.ScoreCore}, + {name: "amazon-vpc-cni alias", kind: "DaemonSet", workload: "amazon-vpc-cni", want: insights.ScoreCore}, + {name: "ebs-csi-node", kind: "DaemonSet", workload: "ebs-csi-node", want: insights.ScoreCore}, + {name: "gce-pd-csi-controller", kind: "Deployment", workload: "gce-pd-csi-controller-sa", want: insights.ScoreCore}, + {name: "cert-manager-webhook", kind: "Deployment", workload: "cert-manager-webhook", want: insights.ScoreCore}, + {name: "ingress-nginx-controller", kind: "Deployment", workload: "ingress-nginx-controller", want: insights.ScoreCore}, + {name: "rke2 ingress alias keeps working through runs", kind: "DaemonSet", workload: "rke2-ingress-nginx-controller", want: insights.ScoreCore}, + {name: "metrics-server", kind: "Deployment", workload: "metrics-server", want: insights.ScoreCore}, + + // letters in the right order are not a match + {name: "concert-manager", kind: "Deployment", namespace: "events", workload: "concert-manager", want: insights.ScoreLow}, + {name: "core-banking-dns-proxy", kind: "Deployment", namespace: "bank", workload: "core-banking-dns-proxy", want: insights.ScoreLow}, + {name: "kubernetes-proxy-docs", kind: "Deployment", namespace: "docs", workload: "kubernetes-proxy-docs", want: insights.ScoreLow}, + {name: "node-metrics-exporter", kind: "Deployment", workload: "node-metrics-exporter", want: insights.ScoreLow}, + + // monitoring agents are apps, not part of the cluster itself + {name: "node-exporter", kind: "DaemonSet", namespace: "monitoring", workload: "prometheus-node-exporter", want: insights.ScoreLow}, + + // priority class wins over everything + {name: "system priority class", kind: "Deployment", workload: "payments", priorityClass: "system-cluster-critical", want: insights.ScoreCore}, + {name: "custom priority class is ignored", kind: "Deployment", workload: "payments", priorityClass: "high", want: insights.ScoreLow}, + + // namespace tier beats kind default, but not the catalog + {name: "kube-system workload", kind: "Deployment", namespace: "kube-system", workload: "foo", want: insights.ScoreCore}, + {name: "istio-system workload", kind: "Deployment", namespace: "istio-system", workload: "foo", want: insights.ScoreCore}, + {name: "plrl-console workload", kind: "Deployment", namespace: "plrl-console", workload: "console", want: insights.ScoreCore}, + {name: "plrl-deploy-operator workload", kind: "Deployment", namespace: "plrl-deploy-operator", workload: "deployment-operator", want: insights.ScoreCore}, + {name: "other plrl namespaces are apps", kind: "Deployment", namespace: "plrl-core", workload: "foo", want: insights.ScoreLow}, + {name: "a namespace that only shares a word is an app", kind: "Deployment", namespace: "console-banking", workload: "foo", want: insights.ScoreLow}, + + // kind defaults + {name: "statefulset", kind: "StatefulSet", workload: "db", want: insights.ScoreLow}, + {name: "daemonset", kind: "DaemonSet", workload: "agent", want: insights.ScoreLow}, + {name: "deployment", kind: "Deployment", workload: "api", want: insights.ScoreLow}, + {name: "ingress", kind: "Ingress", workload: "web", want: insights.ScoreMedium}, + {name: "certificate", kind: "Certificate", workload: "web-tls", want: insights.ScoreMedium}, + {name: "node", kind: "Node", workload: "worker-1", want: insights.ScoreCore}, + {name: "pvc", kind: "PersistentVolumeClaim", workload: "data", want: insights.ScoreMedium}, + {name: "non-workload named like a catalog entry is not special", kind: "ConfigMap", workload: "coredns", want: insights.ScoreLow}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := insights.Classify(tt.kind, tt.namespace, tt.workload, tt.priorityClass) + assert.Equal(t, tt.want, got.Score, got.Reason) + assert.NotEmpty(t, got.Reason) + }) + } +} + +func TestExtract(t *testing.T) { + now := time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) + + deployment := func(spec, status, annotations map[string]any) *unstructured.Unstructured { + meta := map[string]any{"name": "api", "namespace": "apps"} + if annotations != nil { + meta["annotations"] = annotations + } + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "apps/v1", "kind": "Deployment", "metadata": meta, "spec": spec, "status": status, + }} + } + + t.Run("reads replicas, priority class and annotation of a failing root", func(t *testing.T) { + obj := deployment( + map[string]any{"replicas": int64(3), "template": map[string]any{"spec": map[string]any{"priorityClassName": "system-node-critical"}}}, + map[string]any{"readyReplicas": int64(1)}, + map[string]any{insights.AnnotationPriority: " high "}, + ) + + facts := insights.Extract(obj, false, true, "exceeded progress deadline", now) + + assert.Equal(t, int64(3), *facts.DesiredReplicas) + assert.Equal(t, int64(1), *facts.ReadyReplicas) + assert.Equal(t, "system-node-critical", facts.PriorityClass) + assert.Equal(t, "HIGH", facts.PriorityAnnotation) + assert.Equal(t, insights.ScoreCore, facts.Criticality) + assert.Equal(t, "exceeded progress deadline", facts.HealthMessage) + assert.Equal(t, now.Unix(), facts.FailingSince) + }) + + t.Run("reads the last successful run of a cron job", func(t *testing.T) { + cronJob := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "batch/v1", "kind": "CronJob", + "metadata": map[string]any{"name": "nightly", "namespace": "apps"}, + "status": map[string]any{"lastSuccessfulTime": "2026-10-09T10:00:00Z"}, + }} + + facts := insights.Extract(cronJob, false, true, "", now) + + assert.Equal(t, time.Date(2026, 10, 9, 10, 0, 0, 0, time.UTC).Unix(), facts.LastSuccessfulTime) + assert.Zero(t, insights.Extract(deployment(map[string]any{}, map[string]any{"lastSuccessfulTime": "2026-10-09T10:00:00Z"}, nil), false, true, "", now).LastSuccessfulTime, "only cron jobs have one") + }) + + t.Run("defaults desired replicas to one and ready to zero", func(t *testing.T) { + facts := insights.Extract(deployment(map[string]any{}, map[string]any{}, nil), false, true, "", now) + + assert.Equal(t, int64(1), *facts.DesiredReplicas) + assert.Equal(t, int64(0), *facts.ReadyReplicas) + }) + + t.Run("daemon set replicas come from the status", func(t *testing.T) { + obj := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "apps/v1", "kind": "DaemonSet", "metadata": map[string]any{"name": "agent"}, + "status": map[string]any{"desiredNumberScheduled": int64(10), "numberReady": int64(4)}, + }} + + facts := insights.Extract(obj, false, true, "", now) + + assert.Equal(t, int64(10), *facts.DesiredReplicas) + assert.Equal(t, int64(4), *facts.ReadyReplicas) + }) + + t.Run("failing time comes from the earliest failing condition", func(t *testing.T) { + since := now.Add(-3 * time.Hour) + obj := deployment(map[string]any{}, map[string]any{"conditions": []any{ + map[string]any{"type": "Available", "status": "False", "lastTransitionTime": since.Format(time.RFC3339)}, + map[string]any{"type": "Progressing", "status": "False", "lastTransitionTime": since.Add(time.Hour).Format(time.RFC3339)}, + map[string]any{"type": "Ready", "status": "True", "lastTransitionTime": since.Add(-time.Hour).Format(time.RFC3339)}, + }}, nil) + + facts := insights.Extract(obj, false, true, "", now) + + assert.Equal(t, since.Unix(), facts.FailingSince) + }) + + t.Run("unknown annotation values are dropped", func(t *testing.T) { + obj := deployment(map[string]any{}, map[string]any{}, map[string]any{insights.AnnotationPriority: "urgent"}) + + assert.Empty(t, insights.Extract(obj, false, true, "", now).PriorityAnnotation) + }) + + t.Run("running root kinds are classified, other running objects are not", func(t *testing.T) { + assert.NotZero(t, insights.Extract(deployment(map[string]any{}, map[string]any{}, nil), true, false, "", now).Criticality) + + pod := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "v1", "kind": "Pod", "metadata": map[string]any{"name": "p"}}} + facts := insights.Extract(pod, true, false, "", now) + assert.Zero(t, facts.Criticality) + assert.Zero(t, facts.FailingSince) + }) + + t.Run("failing non-root objects are classified", func(t *testing.T) { + node := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "v1", "kind": "Node", "metadata": map[string]any{"name": "n1"}}} + + assert.Equal(t, insights.ScoreCore, insights.Extract(node, false, true, "", now).Criticality) + }) +} diff --git a/go/deployment-operator/pkg/streamline/insights/match.go b/go/deployment-operator/pkg/streamline/insights/match.go new file mode 100644 index 0000000000..8f2ed8a329 --- /dev/null +++ b/go/deployment-operator/pkg/streamline/insights/match.go @@ -0,0 +1,33 @@ +package insights + +import "strings" + +const maxTokens = 8 + +// tokenize lowercases s and splits it on '-', '_' and '.'. +func tokenize(s string) []string { + tokens := strings.FieldsFunc(strings.ToLower(s), func(r rune) bool { + return r == '-' || r == '_' || r == '.' + }) + if len(tokens) > maxTokens { + tokens = tokens[:maxTokens] + } + return tokens +} + +// longestRun returns the longest key of keys that equals a contiguous run of the tokens of s joined +// without separators. Subsequences and near misses never match: "concert-manager" contains the +// letters of "certmanager" in order but its runs are "concert", "manager" and "concertmanager". +func longestRun[V any](s string, keys map[string]V) (key string, value V, ok bool) { + tokens := tokenize(s) + for i := range tokens { + var run strings.Builder + for j := i; j < len(tokens); j++ { + run.WriteString(tokens[j]) + if v, found := keys[run.String()]; found && run.Len() > len(key) { + key, value, ok = run.String(), v, true + } + } + } + return key, value, ok +} diff --git a/go/deployment-operator/pkg/streamline/insights/rollup.go b/go/deployment-operator/pkg/streamline/insights/rollup.go new file mode 100644 index 0000000000..123c52d2b3 --- /dev/null +++ b/go/deployment-operator/pkg/streamline/insights/rollup.go @@ -0,0 +1,223 @@ +package insights + +import ( + "sort" + "time" +) + +// Priority is the urgency of a failing root. +type Priority string + +const ( + PriorityLow Priority = "LOW" + PriorityMedium Priority = "MEDIUM" + PriorityHigh Priority = "HIGH" + PriorityCritical Priority = "CRITICAL" + + // PriorityIgnore is only valid as an annotation or an override. It removes a root from the insights and the health score. + PriorityIgnore Priority = Ignore +) + +// Health score penalties by priority. Low and medium failures are capped so that any number of them can not +// take a cluster out of the green band on their own, while high and critical failures always show. +const ( + penaltyLow = 2 + penaltyMedium = 8 + penaltyHigh = 25 + penaltyCritical = 50 + + capLow = 10 + capMedium = 30 + + maxScore = 100 + + hour = time.Hour + day = 24 * time.Hour +) + +// Root is a failing object that stands for all the failed objects it owns, with everything needed to prioritize it. +type Root struct { + UID string + Group string + Version string + Kind string + Namespace string + Name string + // ServiceID is the Plural service managing the root, inherited from its nearest ancestor that has one. + ServiceID string + PriorityClass string + HealthMessage string + DesiredReplicas *int64 + ReadyReplicas *int64 + // State is the health of the root itself: failed, or pending when only objects it owns are failing. + State string + Labels map[string]string + // Descendants counts the failed and pending objects the root owns by kind, the root itself excluded. + Descendants map[string]Descendants + CreatedAt time.Time + FailingSince time.Time + Criticality int + CriticalityReason string + PriorityAnnotation string +} + +// Descendants counts the unhealthy objects of one kind below a root. +type Descendants struct { + Failed int + Pending int +} + +// Override replaces the built-in priority of a root, it comes from a CustomInsightPriority script. +type Override struct { + Priority Priority + Reason string +} + +// Entry is a root together with its final priority. +type Entry struct { + Root + // Score is the built-in score, criticality plus severity. + Score int + Priority Priority + // Source says where the priority came from: annotation, script or built-in. + Source string + Reason string +} + +// Summary is the result of one ping: the health score and the roots that explain it. +type Summary struct { + HealthScore int64 + // Entries are sorted most urgent first. They never contain ignored roots. + Entries []Entry + // Ignored are the UIDs of the roots that were left out because they are ignored. + Ignored map[string]struct{} +} + +// Score is the built-in score of a root: its criticality plus how severe the failure is. +// Severity grows with the share of replicas that are not ready and with how long the root has been failing. +// +// A criticality below ThresholdHigh marks an app, whose score is capped below HIGH however bad the +// failure is. Only core components, which start at ScoreCore, can be HIGH or CRITICAL on their own. +// The annotation and CustomInsightPriority scripts are applied in Resolve and are not subject to the cap. +func Score(r Root, now time.Time) int { + criticality := r.Criticality + if criticality == 0 { + criticality = ScoreLow + } + + score := criticality + if r.DesiredReplicas != nil && r.ReadyReplicas != nil && *r.DesiredReplicas > 0 { + missing := max(*r.DesiredReplicas-*r.ReadyReplicas, 0) + score += int(float64(SeverityMax)*float64(missing)/float64(*r.DesiredReplicas) + 0.5) + } + + if !r.FailingSince.IsZero() { + switch age := now.Sub(r.FailingSince); { + case age >= day: + score += 5 + case age >= hour: + score += 3 + } + } + + if criticality < ThresholdHigh { + score = min(score, ThresholdHigh-1) + } + + return min(score, maxScore) +} + +// Bucket maps a score to a priority. +func Bucket(score int) Priority { + switch { + case score >= ThresholdCritical: + return PriorityCritical + case score >= ThresholdHigh: + return PriorityHigh + case score >= ThresholdMedium: + return PriorityMedium + default: + return PriorityLow + } +} + +// Resolve decides the priority of a root. The first source that applies wins: the annotation on the object, +// a script override, the built-in score. +func Resolve(r Root, overrides map[string]Override, now time.Time) Entry { + entry := Entry{Root: r, Score: Score(r, now)} + + if p := Priority(r.PriorityAnnotation); p != "" { + entry.Priority, entry.Source, entry.Reason = p, "annotation", "annotation "+AnnotationPriority + return entry + } + + if o, ok := overrides[r.UID]; ok { + entry.Priority, entry.Source, entry.Reason = o.Priority, "script", o.Reason + return entry + } + + entry.Priority, entry.Source, entry.Reason = Bucket(entry.Score), "built-in", r.CriticalityReason + return entry +} + +// Rank orders priorities, the most urgent has the highest rank. +func Rank(p Priority) int { + switch p { + case PriorityCritical: + return 3 + case PriorityHigh: + return 2 + case PriorityMedium: + return 1 + default: + return 0 + } +} + +// Summarize computes the health score and the insights list from the same roots. +// Score is 100 minus the penalties of the roots, one penalty per root, so it does not depend on how many pods a root owns. +func Summarize(roots []Root, overrides map[string]Override, now time.Time) Summary { + var counts [4]int + ignored := make(map[string]struct{}) + entries := make([]Entry, 0, len(roots)) + for _, r := range roots { + e := Resolve(r, overrides, now) + switch e.Priority { + case PriorityIgnore: + ignored[r.UID] = struct{}{} + continue + case PriorityLow: + counts[0]++ + case PriorityMedium: + counts[1]++ + case PriorityHigh: + counts[2]++ + case PriorityCritical: + counts[3]++ + } + entries = append(entries, e) + } + + penalty := min(counts[0]*penaltyLow, capLow) + + min(counts[1]*penaltyMedium, capMedium) + + counts[2]*penaltyHigh + + counts[3]*penaltyCritical + + sort.SliceStable(entries, func(i, j int) bool { + a, b := entries[i], entries[j] + switch { + case Rank(a.Priority) != Rank(b.Priority): + return Rank(a.Priority) > Rank(b.Priority) + case a.Score != b.Score: + return a.Score > b.Score + case a.Kind != b.Kind: + return a.Kind < b.Kind + case a.Namespace != b.Namespace: + return a.Namespace < b.Namespace + default: + return a.Name < b.Name + } + }) + + return Summary{HealthScore: int64(max(maxScore-penalty, 0)), Entries: entries, Ignored: ignored} +} diff --git a/go/deployment-operator/pkg/streamline/insights/rollup_test.go b/go/deployment-operator/pkg/streamline/insights/rollup_test.go new file mode 100644 index 0000000000..f0a6af9352 --- /dev/null +++ b/go/deployment-operator/pkg/streamline/insights/rollup_test.go @@ -0,0 +1,192 @@ +package insights_test + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/insights" +) + +var now = time.Date(2026, 10, 9, 12, 0, 0, 0, time.UTC) + +func root(name string, criticality int) insights.Root { + return insights.Root{UID: name, Kind: "Deployment", Namespace: "apps", Name: name, Criticality: criticality, CriticalityReason: "test"} +} + +func TestScore(t *testing.T) { + tests := []struct { + name string + root insights.Root + want int + }{ + {name: "criticality alone", root: root("a", insights.ScoreMedium), want: insights.ScoreMedium}, + {name: "unclassified roots are low", root: root("a", 0), want: insights.ScoreLow}, + {name: "full outage adds the whole severity", root: withReplicas(root("a", insights.ScoreLow), 3, 0), want: insights.ScoreLow + insights.SeverityMax}, + {name: "partial outage adds a share", root: withReplicas(root("a", insights.ScoreLow), 3, 2), want: insights.ScoreLow + 7}, + {name: "all ready adds nothing", root: withReplicas(root("a", insights.ScoreLow), 3, 3), want: insights.ScoreLow}, + {name: "scaled to zero adds nothing", root: withReplicas(root("a", insights.ScoreLow), 0, 0), want: insights.ScoreLow}, + {name: "more ready than desired adds nothing", root: withReplicas(root("a", insights.ScoreLow), 2, 3), want: insights.ScoreLow}, + {name: "failing for an hour", root: failingFor(root("a", insights.ScoreLow), time.Hour), want: insights.ScoreLow + 3}, + {name: "failing for a day", root: failingFor(root("a", insights.ScoreLow), 25*time.Hour), want: insights.ScoreLow + 5}, + {name: "failing for minutes", root: failingFor(root("a", insights.ScoreLow), 5*time.Minute), want: insights.ScoreLow}, + {name: "capped at 100", root: failingFor(withReplicas(root("a", 95), 1, 0), 48*time.Hour), want: 100}, + + // apps never reach HIGH on their own, however bad the failure is + {name: "an app with a high base is capped below HIGH", root: withReplicas(root("a", insights.ScoreMedium), 1, 0), want: insights.ThresholdHigh - 1}, + {name: "an app failing for days is capped below HIGH", root: failingFor(withReplicas(root("a", insights.ScoreMedium), 2, 0), 72*time.Hour), want: insights.ThresholdHigh - 1}, + {name: "a total app outage is MEDIUM", root: withReplicas(root("a", insights.ScoreLow), 2, 0), want: insights.ScoreLow + insights.SeverityMax}, + + // core components are HIGH when partly down and CRITICAL when most of them are down + {name: "core partial outage stays HIGH", root: withReplicas(root("a", insights.ScoreCore), 2, 1), want: insights.ScoreCore + 10}, + {name: "core three quarters down is CRITICAL", root: withReplicas(root("a", insights.ScoreCore), 4, 1), want: insights.ThresholdCritical}, + {name: "core full outage is CRITICAL", root: withReplicas(root("a", insights.ScoreCore), 2, 0), want: insights.ScoreCore + insights.SeverityMax}, + {name: "core half down for a day is CRITICAL", root: failingFor(withReplicas(root("a", insights.ScoreCore), 2, 1), 25*time.Hour), want: insights.ThresholdCritical}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, insights.Score(tt.root, now)) + }) + } +} + +func TestBucket(t *testing.T) { + assert.Equal(t, insights.PriorityLow, insights.Bucket(0)) + assert.Equal(t, insights.PriorityLow, insights.Bucket(insights.ThresholdMedium-1)) + assert.Equal(t, insights.PriorityMedium, insights.Bucket(insights.ThresholdMedium)) + assert.Equal(t, insights.PriorityHigh, insights.Bucket(insights.ThresholdHigh)) + assert.Equal(t, insights.PriorityCritical, insights.Bucket(insights.ThresholdCritical)) + assert.Equal(t, insights.PriorityCritical, insights.Bucket(100)) + + // a full outage lifts a failure by exactly one bucket, never two. Apps are additionally capped below HIGH in Score. + for _, c := range []struct{ base, want insights.Priority }{ + {insights.Bucket(insights.ScoreLow + insights.SeverityMax), insights.PriorityMedium}, + {insights.Bucket(insights.ScoreMedium + insights.SeverityMax), insights.PriorityHigh}, + {insights.Bucket(insights.ScoreCore + insights.SeverityMax), insights.PriorityCritical}, + } { + assert.Equal(t, c.want, c.base) + } +} + +func TestResolve(t *testing.T) { + overrides := map[string]insights.Override{"a": {Priority: insights.PriorityHigh, Reason: "script says so"}} + + t.Run("built-in", func(t *testing.T) { + e := insights.Resolve(root("b", insights.ScoreMedium), overrides, now) + assert.Equal(t, insights.PriorityMedium, e.Priority) + assert.Equal(t, "built-in", e.Source) + assert.Equal(t, "test", e.Reason) + }) + + t.Run("script overrides built-in", func(t *testing.T) { + e := insights.Resolve(root("a", insights.ScoreLow), overrides, now) + assert.Equal(t, insights.PriorityHigh, e.Priority) + assert.Equal(t, "script", e.Source) + }) + + t.Run("annotation overrides script", func(t *testing.T) { + r := root("a", insights.ScoreLow) + r.PriorityAnnotation = "CRITICAL" + + e := insights.Resolve(r, overrides, now) + + assert.Equal(t, insights.PriorityCritical, e.Priority) + assert.Equal(t, "annotation", e.Source) + }) +} + +func TestSummarize(t *testing.T) { + many := func(n int, criticality int) []insights.Root { + roots := make([]insights.Root, n) + for i := range roots { + roots[i] = root(fmt.Sprintf("r-%d-%d", criticality, i), criticality) + } + return roots + } + + t.Run("third party failures can not take the score below 60", func(t *testing.T) { + roots := append(many(20, insights.ScoreLow), many(20, insights.ScoreMedium)...) + + s := insights.Summarize(roots, nil, now) + + assert.Equal(t, int64(60), s.HealthScore, "LOW cap 10 plus MEDIUM cap 30") + assert.Len(t, s.Entries, 40, "the list is not capped") + }) + + t.Run("a healthy cluster scores 100", func(t *testing.T) { + s := insights.Summarize(nil, nil, now) + assert.Equal(t, int64(100), s.HealthScore) + assert.Empty(t, s.Entries) + }) + + t.Run("one penalty per root by priority", func(t *testing.T) { + assert.Equal(t, int64(98), insights.Summarize(many(1, insights.ScoreLow), nil, now).HealthScore) + assert.Equal(t, int64(92), insights.Summarize(many(1, insights.ScoreMedium), nil, now).HealthScore) + assert.Equal(t, int64(75), insights.Summarize(many(1, insights.ScoreCore), nil, now).HealthScore) + assert.Equal(t, int64(50), insights.Summarize(many(1, insights.ThresholdCritical), nil, now).HealthScore) + }) + + t.Run("low failures can not leave the green band", func(t *testing.T) { + s := insights.Summarize(many(500, insights.ScoreLow), nil, now) + assert.Greater(t, s.HealthScore, int64(80)) + }) + + t.Run("medium failures are capped more loosely", func(t *testing.T) { + s := insights.Summarize(many(500, insights.ScoreMedium), nil, now) + assert.Equal(t, int64(70), s.HealthScore) + }) + + t.Run("high and critical failures are not capped", func(t *testing.T) { + assert.Equal(t, int64(0), insights.Summarize(many(10, insights.ScoreCore), nil, now).HealthScore) + assert.Equal(t, int64(0), insights.Summarize(many(3, insights.ThresholdCritical), nil, now).HealthScore) + }) + + t.Run("ignored roots are in neither the list nor the score", func(t *testing.T) { + roots := many(2, insights.ThresholdCritical) + roots[0].PriorityAnnotation = "IGNORE" + + s := insights.Summarize(roots, nil, now) + + assert.Equal(t, int64(50), s.HealthScore) + require.Len(t, s.Entries, 1) + assert.Equal(t, roots[1].UID, s.Entries[0].UID) + }) + + t.Run("overrides move both the list and the score", func(t *testing.T) { + roots := many(1, insights.ScoreLow) + + s := insights.Summarize(roots, map[string]insights.Override{roots[0].UID: {Priority: insights.PriorityCritical}}, now) + + assert.Equal(t, int64(50), s.HealthScore) + assert.Equal(t, insights.PriorityCritical, s.Entries[0].Priority) + }) + + t.Run("most urgent first, then a stable order", func(t *testing.T) { + pinned := root("pinned", insights.ScoreLow) + pinned.PriorityAnnotation = "CRITICAL" + b, a := root("b", insights.ScoreMedium), root("a", insights.ScoreMedium) + critical := root("critical", insights.ThresholdCritical) + + s := insights.Summarize([]insights.Root{b, a, pinned, critical}, nil, now) + + var names []string + for _, e := range s.Entries { + names = append(names, e.Name) + } + assert.Equal(t, []string{"critical", "pinned", "a", "b"}, names) + }) +} + +func withReplicas(r insights.Root, desired, ready int64) insights.Root { + r.DesiredReplicas, r.ReadyReplicas = &desired, &ready + return r +} + +func failingFor(r insights.Root, d time.Duration) insights.Root { + r.FailingSince = now.Add(-d) + return r +} diff --git a/go/deployment-operator/pkg/streamline/store/common.go b/go/deployment-operator/pkg/streamline/store/common.go index cc6f324948..e3088ca4b0 100644 --- a/go/deployment-operator/pkg/streamline/store/common.go +++ b/go/deployment-operator/pkg/streamline/store/common.go @@ -2,55 +2,12 @@ package store import ( "github.com/pluralsh/console/go/client" - "github.com/sahilm/fuzzy" "github.com/samber/lo" "k8s.io/klog/v2" "zombiezen.com/go/sqlite" "zombiezen.com/go/sqlite/sqlitex" ) -var ( - kindsInsightPriorities = map[string]client.InsightComponentPriority{ - "Ingress": client.InsightComponentPriorityCritical, - "Certificate": client.InsightComponentPriorityCritical, // cert-manager Certificate - "StatefulSet": client.InsightComponentPriorityHigh, - "DaemonSet": client.InsightComponentPriorityMedium, - "Deployment": client.InsightComponentPriorityLow, - } - - resourcesInsightPriorities = map[string]client.InsightComponentPriority{ - "certmanager": client.InsightComponentPriorityCritical, - "coredns": client.InsightComponentPriorityCritical, - "kubeproxy": client.InsightComponentPriorityCritical, - "istio": client.InsightComponentPriorityCritical, - "linkerd": client.InsightComponentPriorityCritical, - "csinode": client.InsightComponentPriorityCritical, - "csicontroller": client.InsightComponentPriorityCritical, - "nodeexporter": client.InsightComponentPriorityHigh, - } -) - -// InsightComponentPriority returns insight priority for a given component. -func InsightComponentPriority(name, namespace, kind string) *client.InsightComponentPriority { - for resource, priority := range resourcesInsightPriorities { - // Fuzzy match to find similar resources - matches := fuzzy.Find(resource, []string{name, namespace}) - - // Only consider first score threshold as it is the best match - if len(matches) > 0 && matches[0].Score >= 200 { - return lo.ToPtr(priority) - } - } - - // Check if the kind is directly mapped to a priority - if priority, exists := kindsInsightPriorities[kind]; exists { - return lo.ToPtr(priority) - } - - // Default to low priority if no matches found - return lo.ToPtr(client.InsightComponentPriorityLow) -} - // NodeStatisticHealth returns health status based on the number of pending pods. func NodeStatisticHealth(pendingPods int64) *client.NodeStatisticHealth { switch { @@ -91,3 +48,17 @@ func WithTransaction(conn *sqlite.Conn, fn func() error) (err error) { err = fn() return err } + +// withReadTransaction runs fn in a deferred transaction, so all its reads see one snapshot and no write lock is taken. +func withReadTransaction(conn *sqlite.Conn, fn func() error) (err error) { + if err = sqlitex.Execute(conn, "BEGIN", nil); err != nil { + return err + } + defer func() { + if e := sqlitex.Execute(conn, "ROLLBACK", nil); e != nil && err == nil { + err = e + } + }() + + return fn() +} diff --git a/go/deployment-operator/pkg/streamline/store/db_queries.go b/go/deployment-operator/pkg/streamline/store/db_queries.go index 149d4c7ddd..470f0ca0fb 100644 --- a/go/deployment-operator/pkg/streamline/store/db_queries.go +++ b/go/deployment-operator/pkg/streamline/store/db_queries.go @@ -1,5 +1,43 @@ package store +// failingSinceUpsert keeps the earliest known failing time while the same object keeps failing, and resets it once the +// component recovers. A replacement with the same name but another uid starts its own failing time. +const failingSinceUpsert = `CASE + WHEN excluded.health != 2 THEN 0 + WHEN COALESCE(component.failing_since, 0) > 0 AND component.health = 2 + AND COALESCE(component.uid, '') IN ('', excluded.uid) THEN MIN(component.failing_since, excluded.failing_since) + ELSE excluded.failing_since + END` + +// ownershipUpsert keeps the parent and service of a component when a watch event carries neither. Objects rendered by +// Flux from a HelmRelease have no owner reference or inventory annotation, so the values set from the HelmRelease +// would be lost on every event. A new owner reference or inventory annotation still replaces them. +// Only the same object keeps them. A replacement that has the same name but another uid does not inherit the owner +// of the object it replaced, a stored row without a uid is a placeholder of the same object. +const ownershipUpsert = `parent_uid = CASE + WHEN COALESCE(excluded.parent_uid, '') != '' THEN excluded.parent_uid + WHEN COALESCE(excluded.service_id, '') = '' AND COALESCE(component.service_id, '') != '' + AND COALESCE(component.uid, '') IN ('', excluded.uid) THEN component.parent_uid + ELSE excluded.parent_uid + END, + service_id = CASE + WHEN COALESCE(excluded.service_id, '') != '' THEN excluded.service_id + WHEN COALESCE(excluded.parent_uid, '') = '' AND COALESCE(component.parent_uid, '') != '' + AND COALESCE(component.uid, '') IN ('', excluded.uid) THEN component.service_id + ELSE excluded.service_id + END,` + +// factsUpsert refreshes the insight attributes derived from the object. +const factsUpsert = `criticality = excluded.criticality, + criticality_reason = excluded.criticality_reason, + priority_class = excluded.priority_class, + failing_since = ` + failingSinceUpsert + `, + desired_replicas = excluded.desired_replicas, + ready_replicas = excluded.ready_replicas, + priority_annotation = excluded.priority_annotation, + health_message = excluded.health_message, + last_success = excluded.last_success` + const ( createTables = ` CREATE TABLE IF NOT EXISTS component ( @@ -23,7 +61,16 @@ const ( server_sha TEXT, manifest BOOLEAN DEFAULT 0, -- Indicates if the component was created from an original manifest set of a service applied BOOLEAN DEFAULT 0, -- Indicates if the component was already applied to the cluster - labels TEXT -- JSON object of Kubernetes object labels + labels TEXT, -- JSON object of Kubernetes object labels + criticality INT, -- built-in criticality score of the component, see the insights package + criticality_reason TEXT, + priority_class TEXT, -- priority class name of the pod template + failing_since INT DEFAULT 0, -- unix time the component started to fail, 0 if it is not failing + desired_replicas INT, -- NULL when unknown + ready_replicas INT, -- NULL when unknown + priority_annotation TEXT, -- normalized value of the insight priority annotation + health_message TEXT, + last_success INT DEFAULT 0 -- unix time of the last successful run of a CronJob, 0 when unknown ); CREATE UNIQUE INDEX IF NOT EXISTS idx_unique_component ON component("group", version, kind, namespace, name); CREATE INDEX IF NOT EXISTS idx_parent ON component(parent_uid); @@ -80,6 +127,14 @@ const ( WHERE uid = ? AND applied = 1 ` + getPriorityAnnotation = ` + SELECT COALESCE(priority_annotation, '') + FROM component + WHERE uid = ? AND applied = 1 + ORDER BY id + LIMIT 1 + ` + getAppliedComponentsByGVK = ` SELECT uid, "group", version, kind, namespace, name, server_sha, delete_phase, manifest FROM component @@ -102,7 +157,16 @@ const ( delete_phase, server_sha, applied, - labels + labels, + criticality, + criticality_reason, + priority_class, + failing_since, + desired_replicas, + ready_replicas, + priority_annotation, + health_message, + last_success ) VALUES ( ?, ?, @@ -118,18 +182,27 @@ const ( ?, ?, ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, ? ) ON CONFLICT("group", version, kind, namespace, name) DO UPDATE SET uid = excluded.uid, - parent_uid = excluded.parent_uid, + ` + ownershipUpsert + ` health = excluded.health, node = excluded.node, created_at = excluded.created_at, - service_id = excluded.service_id, delete_phase = excluded.delete_phase, server_sha = excluded.server_sha, applied = excluded.applied, - labels = excluded.labels + labels = excluded.labels, + ` + factsUpsert + ` ` setComponentUnsynced = ` @@ -142,7 +215,8 @@ const ( manifest_sha = '', transient_manifest_sha = '', apply_sha = '', - labels = '' + labels = '', + failing_since = 0 WHERE "group" = ? AND version = ? AND kind = ? AND namespace = ? AND name = ? ` @@ -181,26 +255,6 @@ const ( AND name = ? ` - clusterHealthScore = ` - WITH base_score AS ( - SELECT CAST(AVG(CASE WHEN health = 0 THEN 100 ELSE 0 END) as INTEGER) as score - FROM component - ), - deductions AS ( - SELECT - SUM(CASE - WHEN kind = 'Certificate' AND health = 2 THEN 10 - WHEN namespace = 'kube-system' AND health = 2 THEN 20 - WHEN kind = 'PersistentVolume' AND health = 2 THEN 10 - WHEN (namespace = 'istio-system' OR name LIKE '%coredns%' OR name LIKE '%aws-cni%') AND health = 2 THEN 50 - WHEN (namespace LIKE '%ingress%' OR namespace LIKE '%traefik%') AND kind = 'Service' AND health = 2 THEN 50 - ELSE 0 - END) as total_deductions - FROM component - ) - SELECT MAX(0, (SELECT score FROM base_score) - (SELECT COALESCE(total_deductions, 0) FROM deductions)) as score - ` - nodeStatistics = ` SELECT node, COUNT(*) FROM component @@ -221,7 +275,10 @@ const ( applied, node, created_at, - service_id + service_id, + criticality, + criticality_reason, + failing_since ) VALUES ( ?, ?, @@ -234,6 +291,9 @@ const ( ?, ?, ?, + ?, + ?, + ?, ? ) ON CONFLICT("group", version, kind, namespace, name) DO UPDATE SET uid = excluded.uid, @@ -242,45 +302,115 @@ const ( node = excluded.node, created_at = excluded.created_at, service_id = excluded.service_id, - applied = excluded.applied + applied = excluded.applied, + criticality = excluded.criticality, + criticality_reason = excluded.criticality_reason, + failing_since = ` + failingSinceUpsert + ` ` - failedComponents = ` - WITH RECURSIVE component_chain AS ( - -- Start with parent components of specified kinds - SELECT *, 1 as level, uid as root_uid - FROM component - WHERE kind IN ('Deployment', 'StatefulSet', 'Ingress', 'DaemonSet', 'Certificate') AND applied = 1 - + // failingRootsPrefix selects the roots of all failures, see failingRoots. + failingRootsPrefix = ` + WITH RECURSIVE climb(fid, uid, parent_uid, kind, depth) AS ( + SELECT id, uid, parent_uid, kind, 1 + FROM component + WHERE health = 2 AND applied = 1 + UNION ALL - - -- Get children of components in the chain, carrying the root component UID - SELECT c.*, cc.level + 1, cc.root_uid - FROM component c - JOIN component_chain cc ON c.parent_uid = cc.uid AND c.parent_uid != '' - WHERE cc.level < 4 AND c.applied = 1 + + SELECT climb.fid, p.uid, p.parent_uid, p.kind, climb.depth + 1 + FROM climb + JOIN component p ON p.uid = climb.parent_uid AND p.applied = 1 AND p.kind != 'Node' + WHERE climb.parent_uid != '' + AND climb.depth < 4 + AND climb.kind NOT IN ('Deployment', 'StatefulSet', 'DaemonSet', 'Ingress', 'Certificate', 'CronJob') + ), + top AS ( + SELECT fid, uid AS root_uid + FROM climb + WHERE kind IN ('Deployment', 'StatefulSet', 'DaemonSet', 'Ingress', 'Certificate', 'CronJob') + OR NOT EXISTS ( + SELECT 1 FROM component p + WHERE p.uid = climb.parent_uid AND climb.parent_uid != '' AND p.applied = 1 AND p.kind != 'Node' + ) ), - -- Find all failed components in the chain - failed_roots AS ( - -- Get root UIDs where any component in the chain is failed - SELECT DISTINCT root_uid - FROM component_chain - WHERE health = 2 + roots AS ( + SELECT top.root_uid, MIN(NULLIF(f.failing_since, 0)) AS failing_since + FROM top + JOIN component f ON f.id = top.fid + GROUP BY top.root_uid + )` + + // failingRoots returns the objects that represent the failures of the cluster, one row per root. + // Every failed component is walked up its owners, at most 4 levels, to the first root kind. A chain that ends + // without one is represented by its top-most object, and a chain that is too deep to reach either is ignored. + // A root inherits the service of its nearest owner that has one. + failingRoots = failingRootsPrefix + `, + owner(root_uid, uid, parent_uid, service_id, depth) AS ( + SELECT r.root_uid, c.uid, c.parent_uid, COALESCE(c.service_id, ''), 1 + FROM roots r + JOIN component c ON c.id = (SELECT MIN(x.id) FROM component x WHERE x.uid = r.root_uid AND x.applied = 1) + + UNION ALL + + SELECT owner.root_uid, p.uid, p.parent_uid, COALESCE(p.service_id, ''), owner.depth + 1 + FROM owner + JOIN component p ON p.uid = owner.parent_uid AND p.applied = 1 + WHERE owner.service_id = '' AND owner.parent_uid != '' AND owner.depth < 4 + ) + SELECT + c.uid, + c."group", + c.version, + c.kind, + c.namespace, + c.name, + COALESCE((SELECT o.service_id FROM owner o WHERE o.root_uid = c.uid AND o.service_id != '' LIMIT 1), ''), + COALESCE(c.priority_class, ''), + COALESCE(c.health_message, ''), + c.desired_replicas, + c.ready_replicas, + COALESCE(c.created_at, 0), + COALESCE(r.failing_since, 0), + COALESCE(c.criticality, 0), + COALESCE(c.criticality_reason, ''), + COALESCE(c.priority_annotation, ''), + COALESCE(c.labels, ''), + c.health + FROM roots r + JOIN component c ON c.id = (SELECT MIN(x.id) FROM component x WHERE x.uid = r.root_uid AND x.applied = 1) + WHERE (c.kind != 'Node' OR (COALESCE(r.failing_since, 0) > 0 AND r.failing_since <= CAST(strftime('%s', 'now') AS INTEGER) - 300)) + AND (c.kind != 'CronJob' OR EXISTS ( + SELECT 1 FROM ( + SELECT j.health AS health, j.created_at AS created_at FROM component j + WHERE j.parent_uid = c.uid AND j.kind = 'Job' AND j.applied = 1 + ORDER BY j.created_at DESC LIMIT 1 + ) newest + WHERE newest.health = 2 AND (COALESCE(c.last_success, 0) = 0 OR c.last_success < newest.created_at) + )) + ` + + // failingRootsDescendants counts the failed and pending objects below every root of failingRoots, by kind. + // An object watched through several API versions has a row per version, and each of them can have its own parent, + // so every row is walked and the objects are counted once by uid. + failingRootsDescendants = failingRootsPrefix + `, + down(root_uid, uid, kind, health, depth) AS ( + SELECT r.root_uid, c.uid, c.kind, c.health, 0 + FROM roots r + JOIN component c ON c.id = (SELECT MIN(x.id) FROM component x WHERE x.uid = r.root_uid AND x.applied = 1) + + UNION ALL + + SELECT down.root_uid, ch.uid, ch.kind, ch.health, down.depth + 1 + FROM down + JOIN component ch ON ch.parent_uid = down.uid AND ch.applied = 1 + WHERE down.uid != '' AND down.depth < 3 AND down.kind != 'Node' ) - -- Return both the failed components and their original parent components - SELECT DISTINCT cc.uid, cc."group", cc.version, cc.kind, cc.namespace, cc.name - FROM component_chain cc - WHERE (cc.health = 2 -- The component itself is failed - OR cc.uid IN ( -- OR it's a direct parent of a failed component - SELECT parent_uid - FROM component_chain - WHERE health = 2 AND parent_uid IS NOT NULL - ) - OR (cc.uid IN ( -- OR it's the original root component of a chain with failures - SELECT root_uid FROM failed_roots - ) AND cc.kind IN ('Deployment', 'StatefulSet', 'Ingress', 'DaemonSet', 'Certificate'))) - AND cc.kind IN ('Deployment', 'StatefulSet', 'Ingress', 'DaemonSet', 'Certificate') - AND (cc.service_id IS NULL OR cc.service_id = '') + SELECT root_uid, kind, + COUNT(DISTINCT CASE WHEN health = 2 THEN uid END), + COUNT(DISTINCT CASE WHEN health = 1 THEN uid END) + FROM down + WHERE depth > 0 AND health IN (1, 2) + GROUP BY root_uid, kind ` serverCounts = ` diff --git a/go/deployment-operator/pkg/streamline/store/db_store.go b/go/deployment-operator/pkg/streamline/store/db_store.go index 35f920ac44..218e3b4e2e 100644 --- a/go/deployment-operator/pkg/streamline/store/db_store.go +++ b/go/deployment-operator/pkg/streamline/store/db_store.go @@ -25,6 +25,7 @@ import ( "github.com/pluralsh/console/go/deployment-operator/pkg/log" "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/api" smcommon "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/common" + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/insights" ) const ( @@ -35,6 +36,11 @@ const ( defaultPoolSize = 10 defaultQueryTimeout = 10 * time.Second + + // schemaVersion is stored in the database as PRAGMA user_version. Bump it whenever the component table changes + // or the meaning of a stored column does: version 3 changed what the stored criticality stands for, core + // components start at 60 and apps are capped below HIGH. Version 4 added the last successful run of a CronJob. + schemaVersion = 4 ) // Option represents a function that configures the database store. @@ -131,29 +137,36 @@ func (in *DatabaseStore) init() error { cancelFunc() }() - if err := sqlitex.ExecuteScript(conn, createTables, nil); err != nil { + if err := in.migrate(conn); err != nil { return err } - return in.ensureLabelsColumn(conn) + return sqlitex.ExecuteScript(conn, createTables, nil) } -func (in *DatabaseStore) ensureLabelsColumn(conn *sqlite.Conn) error { - hasLabels := false - err := sqlitex.ExecuteTransient(conn, `SELECT 1 FROM pragma_table_info('component') WHERE name = 'labels'`, &sqlitex.ExecOptions{ - ResultFunc: func(_ *sqlite.Stmt) error { - hasLabels = true +// migrate rebuilds the component table when the schema of an existing database is outdated. +// The store is a cache of the cluster state that is repopulated by the synchronizers, so there is no point in migrating it in place. +func (in *DatabaseStore) migrate(conn *sqlite.Conn) error { + var version int + err := sqlitex.ExecuteTransient(conn, `PRAGMA user_version`, &sqlitex.ExecOptions{ + ResultFunc: func(stmt *sqlite.Stmt) error { + version = stmt.ColumnInt(0) return nil }, }) if err != nil { return err } - if hasLabels { + if version == schemaVersion { return nil } - return sqlitex.Execute(conn, `ALTER TABLE component ADD COLUMN labels TEXT`, nil) + klog.V(log.LogLevelDefault).InfoS("rebuilding component table", "from", version, "to", schemaVersion) + if err := sqlitex.ExecuteTransient(conn, `DROP TABLE IF EXISTS component`, nil); err != nil { + return err + } + + return sqlitex.ExecuteTransient(conn, fmt.Sprintf(`PRAGMA user_version = %d`, schemaVersion), nil) } func (in *DatabaseStore) take() (*sqlite.Conn, context.CancelFunc, error) { @@ -235,7 +248,7 @@ func (in *DatabaseStore) GetResourceHealth(resources []unstructured.Unstructured func (in *DatabaseStore) SaveComponent(obj unstructured.Unstructured) error { gvk := obj.GroupVersionKind() - status := common.ToStatus(&obj) + status, message := common.ToStatusWithMessage(&obj) state := NewComponentState(status) serviceID := smcommon.GetOwningInventory(obj) @@ -282,10 +295,11 @@ func (in *DatabaseStore) SaveComponent(obj unstructured.Unstructured) error { in.maybeSaveHookComponent(conn, obj, lo.FromPtr(status), serviceID) - return in.upsertAppliedComponent(conn, obj, lo.FromPtr(ownerRef), nodeName, serviceID, serverSHA) + facts := insights.Extract(&obj, state == ComponentStateRunning, state == ComponentStateFailed, message, time.Now()) + return in.upsertAppliedComponent(conn, obj, lo.FromPtr(ownerRef), nodeName, serviceID, serverSHA, state, facts) } -func (in *DatabaseStore) upsertAppliedComponent(conn *sqlite.Conn, obj unstructured.Unstructured, ownerRef, nodeName, serviceID, serverSHA string) error { +func (in *DatabaseStore) upsertAppliedComponent(conn *sqlite.Conn, obj unstructured.Unstructured, ownerRef, nodeName, serviceID, serverSHA string, state ComponentState, facts insights.Facts) error { labels, err := encodeComponentLabels(obj) if err != nil { return err @@ -301,7 +315,7 @@ func (in *DatabaseStore) upsertAppliedComponent(conn *sqlite.Conn, obj unstructu gvk.Kind, obj.GetNamespace(), obj.GetName(), - NewComponentState(common.ToStatus(&obj)), + state, nodeName, obj.GetCreationTimestamp().Unix(), serviceID, @@ -309,6 +323,15 @@ func (in *DatabaseStore) upsertAppliedComponent(conn *sqlite.Conn, obj unstructu serverSHA, true, labels, + facts.Criticality, + facts.CriticalityReason, + facts.PriorityClass, + facts.FailingSince, + nullablePtr(facts.DesiredReplicas), + nullablePtr(facts.ReadyReplicas), + facts.PriorityAnnotation, + facts.HealthMessage, + facts.LastSuccessfulTime, }, }) } @@ -349,7 +372,16 @@ func (in *DatabaseStore) SaveComponents(objects []unstructured.Unstructured) err delete_phase, server_sha, applied, - labels + labels, + criticality, + criticality_reason, + priority_class, + failing_since, + desired_replicas, + ready_replicas, + priority_annotation, + health_message, + last_success ) VALUES `) valueStrings := make([]string, 0, len(objects)) @@ -358,7 +390,8 @@ func (in *DatabaseStore) SaveComponents(objects []unstructured.Unstructured) err var nodeName string gvk := obj.GroupVersionKind() serviceID := smcommon.GetOwningInventory(obj) - state := NewComponentState(common.ToStatus(&obj)) + status, message := common.ToStatusWithMessage(&obj) + state := NewComponentState(status) if gvk.Group == "" && gvk.Kind == common.PodKind { if nodeName, _, _ = unstructured.NestedString(obj.Object, "spec", "nodeName"); len(nodeName) == 0 { @@ -397,7 +430,9 @@ func (in *DatabaseStore) SaveComponents(objects []unstructured.Unstructured) err continue } - valueStrings = append(valueStrings, fmt.Sprintf("('%s','%s','%s','%s','%s','%s','%s',%d,'%s',%d,'%s','%s','%s', 1, '%s')", + facts := insights.Extract(&obj, state == ComponentStateRunning, state == ComponentStateFailed, message, now) + + valueStrings = append(valueStrings, fmt.Sprintf("('%s','%s','%s','%s','%s','%s','%s',%d,'%s',%d,'%s','%s','%s', 1, '%s', %d,'%s','%s',%d,%s,%s,'%s','%s',%d)", obj.GetUID(), lo.FromPtr(ownerRef), gvk.Group, @@ -412,6 +447,15 @@ func (in *DatabaseStore) SaveComponents(objects []unstructured.Unstructured) err smcommon.GetDeletePhase(obj), serverSHA, escapeSQLString(labels), + facts.Criticality, + escapeSQLString(facts.CriticalityReason), + escapeSQLString(facts.PriorityClass), + facts.FailingSince, + sqlPtr(facts.DesiredReplicas), + sqlPtr(facts.ReadyReplicas), + escapeSQLString(facts.PriorityAnnotation), + escapeSQLString(facts.HealthMessage), + facts.LastSuccessfulTime, )) } @@ -422,15 +466,15 @@ func (in *DatabaseStore) SaveComponents(objects []unstructured.Unstructured) err sb.WriteString(strings.Join(valueStrings, ",")) sb.WriteString(` ON CONFLICT("group", version, kind, namespace, name) DO UPDATE SET uid = excluded.uid, - parent_uid = excluded.parent_uid, + ` + ownershipUpsert + ` health = excluded.health, node = excluded.node, created_at = excluded.created_at, - service_id = excluded.service_id, delete_phase = excluded.delete_phase, server_sha = excluded.server_sha, applied = excluded.applied, - labels = excluded.labels + labels = excluded.labels, + ` + factsUpsert + ` `) in.maybeSaveHookComponents(conn, objects) @@ -541,8 +585,18 @@ func (in *DatabaseStore) SaveComponentAttributes(obj client.ComponentChildAttrib args = []any{nil, nil, nil} } + state := NewComponentState(obj.State) + var criticality insights.Criticality + if state != ComponentStateRunning || insights.IsRootKind(obj.Kind) { + criticality = insights.Classify(obj.Kind, lo.FromPtr(obj.Namespace), obj.Name, "") + } + var failingSince int64 + if state == ComponentStateFailed { + failingSince = time.Now().Unix() + } + return sqlitex.ExecuteTransient(conn, setComponent, &sqlitex.ExecOptions{ - Args: append([]interface{}{ + Args: append(append([]interface{}{ obj.UID, lo.FromPtr(obj.ParentUID), lo.FromPtr(obj.Group), @@ -550,9 +604,9 @@ func (in *DatabaseStore) SaveComponentAttributes(obj client.ComponentChildAttrib obj.Kind, lo.FromPtr(obj.Namespace), obj.Name, - NewComponentState(obj.State), + state, true, - }, args...), + }, args...), criticality.Score, criticality.Reason, failingSince), }) } @@ -802,6 +856,27 @@ func (in *DatabaseStore) GetAppliedComponentByUID(uid types.UID) (result *client return result, err } +func (in *DatabaseStore) GetPriorityAnnotation(uid types.UID) (annotation string, found bool, err error) { + conn, cancelFunc, err := in.take() + if err != nil { + return "", false, err + } + defer func() { + in.pool.Put(conn) + cancelFunc() + }() + + err = sqlitex.ExecuteTransient(conn, getPriorityAnnotation, &sqlitex.ExecOptions{ + Args: []interface{}{uid}, + ResultFunc: func(stmt *sqlite.Stmt) error { + annotation, found = stmt.ColumnText(0), true + return nil + }, + }) + + return annotation, found, err +} + func (in *DatabaseStore) GetAppliedComponentsByGVK(gvk schema.GroupVersionKind) (result []smcommon.Component, err error) { conn, cancelFunc, err := in.take() if err != nil { @@ -1080,36 +1155,125 @@ func (in *DatabaseStore) GetServiceComponentsWithChildren(serviceID string, only return result, nil } -func (in *DatabaseStore) GetComponentInsights() (result []client.ClusterInsightComponentAttributes, err error) { +// GetFailingRoots returns the roots of all failures in the cluster, see failingRoots. +func (in *DatabaseStore) GetFailingRoots() (result []insights.Root, err error) { conn, cancelFunc, err := in.take() if err != nil { - return result, err + return nil, err } defer func() { in.pool.Put(conn) cancelFunc() }() - err = sqlitex.ExecuteTransient(conn, failedComponents, &sqlitex.ExecOptions{ - ResultFunc: func(stmt *sqlite.Stmt) error { - name := stmt.ColumnText(5) - namespace := stmt.ColumnText(4) - kind := stmt.ColumnText(3) - result = append(result, client.ClusterInsightComponentAttributes{ - Group: lo.ToPtr(stmt.ColumnText(1)), - Version: stmt.ColumnText(2), - Kind: kind, - Namespace: lo.EmptyableToPtr(namespace), - Name: name, - Priority: InsightComponentPriority(name, namespace, kind), - }) - return nil - }, + // both statements must see the same state of the store + err = withReadTransaction(conn, func() error { + err := sqlitex.ExecuteTransient(conn, failingRoots, &sqlitex.ExecOptions{ + ResultFunc: func(stmt *sqlite.Stmt) error { + root, err := scanRoot(stmt) + if err != nil { + return err + } + result = append(result, root) + return nil + }, + }) + if err != nil || len(result) == 0 { + return err + } + + byUID := make(map[string]*insights.Root, len(result)) + for i := range result { + byUID[result[i].UID] = &result[i] + } + + return sqlitex.ExecuteTransient(conn, failingRootsDescendants, &sqlitex.ExecOptions{ + ResultFunc: func(stmt *sqlite.Stmt) error { + root, ok := byUID[stmt.ColumnText(0)] + if !ok { + return nil + } + if root.Descendants == nil { + root.Descendants = make(map[string]insights.Descendants) + } + root.Descendants[stmt.ColumnText(1)] = insights.Descendants{ + Failed: int(stmt.ColumnInt64(2)), + Pending: int(stmt.ColumnInt64(3)), + } + return nil + }, + }) }) return result, err } +// rootState is failed when the root itself is failing, and pending otherwise, as a root is either failing or owns +// something that is. The stored state of a healthy owner of a failing object must not leak to scripts. +func rootState(state ComponentState) string { + if state == ComponentStateFailed { + return "failed" + } + + return "pending" +} + +func scanRoot(stmt *sqlite.Stmt) (insights.Root, error) { + labels, err := decodeComponentLabels(stmt.ColumnText(16)) + if err != nil { + return insights.Root{}, fmt.Errorf("failed to decode labels of %s: %w", stmt.ColumnText(5), err) + } + + root := insights.Root{ + UID: stmt.ColumnText(0), + Group: stmt.ColumnText(1), + Version: stmt.ColumnText(2), + Kind: stmt.ColumnText(3), + Namespace: stmt.ColumnText(4), + Name: stmt.ColumnText(5), + ServiceID: stmt.ColumnText(6), + PriorityClass: stmt.ColumnText(7), + HealthMessage: stmt.ColumnText(8), + CreatedAt: unixTime(stmt.ColumnInt64(11)), + FailingSince: unixTime(stmt.ColumnInt64(12)), + Criticality: int(stmt.ColumnInt64(13)), + CriticalityReason: stmt.ColumnText(14), + PriorityAnnotation: stmt.ColumnText(15), + Labels: labels, + State: rootState(ComponentState(stmt.ColumnInt32(17))), + } + if root.Criticality == 0 { + // Only failing objects and root kinds are classified when they are written. The top-most owner of a failure can be + // a healthy object of any other kind, like a ReplicaSet owned by a Rollout. + c := insights.Classify(root.Kind, root.Namespace, root.Name, root.PriorityClass) + root.Criticality, root.CriticalityReason = c.Score, c.Reason + } + if stmt.ColumnType(9) != sqlite.TypeNull { + root.DesiredReplicas = new(stmt.ColumnInt64(9)) + } + if stmt.ColumnType(10) != sqlite.TypeNull { + root.ReadyReplicas = new(stmt.ColumnInt64(10)) + } + return root, nil +} + +func unixTime(sec int64) time.Time { + if sec <= 0 { + return time.Time{} + } + return time.Unix(sec, 0) +} + +// GetComponentInsights returns the failing roots with their built-in priorities. +func (in *DatabaseStore) GetComponentInsights() ([]client.ClusterInsightComponentAttributes, error) { + roots, err := in.GetFailingRoots() + if err != nil { + return nil, err + } + + return insights.Attributes(insights.Summarize(roots, nil, time.Now()).Entries), nil +} + func (in *DatabaseStore) GetComponentCounts() (nodeCount, namespaceCount int64, err error) { conn, cancelFunc, err := in.take() if err != nil { @@ -1155,24 +1319,14 @@ func (in *DatabaseStore) GetNodeStatistics() ([]*client.NodeStatisticAttributes, return result, err } +// GetHealthScore returns the health score computed from the failing roots with their built-in priorities. func (in *DatabaseStore) GetHealthScore() (int64, error) { - conn, cancelFunc, err := in.take() + roots, err := in.GetFailingRoots() if err != nil { return 0, err } - defer func() { - in.pool.Put(conn) - cancelFunc() - }() - var ratio int64 - err = sqlitex.ExecuteTransient(conn, clusterHealthScore, &sqlitex.ExecOptions{ - ResultFunc: func(stmt *sqlite.Stmt) error { - ratio = stmt.ColumnInt64(0) - return nil - }, - }) - return ratio, err + return insights.Summarize(roots, nil, time.Now()).HealthScore, nil } func (in *DatabaseStore) UpdateComponentSHA(obj unstructured.Unstructured, shaType SHAType) error { diff --git a/go/deployment-operator/pkg/streamline/store/db_store_bench_test.go b/go/deployment-operator/pkg/streamline/store/db_store_bench_test.go new file mode 100644 index 0000000000..b3851df527 --- /dev/null +++ b/go/deployment-operator/pkg/streamline/store/db_store_bench_test.go @@ -0,0 +1,167 @@ +package store_test + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/api" + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/store" +) + +const ( + benchApps = 3000 + benchFailingEvery = 50 // 2% of apps are failing + benchManagedEvery = 10 + benchConfigObjects = 8000 +) + +func benchObject(gvk schema.GroupVersionKind, uid, namespace, name string, owner *unstructured.Unstructured, status map[string]any) unstructured.Unstructured { + u := unstructured.Unstructured{Object: map[string]any{}} + u.SetGroupVersionKind(gvk) + u.SetUID(types.UID(uid)) + u.SetNamespace(namespace) + u.SetName(name) + if owner != nil { + u.SetOwnerReferences([]metav1.OwnerReference{{ + APIVersion: owner.GetAPIVersion(), + Kind: owner.GetKind(), + Name: owner.GetName(), + UID: owner.GetUID(), + }}) + } + if status != nil { + _ = unstructured.SetNestedField(u.Object, status, "status") + } + return u +} + +// benchCluster builds a ~20k object cluster: Deployment -> ReplicaSet -> 2 Pods per app plus +// healthy config objects. Every benchFailingEvery-th app is failing (ProgressDeadlineExceeded, failed pods). +func benchCluster() []unstructured.Unstructured { + deploymentGVK := schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"} + replicaSetGVK := schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "ReplicaSet"} + podGVK := schema.GroupVersionKind{Version: "v1", Kind: "Pod"} + configMapGVK := schema.GroupVersionKind{Version: "v1", Kind: "ConfigMap"} + + objs := make([]unstructured.Unstructured, 0, benchApps*4+benchConfigObjects) + for i := 0; i < benchApps; i++ { + ns := fmt.Sprintf("ns-%d", i%100) + name := fmt.Sprintf("app-%d", i) + failing := i%benchFailingEvery == 0 + + deployStatus := map[string]any{"replicas": int64(2), "readyReplicas": int64(2), "availableReplicas": int64(2)} + podPhase := "Running" + if failing { + deployStatus = map[string]any{ + "replicas": int64(2), + "readyReplicas": int64(0), + "conditions": []any{map[string]any{ + "type": "Progressing", "status": "False", "reason": "ProgressDeadlineExceeded", + }}, + } + podPhase = "Failed" + } + + deploy := benchObject(deploymentGVK, "d-"+name, ns, name, nil, deployStatus) + if i%benchManagedEvery == 0 { + deploy.SetAnnotations(map[string]string{"config.k8s.io/owning-inventory": fmt.Sprintf("svc-%d", i%50)}) + } + deploy.Object["spec"] = map[string]any{"replicas": int64(2)} + rs := benchObject(replicaSetGVK, "rs-"+name, ns, name+"-rs", &deploy, nil) + objs = append(objs, deploy, rs) + for p := 0; p < 2; p++ { + pod := benchObject(podGVK, fmt.Sprintf("p-%s-%d", name, p), ns, fmt.Sprintf("%s-rs-%d", name, p), &rs, + map[string]any{"phase": podPhase}) + _ = unstructured.SetNestedField(pod.Object, "node-1", "spec", "nodeName") + objs = append(objs, pod) + } + } + for i := 0; i < benchConfigObjects; i++ { + objs = append(objs, benchObject(configMapGVK, fmt.Sprintf("cm-%d", i), fmt.Sprintf("ns-%d", i%100), fmt.Sprintf("cm-%d", i), nil, nil)) + } + return objs +} + +func newBenchStore(b *testing.B) store.Store { + b.Helper() + s, err := store.NewDatabaseStore(context.Background(), store.WithStorage(api.StorageFile)) + require.NoError(b, err) + return s +} + +// BenchmarkPingInsights measures the ping read path: the cluster health score and the insight list. +func BenchmarkPingInsights(b *testing.B) { + s := newBenchStore(b) + defer func() { _ = s.Shutdown() }() + require.NoError(b, s.SaveComponents(benchCluster())) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := s.GetHealthScore(); err != nil { + b.Fatal(err) + } + if _, err := s.GetComponentInsights(); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSaveComponent measures a single watch-event upsert, cycling through healthy and failing objects of all kinds. +func BenchmarkSaveComponent(b *testing.B) { + s := newBenchStore(b) + defer func() { _ = s.Shutdown() }() + objs := benchCluster() + require.NoError(b, s.SaveComponents(objs)) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := s.SaveComponent(objs[i%len(objs)]); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkSaveComponents measures the bulk initial-list save. +func BenchmarkSaveComponents(b *testing.B) { + objs := benchCluster() + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + s := newBenchStore(b) + b.StartTimer() + if err := s.SaveComponents(objs); err != nil { + b.Fatal(err) + } + b.StopTimer() + _ = s.Shutdown() + } +} + +// BenchmarkSaveUnsyncedComponents measures the save path used before manifests are applied. +func BenchmarkSaveUnsyncedComponents(b *testing.B) { + objs := benchCluster() + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + s := newBenchStore(b) + b.StartTimer() + if err := s.SaveUnsyncedComponents(objs); err != nil { + b.Fatal(err) + } + b.StopTimer() + _ = s.Shutdown() + } +} diff --git a/go/deployment-operator/pkg/streamline/store/db_store_insights_test.go b/go/deployment-operator/pkg/streamline/store/db_store_insights_test.go index e40dcb84e7..2d7e29d75a 100644 --- a/go/deployment-operator/pkg/streamline/store/db_store_insights_test.go +++ b/go/deployment-operator/pkg/streamline/store/db_store_insights_test.go @@ -2,18 +2,20 @@ package store_test import ( "context" + "fmt" "sort" "testing" + "time" - "github.com/pluralsh/console/go/client" - "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/api" - "github.com/pluralsh/console/go/polly/algorithms" "github.com/samber/lo" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "github.com/pluralsh/console/go/client" + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/api" "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/store" + "github.com/pluralsh/console/go/polly/algorithms" ) func createComponentAttributes(uid string, parentUID *string, option ...CreateComponentAttributesOption) client.ComponentChildAttributes { @@ -87,7 +89,7 @@ func TestComponentInsights(t *testing.T) { // 1-level Failed components createComponentAttributes("app-redis-1", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-redis-1")), - createComponentAttributes("app-cronjob-1", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-cronjob-1")), + createComponentAttributes("app-worker-1", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-worker-1")), // Pending component createComponentAttributes("app-migration-1", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStatePending), WithAttributesName("app-migration-1")), @@ -96,7 +98,7 @@ func TestComponentInsights(t *testing.T) { createComponentAttributes("app-ingress-1", nil, WithAttributesKind("Ingress"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-ingress-1")), createComponentAttributes("app-certificate-1", lo.ToPtr("app-ingress-1"), WithAttributesKind("Certificate"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-certificate-1")), - // Ingress (pending) -> Certificate (failed) + // Ingress (pending) -> Certificate (failed): the Certificate is the root of its own failure, the Ingress is not failing createComponentAttributes("app-ingress-2", nil, WithAttributesKind("Ingress"), WithAttributesState(client.ComponentStatePending), WithAttributesName("app-ingress-2")), createComponentAttributes("app-certificate-2", lo.ToPtr("app-ingress-2"), WithAttributesKind("Certificate"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-certificate-2")), @@ -110,18 +112,18 @@ func TestComponentInsights(t *testing.T) { createComponentAttributes("app-deployment-1", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStatePending), WithAttributesName("app-deployment-1")), createComponentAttributes("app-pod-1", lo.ToPtr("app-deployment-1"), WithAttributesKind("Pod"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-pod-1")), - // CRD (pending) -> Deployment (pending) -> Pod (failed) - createComponentAttributes("app-crd-1", nil, WithAttributesKind("CustomResourceDefinition"), WithAttributesState(client.ComponentStatePending), WithAttributesName("app-crd-1")), + // Widget CR (pending) -> Deployment (pending) -> Pod (failed) + createComponentAttributes("app-crd-1", nil, WithAttributesKind("Widget"), WithAttributesState(client.ComponentStatePending), WithAttributesName("app-crd-1")), createComponentAttributes("app-deployment-2", lo.ToPtr("app-crd-1"), WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStatePending), WithAttributesName("app-deployment-2")), createComponentAttributes("app-pod-2", lo.ToPtr("app-deployment-2"), WithAttributesKind("Pod"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-pod-2")), - // CRD (failed) -> Deployment (failed) -> Pod (failed) - createComponentAttributes("app-crd-2", nil, WithAttributesKind("CustomResourceDefinition"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-crd-2")), + // Widget CR (failed) -> Deployment (failed) -> Pod (failed) + createComponentAttributes("app-crd-2", nil, WithAttributesKind("Widget"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-crd-2")), createComponentAttributes("app-deployment-3", lo.ToPtr("app-crd-2"), WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-deployment-3")), createComponentAttributes("app-pod-3", lo.ToPtr("app-deployment-3"), WithAttributesKind("Pod"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-pod-3")), - // CRD (failed) -> Deployment (failed) -> ReplicaSet (failed) -> Pod (failed) - createComponentAttributes("app-crd-3", nil, WithAttributesKind("CustomResourceDefinition"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-crd-3")), + // Widget CR (failed) -> Deployment (failed) -> ReplicaSet (failed) -> Pod (failed) + createComponentAttributes("app-crd-3", nil, WithAttributesKind("Widget"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-crd-3")), createComponentAttributes("app-deployment-4", lo.ToPtr("app-crd-3"), WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-deployment-4")), createComponentAttributes("app-replicaset-1", lo.ToPtr("app-deployment-4"), WithAttributesKind("ReplicaSet"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-replicaset-1")), createComponentAttributes("app-pod-4", lo.ToPtr("app-replicaset-1"), WithAttributesKind("Pod"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-pod-4")), @@ -135,16 +137,17 @@ func TestComponentInsights(t *testing.T) { expectedComponents := []string{ "app-redis-1", - "app-cronjob-1", + "app-worker-1", "app-ingress-1", "app-certificate-1", - "app-ingress-2", "app-certificate-2", "app-statefulset-1", "app-daemonset-1", "app-deployment-1", "app-deployment-2", + "app-crd-2", "app-deployment-3", + "app-crd-3", "app-deployment-4", "app-deployment-5", } @@ -187,31 +190,31 @@ func TestComponentInsights(t *testing.T) { // Define test components with various kinds to test priority assignment testComponents := []client.ComponentChildAttributes{ - // Critical priority resources + // An app's Ingress or Certificate is MEDIUM, apps are never above that on their own createComponentAttributes("ingress-1", nil, WithAttributesKind("Ingress"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("ingress-1")), createComponentAttributes("certificate-1", nil, WithAttributesKind("Certificate"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("certificate-1")), + + // Core components by name or namespace. Without replica counts a failure is HIGH, a mostly down one is CRITICAL createComponentAttributes("cert-manager-1", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("cert-manager-webhook"), WithAttributesNamespace("cert-manager")), createComponentAttributes("coredns-1", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("coredns")), - // High priority resources + // Third party workloads, whatever their kind, are LOW without replica counts createComponentAttributes("statefulset-1", nil, WithAttributesKind("StatefulSet"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("statefulset-1")), createComponentAttributes("node-exporter", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("node-exporter")), - // Medium priority resources createComponentAttributes("daemonset-1", nil, WithAttributesKind("DaemonSet"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("daemonset-1")), - // Low priority resources (default) createComponentAttributes("deployment-1", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("deployment-1")), } expectedComponentPriorityMap := map[string]client.InsightComponentPriority{ - "ingress-1": client.InsightComponentPriorityCritical, - "certificate-1": client.InsightComponentPriorityCritical, - "cert-manager-webhook": client.InsightComponentPriorityCritical, - "coredns": client.InsightComponentPriorityCritical, - "statefulset-1": client.InsightComponentPriorityHigh, - "node-exporter": client.InsightComponentPriorityHigh, - "daemonset-1": client.InsightComponentPriorityMedium, + "ingress-1": client.InsightComponentPriorityMedium, + "certificate-1": client.InsightComponentPriorityMedium, + "cert-manager-webhook": client.InsightComponentPriorityHigh, + "coredns": client.InsightComponentPriorityHigh, + "statefulset-1": client.InsightComponentPriorityLow, + "node-exporter": client.InsightComponentPriorityLow, + "daemonset-1": client.InsightComponentPriorityLow, "deployment-1": client.InsightComponentPriorityLow, } @@ -236,7 +239,7 @@ func TestComponentInsights(t *testing.T) { } }) - t.Run("should assign priorities based on string similarity in resource names and namespaces", func(t *testing.T) { + t.Run("should assign priorities based on token matches in resource names and namespaces", func(t *testing.T) { storeInstance, err := store.NewDatabaseStore(context.Background(), store.WithStorage(api.StorageFile)) assert.NoError(t, err) defer func() { @@ -257,9 +260,9 @@ func TestComponentInsights(t *testing.T) { // Components with namespaces containing priority keywords createComponentAttributes("app-in-cert-manager", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-in-sensitive-ns-1"), WithAttributesNamespace("cert-manager")), - createComponentAttributes("app-in-kube-proxy", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-in-sensitive-ns-2"), WithAttributesNamespace("kube-proxy")), + createComponentAttributes("app-in-kube-system", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-in-sensitive-ns-2"), WithAttributesNamespace("kube-system")), - // Components with partial name matches to high priority resources + // Names that only share some words with a high priority resource are not matched createComponentAttributes("node-exporter-similar", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("node-metrics-exporter"), WithAttributesNamespace("monitoring")), // Components with no special priority that could be slightly similar to other resources @@ -268,15 +271,16 @@ func TestComponentInsights(t *testing.T) { } expectedComponentPriorityMap := map[string]client.InsightComponentPriority{ - "cert-manager": client.InsightComponentPriorityCritical, - "core-dns": client.InsightComponentPriorityCritical, - "istio-ingressgateway": client.InsightComponentPriorityCritical, - "linkerd-proxy": client.InsightComponentPriorityCritical, - "ebs-csi-node": client.InsightComponentPriorityCritical, - "gce-pd-csi-controller-sa": client.InsightComponentPriorityCritical, - "app-in-sensitive-ns-1": client.InsightComponentPriorityCritical, - "app-in-sensitive-ns-2": client.InsightComponentPriorityCritical, - "node-metrics-exporter": client.InsightComponentPriorityHigh, + // everything that belongs to a core component or namespace is HIGH without replica counts + "cert-manager": client.InsightComponentPriorityHigh, + "core-dns": client.InsightComponentPriorityHigh, + "istio-ingressgateway": client.InsightComponentPriorityHigh, + "linkerd-proxy": client.InsightComponentPriorityHigh, + "ebs-csi-node": client.InsightComponentPriorityHigh, + "gce-pd-csi-controller-sa": client.InsightComponentPriorityHigh, + "app-in-sensitive-ns-1": client.InsightComponentPriorityHigh, + "app-in-sensitive-ns-2": client.InsightComponentPriorityHigh, + "node-metrics-exporter": client.InsightComponentPriorityLow, "iso-ist-app": client.InsightComponentPriorityLow, "custom-cert-provisioner": client.InsightComponentPriorityLow, } @@ -316,10 +320,10 @@ func TestComponentInsights(t *testing.T) { insights, err := storeInstance.GetComponentInsights() require.NoError(t, err, "Failed to get component insights from empty cache") - require.Nil(t, insights, "Expected non-nil insights object from empty cache") + require.Nil(t, insights, "Expected nil insights from empty cache") }) - t.Run("should exclude failed components managed by plural services", func(t *testing.T) { + t.Run("should include failed components managed by plural services with their service id", func(t *testing.T) { storeInstance, err := store.NewDatabaseStore(context.Background(), store.WithStorage(api.StorageFile)) assert.NoError(t, err) defer func() { @@ -328,59 +332,39 @@ func TestComponentInsights(t *testing.T) { } }() - // Define test components where some are managed by plural services - testComponents := []client.ComponentChildAttributes{ - // Failed deployment NOT managed by a plural service (should be included) + unmanaged := []client.ComponentChildAttributes{ createComponentAttributes("app-unmanaged-1", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-unmanaged-1")), - - // Failed deployment managed by plural service with service_id (should be excluded) - createComponentAttributes("app-plural-1", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-plural-1")), - - // Failed ingress NOT managed by plural service (should be included) createComponentAttributes("ingress-unmanaged", nil, WithAttributesKind("Ingress"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("ingress-unmanaged")), - - // Failed ingress managed by plural service (should be excluded) + } + managed := []client.ComponentChildAttributes{ + createComponentAttributes("app-plural-1", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-plural-1")), createComponentAttributes("ingress-plural", nil, WithAttributesKind("Ingress"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("ingress-plural")), - - // Running component managed by plural service (should not be in insights anyway) + // healthy components never show up createComponentAttributes("app-plural-running", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateRunning), WithAttributesName("app-plural-running")), } - expectedComponentNames := []string{ - "app-unmanaged-1", - "ingress-unmanaged", - } - - // Insert test components without service_id (unmanaged) - for _, tc := range []client.ComponentChildAttributes{testComponents[0], testComponents[2]} { - err := storeInstance.SaveComponentAttributes(tc) - require.NoError(t, err, "Failed to add unmanaged component to cache") + for _, tc := range unmanaged { + require.NoError(t, storeInstance.SaveComponentAttributes(tc), "Failed to add unmanaged component to cache") } - - // Insert test components with service_id (managed by plural service) // The SaveComponentAttributes signature accepts args for node, created_at, and service_id - for _, tc := range []client.ComponentChildAttributes{testComponents[1], testComponents[3], testComponents[4]} { - err := storeInstance.SaveComponentAttributes(tc, nil, nil, "plural-service-1") - require.NoError(t, err, "Failed to add plural-managed component to cache") + for _, tc := range managed { + require.NoError(t, storeInstance.SaveComponentAttributes(tc, nil, nil, "plural-service-1"), "Failed to add plural-managed component to cache") } - // Get component insights insights, err := storeInstance.GetComponentInsights() require.NoError(t, err, "Failed to get component insights") - actualNames := algorithms.Map( - insights, - func(i client.ClusterInsightComponentAttributes) string { return i.Name }, - ) - - sort.Strings(actualNames) - sort.Strings(expectedComponentNames) + serviceIDs := make(map[string]string, len(insights)) + for _, insight := range insights { + serviceIDs[insight.Name] = lo.FromPtr(insight.ServiceID) + } - require.Equal(t, - expectedComponentNames, - actualNames, - "Components managed by plural services should be excluded from insights", - ) + require.Equal(t, map[string]string{ + "app-unmanaged-1": "", + "ingress-unmanaged": "", + "app-plural-1": "plural-service-1", + "ingress-plural": "plural-service-1", + }, serviceIDs, "Managed failures should be reported with the id of the service managing them") }) t.Run("should include all unmanaged failed components in insights", func(t *testing.T) { @@ -452,7 +436,6 @@ func TestComponentInsights(t *testing.T) { } }() - // Define test components with mixed service management testComponents := []struct { component client.ComponentChildAttributes serviceID string // empty string means unmanaged @@ -460,11 +443,9 @@ func TestComponentInsights(t *testing.T) { // Unmanaged failed components { component: createComponentAttributes("app-unmanaged-1", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-unmanaged-1")), - serviceID: "", }, { component: createComponentAttributes("app-unmanaged-2", nil, WithAttributesKind("Ingress"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-unmanaged-2")), - serviceID: "", }, // Service1-managed failed components @@ -477,55 +458,360 @@ func TestComponentInsights(t *testing.T) { serviceID: "service-1", }, - // Service2-managed failed components + // Service2-managed failed component { component: createComponentAttributes("app-service2-1", nil, WithAttributesKind("StatefulSet"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-service2-1")), serviceID: "service-2", }, - // Service1-managed pending component with unmanaged failed child (that will be skipped because it is a Pod) + // Service1-managed pending root with a failed pod. The pod has no service id of its own, it is + // reported through its root (never as a separate row) and inherits the root's service. { component: createComponentAttributes("app-service1-parent", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStatePending), WithAttributesName("app-service1-parent")), serviceID: "service-1", }, { - component: createComponentAttributes("app-service1-child", lo.ToPtr("app-service1-parent"), WithAttributesKind("Pod"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-service1-child")), - serviceID: "", + component: createComponentAttributes("app-service1-child", new("app-service1-parent"), WithAttributesKind("Pod"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("app-service1-child")), }, } - expectedComponentNames := []string{ - "app-unmanaged-1", - "app-unmanaged-2", - } - - // Insert all test components with their respective service_id for _, tc := range testComponents { if tc.serviceID == "" { - err := storeInstance.SaveComponentAttributes(tc.component) - require.NoError(t, err, "Failed to add unmanaged component to cache") + require.NoError(t, storeInstance.SaveComponentAttributes(tc.component), "Failed to add unmanaged component to cache") } else { - err := storeInstance.SaveComponentAttributes(tc.component, nil, nil, tc.serviceID) - require.NoError(t, err, "Failed to add managed component to cache") + require.NoError(t, storeInstance.SaveComponentAttributes(tc.component, nil, nil, tc.serviceID), "Failed to add managed component to cache") } } - // Get component insights insights, err := storeInstance.GetComponentInsights() require.NoError(t, err, "Failed to get component insights") - actualNames := algorithms.Map( - insights, - func(i client.ClusterInsightComponentAttributes) string { return i.Name }, - ) + serviceIDs := make(map[string]string, len(insights)) + for _, insight := range insights { + serviceIDs[insight.Name] = lo.FromPtr(insight.ServiceID) + } - sort.Strings(actualNames) - sort.Strings(expectedComponentNames) + require.Equal(t, map[string]string{ + "app-unmanaged-1": "", + "app-unmanaged-2": "", + "app-service1-1": "service-1", + "app-service1-2": "service-1", + "app-service2-1": "service-2", + "app-service1-parent": "service-1", + }, serviceIDs, "Managed and unmanaged failures should both be reported, each with its own service id") + }) - require.Equal(t, - expectedComponentNames, - actualNames, - "Only unmanaged components should appear in insights when mixed with managed ones", - ) + t.Run("should not elevate priority of unrelated components that only fuzzy match critical names", func(t *testing.T) { + storeInstance, err := store.NewDatabaseStore(context.Background(), store.WithStorage(api.StorageFile)) + assert.NoError(t, err) + defer func() { + if err := storeInstance.Shutdown(); err != nil { + t.Fatalf("Failed to close component cache: %v", err) + } + }() + + // Business workloads whose names contain the letters of a critical keyword in order + testComponents := []client.ComponentChildAttributes{ + // c-o-r-e...d-n-s -> "coredns" + createComponentAttributes("core-banking-dns-proxy", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("core-banking-dns-proxy"), WithAttributesNamespace("bank")), + // con-cert-manager -> "certmanager" + createComponentAttributes("concert-manager", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("concert-manager"), WithAttributesNamespace("events")), + // kube-rnetes-proxy -> "kubeproxy" + createComponentAttributes("kubernetes-proxy-docs", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("kubernetes-proxy-docs"), WithAttributesNamespace("docs")), + } + + for _, tc := range testComponents { + err := storeInstance.SaveComponentAttributes(tc) + require.NoError(t, err, "Failed to add component %s to cache", tc.UID) + } + + insights, err := storeInstance.GetComponentInsights() + require.NoError(t, err, "Failed to get component insights") + + priorityMap := make(map[string]client.InsightComponentPriority) + for _, insight := range insights { + priorityMap[insight.Name] = *insight.Priority + } + + for _, tc := range testComponents { + assert.Equal(t, client.InsightComponentPriorityLow, priorityMap[tc.Name], "Priority for %s should be %s", tc.Name, client.InsightComponentPriorityLow) + } + }) + + t.Run("should assign high priority to cluster networking components", func(t *testing.T) { + storeInstance, err := store.NewDatabaseStore(context.Background(), store.WithStorage(api.StorageFile)) + assert.NoError(t, err) + defer func() { + if err := storeInstance.Shutdown(); err != nil { + t.Fatalf("Failed to close component cache: %v", err) + } + }() + + // When any of these is down, pod networking or all ingress traffic is down for the whole cluster. + // Without replica counts the failure is HIGH, the tests below cover a component that is mostly down. + testComponents := []client.ComponentChildAttributes{ + // AWS VPC CNI + createComponentAttributes("aws-node", nil, WithAttributesKind("DaemonSet"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("aws-node"), WithAttributesNamespace("kube-system")), + createComponentAttributes("cilium", nil, WithAttributesKind("DaemonSet"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("cilium"), WithAttributesNamespace("kube-system")), + createComponentAttributes("calico-node", nil, WithAttributesKind("DaemonSet"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("calico-node"), WithAttributesNamespace("calico-system")), + createComponentAttributes("ingress-nginx-controller", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("ingress-nginx-controller"), WithAttributesNamespace("ingress-nginx")), + } + + for _, tc := range testComponents { + err := storeInstance.SaveComponentAttributes(tc) + require.NoError(t, err, "Failed to add component %s to cache", tc.UID) + } + + insights, err := storeInstance.GetComponentInsights() + require.NoError(t, err, "Failed to get component insights") + + priorityMap := make(map[string]client.InsightComponentPriority) + for _, insight := range insights { + priorityMap[insight.Name] = *insight.Priority + } + + for _, tc := range testComponents { + assert.Equal(t, client.InsightComponentPriorityHigh, priorityMap[tc.Name], "Priority for %s should be %s", tc.Name, client.InsightComponentPriorityHigh) + } + }) + + t.Run("should only make core components critical and only when most of them are down", func(t *testing.T) { + storeInstance, err := store.NewDatabaseStore(context.Background(), store.WithStorage(api.StorageFile)) + assert.NoError(t, err) + defer func() { + if err := storeInstance.Shutdown(); err != nil { + t.Fatalf("Failed to close component cache: %v", err) + } + }() + + // Running pods without a service are not stored, so every workload has one stored failed pod and only the + // desired and ready replica counts of the Deployment tell how much of it is down. + for _, w := range []struct { + name string + desired, ready int64 + }{ + {name: "coredns", desired: 2, ready: 0}, // core, fully down + {name: "ingress-nginx-controller", desired: 4, ready: 1}, // core, three quarters down + {name: "cert-manager-webhook", desired: 2, ready: 1}, // core, half down + {name: "orders-api", desired: 3, ready: 0}, // app, fully down + {name: "orders-worker", desired: 3, ready: 2}, // app, partly down + } { + saveProgressingDeployment(t, storeInstance, w.name, w.desired, w.ready) + require.NoError(t, storeInstance.SaveComponentAttributes(createComponentAttributes(w.name+"-rs", new(w.name), WithAttributesKind("ReplicaSet"), WithAttributesState(client.ComponentStatePending), WithAttributesName(w.name+"-rs")))) + require.NoError(t, storeInstance.SaveComponentAttributes(createComponentAttributes(w.name+"-pod", new(w.name+"-rs"), WithAttributesKind("Pod"), WithAttributesState(client.ComponentStateFailed), WithAttributesName(w.name+"-pod")))) + } + + // A core DaemonSet that is down on every node, its pods are owned by the DaemonSet itself + daemonSet := unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "apps/v1", + "kind": "DaemonSet", + "metadata": map[string]any{"name": "aws-node", "namespace": "kube-system", "uid": "aws-node", "generation": int64(1)}, + "status": map[string]any{"desiredNumberScheduled": int64(3), "numberReady": int64(0), "observedGeneration": int64(1)}, + }} + require.NoError(t, storeInstance.SaveComponent(daemonSet)) + require.NoError(t, storeInstance.SaveComponentAttributes(createComponentAttributes("aws-node-pod", new("aws-node"), WithAttributesKind("Pod"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("aws-node-pod")))) + + insights, err := storeInstance.GetComponentInsights() + require.NoError(t, err, "Failed to get component insights") + + priorityMap := make(map[string]client.InsightComponentPriority) + for _, insight := range insights { + priorityMap[insight.Name] = *insight.Priority + } + + assert.Equal(t, client.InsightComponentPriorityCritical, priorityMap["aws-node"], "a core DaemonSet that is down everywhere is CRITICAL") + assert.Equal(t, client.InsightComponentPriorityCritical, priorityMap["coredns"]) + assert.Equal(t, client.InsightComponentPriorityCritical, priorityMap["ingress-nginx-controller"]) + assert.Equal(t, client.InsightComponentPriorityHigh, priorityMap["cert-manager-webhook"]) + assert.Equal(t, client.InsightComponentPriorityMedium, priorityMap["orders-api"], "an app is at most MEDIUM, even fully down") + assert.Equal(t, client.InsightComponentPriorityLow, priorityMap["orders-worker"]) + }) + + t.Run("should rank a full outage above a partial degradation of the same workload", func(t *testing.T) { + storeInstance, err := store.NewDatabaseStore(context.Background(), store.WithStorage(api.StorageFile)) + assert.NoError(t, err) + defer func() { + if err := storeInstance.Shutdown(); err != nil { + t.Fatalf("Failed to close component cache: %v", err) + } + }() + + // Running pods without a service are not stored, so both workloads have exactly one stored failed pod. + // Only the desired and ready replica counts of the Deployment tell a full outage from a partial one. + saveProgressingDeployment(t, storeInstance, "api-down", 3, 0) + saveProgressingDeployment(t, storeInstance, "api-degraded", 3, 2) + for _, name := range []string{"api-down", "api-degraded"} { + require.NoError(t, storeInstance.SaveComponentAttributes(createComponentAttributes(name+"-rs", new(name), WithAttributesKind("ReplicaSet"), WithAttributesState(client.ComponentStatePending), WithAttributesName(name+"-rs")))) + require.NoError(t, storeInstance.SaveComponentAttributes(createComponentAttributes(name+"-pod", new(name+"-rs"), WithAttributesKind("Pod"), WithAttributesState(client.ComponentStateFailed), WithAttributesName(name+"-pod")))) + } + + insights, err := storeInstance.GetComponentInsights() + require.NoError(t, err, "Failed to get component insights") + + priorityMap := make(map[string]client.InsightComponentPriority) + for _, insight := range insights { + priorityMap[insight.Name] = *insight.Priority + } + + require.Contains(t, priorityMap, "api-down") + require.Contains(t, priorityMap, "api-degraded") + assert.Greater(t, priorityRank(priorityMap["api-down"]), priorityRank(priorityMap["api-degraded"]), + "Full outage priority (%s) should be higher than partial degradation priority (%s)", priorityMap["api-down"], priorityMap["api-degraded"]) + }) +} + +// saveProgressingDeployment stores a Deployment that is rolling out with the given desired and ready replicas, +// the state a crashlooping Deployment is in before its progress deadline passes. +func saveProgressingDeployment(t *testing.T, storeInstance store.Store, name string, desired, ready int64) { + t.Helper() + deployment := unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]any{"name": name, "namespace": testNamespace, "uid": name, "generation": int64(1)}, + "spec": map[string]any{"replicas": desired}, + "status": map[string]any{ + "observedGeneration": int64(1), + "replicas": desired, + "updatedReplicas": desired, + "readyReplicas": ready, + "availableReplicas": ready, + }, + }} + require.NoError(t, storeInstance.SaveComponent(deployment), "Failed to add deployment %s to cache", name) +} + +// saveFailingDeployment stores a Deployment that exceeded its progress deadline with the given desired and ready +// replicas, so it is a failing root by itself. +func saveFailingDeployment(t *testing.T, storeInstance store.Store, namespace, name string, desired, ready int64) { + t.Helper() + deployment := unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": map[string]any{"name": name, "namespace": namespace, "uid": namespace + "/" + name, "generation": int64(1)}, + "spec": map[string]any{"replicas": desired}, + "status": map[string]any{ + "observedGeneration": int64(1), + "replicas": desired, + "readyReplicas": ready, + "conditions": []any{map[string]any{ + "type": "Progressing", "status": "False", "reason": "ProgressDeadlineExceeded", + "lastTransitionTime": time.Now().UTC().Format(time.RFC3339), + }}, + }, + }} + require.NoError(t, storeInstance.SaveComponent(deployment), "Failed to add deployment %s/%s to cache", namespace, name) +} + +func priorityRank(priority client.InsightComponentPriority) int { + return lo.IndexOf(client.AllInsightComponentPriority, priority) +} + +func saveHealthyComponents(t *testing.T, storeInstance store.Store, count int) { + for i := range count { + name := fmt.Sprintf("healthy-%d", i) + err := storeInstance.SaveComponentAttributes(createComponentAttributes(name, nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateRunning), WithAttributesName(name))) + require.NoError(t, err, "Failed to add component %s to cache", name) + } +} + +func TestClusterHealthScore(t *testing.T) { + t.Run("should not scale deductions with the number of pods of a failing workload", func(t *testing.T) { + scoreWithFailedPods := func(pods int) int64 { + storeInstance, err := store.NewDatabaseStore(context.Background(), store.WithStorage(api.StorageFile)) + assert.NoError(t, err) + defer func() { + if err := storeInstance.Shutdown(); err != nil { + t.Fatalf("Failed to close component cache: %v", err) + } + }() + + saveHealthyComponents(t, storeInstance, 100) + + // One failing DaemonSet with a failed pod on each node + err = storeInstance.SaveComponentAttributes(createComponentAttributes("ebs-csi-node", nil, WithAttributesKind("DaemonSet"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("ebs-csi-node"), WithAttributesNamespace("kube-system"))) + require.NoError(t, err) + for i := range pods { + name := fmt.Sprintf("ebs-csi-node-%d", i) + err := storeInstance.SaveComponentAttributes(createComponentAttributes(name, new("ebs-csi-node"), WithAttributesKind("Pod"), WithAttributesState(client.ComponentStateFailed), WithAttributesName(name), WithAttributesNamespace("kube-system"))) + require.NoError(t, err) + } + + score, err := storeInstance.GetHealthScore() + require.NoError(t, err) + return score + } + + threeNodes := scoreWithFailedPods(3) + tenNodes := scoreWithFailedPods(10) + + assert.Positive(t, tenNodes, "A single failing DaemonSet should not zero the cluster health score") + assert.Equal(t, threeNodes, tenNodes, "The same failure should score the same on a 3-node and a 10-node cluster") + }) + + t.Run("should apply the critical deduction when a core component is mostly down", func(t *testing.T) { + // The critical deduction is 50 points, so with an otherwise healthy cluster the score must not exceed 50. + cases := []struct { + name string + namespace string + }{ + {name: "coredns", namespace: "kube-system"}, + {name: "coredns", namespace: "dns"}, + {name: "istiod", namespace: "istio-system"}, + } + + for _, tc := range cases { + t.Run(fmt.Sprintf("%s/%s", tc.namespace, tc.name), func(t *testing.T) { + storeInstance, err := store.NewDatabaseStore(context.Background(), store.WithStorage(api.StorageFile)) + assert.NoError(t, err) + defer func() { + if err := storeInstance.Shutdown(); err != nil { + t.Fatalf("Failed to close component cache: %v", err) + } + }() + + saveHealthyComponents(t, storeInstance, 100) + saveFailingDeployment(t, storeInstance, tc.namespace, tc.name, 2, 0) + + score, err := storeInstance.GetHealthScore() + require.NoError(t, err) + assert.Equal(t, int64(50), score, "%s in %s is fully down and should take the critical deduction", tc.name, tc.namespace) + }) + } + }) + + t.Run("should apply the high deduction when a core component is partly down", func(t *testing.T) { + storeInstance, err := store.NewDatabaseStore(context.Background(), store.WithStorage(api.StorageFile)) + assert.NoError(t, err) + defer func() { + if err := storeInstance.Shutdown(); err != nil { + t.Fatalf("Failed to close component cache: %v", err) + } + }() + + saveHealthyComponents(t, storeInstance, 100) + saveFailingDeployment(t, storeInstance, "kube-system", "coredns", 2, 1) + + score, err := storeInstance.GetHealthScore() + require.NoError(t, err) + assert.Equal(t, int64(75), score, "one of two coredns replicas down is HIGH, 25 points") + }) + + t.Run("should not take the score below 60 for third party failures", func(t *testing.T) { + storeInstance, err := store.NewDatabaseStore(context.Background(), store.WithStorage(api.StorageFile)) + assert.NoError(t, err) + defer func() { + if err := storeInstance.Shutdown(); err != nil { + t.Fatalf("Failed to close component cache: %v", err) + } + }() + + saveHealthyComponents(t, storeInstance, 100) + for i := range 15 { + saveFailingDeployment(t, storeInstance, "shop", fmt.Sprintf("app-%d", i), 3, 0) + } + + score, err := storeInstance.GetHealthScore() + require.NoError(t, err) + assert.Equal(t, int64(70), score, "total outages of apps are MEDIUM, capped at 30 points") }) } diff --git a/go/deployment-operator/pkg/streamline/store/db_store_profiled.go b/go/deployment-operator/pkg/streamline/store/db_store_profiled.go index d40753ba1f..32ff3a0cad 100644 --- a/go/deployment-operator/pkg/streamline/store/db_store_profiled.go +++ b/go/deployment-operator/pkg/streamline/store/db_store_profiled.go @@ -15,6 +15,7 @@ import ( "github.com/pluralsh/console/go/deployment-operator/pkg/log" smcommon "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/common" + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/insights" ) // ProfiledStore is a wrapper around a Store that adds Datadog tracing @@ -114,6 +115,20 @@ func (p *ProfiledStore) GetAppliedComponentByUID(uid types.UID) (*client.Compone return res, err } +// GetPriorityAnnotation wraps Store.GetPriorityAnnotation with tracing. +func (p *ProfiledStore) GetPriorityAnnotation(uid types.UID) (string, bool, error) { + var ( + annotation string + found bool + err error + ) + _ = trace(context.Background(), "GetPriorityAnnotation", func() error { + annotation, found, err = p.inner.GetPriorityAnnotation(uid) + return err + }) + return annotation, found, err +} + // GetAppliedComponentsByGVK wraps Store.GetAppliedComponentsByGVK with tracing. func (p *ProfiledStore) GetAppliedComponentsByGVK(gvk schema.GroupVersionKind) ([]smcommon.Component, error) { var ( @@ -290,6 +305,19 @@ func (p *ProfiledStore) GetHookComponents(serviceID string) ([]smcommon.HookComp return res, err } +// GetFailingRoots wraps Store.GetFailingRoots with tracing. +func (p *ProfiledStore) GetFailingRoots() ([]insights.Root, error) { + var ( + res []insights.Root + err error + ) + _ = trace(context.Background(), "GetFailingRoots", func() error { + res, err = p.inner.GetFailingRoots() + return err + }) + return res, err +} + // GetComponentInsights wraps Store.GetComponentInsights with tracing. func (p *ProfiledStore) GetComponentInsights() ([]client.ClusterInsightComponentAttributes, error) { var ( @@ -307,7 +335,7 @@ func (p *ProfiledStore) GetComponentInsights() ([]client.ClusterInsightComponent func (p *ProfiledStore) SaveComponentAttributes(obj client.ComponentChildAttributes, args ...any) error { var err error _ = trace(context.Background(), "SaveComponentAttributes", func() error { - err = p.inner.SaveComponentAttributes(obj, args) + err = p.inner.SaveComponentAttributes(obj, args...) return err }) return err diff --git a/go/deployment-operator/pkg/streamline/store/db_store_profiled_local.go b/go/deployment-operator/pkg/streamline/store/db_store_profiled_local.go index 32c0b0dc62..9e77627ff3 100644 --- a/go/deployment-operator/pkg/streamline/store/db_store_profiled_local.go +++ b/go/deployment-operator/pkg/streamline/store/db_store_profiled_local.go @@ -7,6 +7,7 @@ import ( "github.com/pluralsh/console/go/client" "github.com/pluralsh/console/go/deployment-operator/pkg/log" smcommon "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/common" + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/insights" "github.com/pluralsh/console/go/polly/containers" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -109,6 +110,20 @@ func (p *ProfiledStoreLocal) GetAppliedComponentByUID(uid types.UID) (*client.Co return res, err } +// GetPriorityAnnotation wraps Store.GetPriorityAnnotation with tracing. +func (p *ProfiledStoreLocal) GetPriorityAnnotation(uid types.UID) (string, bool, error) { + var ( + annotation string + found bool + err error + ) + _ = traceLocal(context.Background(), "GetPriorityAnnotation", func() error { + annotation, found, err = p.inner.GetPriorityAnnotation(uid) + return err + }) + return annotation, found, err +} + // GetAppliedComponentsByGVK wraps Store.GetAppliedComponentsByGVK with tracing. func (p *ProfiledStoreLocal) GetAppliedComponentsByGVK(gvk schema.GroupVersionKind) ([]smcommon.Component, error) { var ( @@ -156,6 +171,19 @@ func (p *ProfiledStoreLocal) GetServiceComponents(serviceID string, onlyApplied return res, err } +// GetFailingRoots wraps Store.GetFailingRoots with tracing. +func (p *ProfiledStoreLocal) GetFailingRoots() ([]insights.Root, error) { + var ( + res []insights.Root + err error + ) + _ = traceLocal(context.Background(), "GetFailingRoots", func() error { + res, err = p.inner.GetFailingRoots() + return err + }) + return res, err +} + // GetComponentInsights wraps Store.GetComponentInsights with tracing. func (p *ProfiledStoreLocal) GetComponentInsights() ([]client.ClusterInsightComponentAttributes, error) { var ( @@ -329,7 +357,7 @@ func (p *ProfiledStoreLocal) SetComponentUnsynced(obj unstructured.Unstructured) func (p *ProfiledStoreLocal) SaveComponentAttributes(obj client.ComponentChildAttributes, args ...any) error { var err error _ = traceLocal(context.Background(), "SaveComponentAttributes", func() error { - err = p.inner.SaveComponentAttributes(obj, args) + err = p.inner.SaveComponentAttributes(obj, args...) return err }) return err diff --git a/go/deployment-operator/pkg/streamline/store/db_store_roots_test.go b/go/deployment-operator/pkg/streamline/store/db_store_roots_test.go new file mode 100644 index 0000000000..e70dd3a663 --- /dev/null +++ b/go/deployment-operator/pkg/streamline/store/db_store_roots_test.go @@ -0,0 +1,466 @@ +package store_test + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "zombiezen.com/go/sqlite" + "zombiezen.com/go/sqlite/sqlitex" + + "github.com/pluralsh/console/go/client" + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/api" + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/insights" + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/store" +) + +func newTestStore(t *testing.T, options ...store.Option) store.Store { + t.Helper() + s, err := store.NewDatabaseStore(context.Background(), append([]store.Option{store.WithStorage(api.StorageFile)}, options...)...) + require.NoError(t, err) + t.Cleanup(func() { _ = s.Shutdown() }) + return s +} + +func rootNames(t *testing.T, s store.Store) []string { + t.Helper() + roots, err := s.GetFailingRoots() + require.NoError(t, err) + return lo.Map(roots, func(r insights.Root, _ int) string { return r.Name }) +} + +func TestFailingRoots(t *testing.T) { + t.Run("a cron job is reported only while its newest job failed", func(t *testing.T) { + s := newTestStore(t) + now := time.Now().Unix() + save := func(c client.ComponentChildAttributes, created int64) { + require.NoError(t, s.SaveComponentAttributes(c, nil, created, nil)) + } + + save(createComponentAttributes("nightly", nil, WithAttributesKind("CronJob"), WithAttributesState(client.ComponentStatePending), WithAttributesName("nightly")), now-7200) + save(createComponentAttributes("nightly-1", new("nightly"), WithAttributesKind("Job"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("nightly-1")), now-3600) + save(createComponentAttributes("nightly-2", new("nightly"), WithAttributesKind("Job"), WithAttributesState(client.ComponentStateRunning), WithAttributesName("nightly-2")), now-60) + + assert.Empty(t, rootNames(t, s), "a later successful job means the cron job recovered") + + save(createComponentAttributes("nightly-2", new("nightly"), WithAttributesKind("Job"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("nightly-2")), now-60) + + assert.ElementsMatch(t, []string{"nightly"}, rootNames(t, s)) + }) + + t.Run("a cron job is not reported for an old failed job that outlived its successful runs", func(t *testing.T) { + // successfulJobsHistoryLimit: 0 deletes successful jobs at once, so the only job left is an old failed one + s := newTestStore(t) + now := time.Now() + cronJob := func(lastSuccess *time.Time) unstructured.Unstructured { + status := map[string]any{} + if lastSuccess != nil { + status["lastSuccessfulTime"] = lastSuccess.UTC().Format(time.RFC3339) + } + return unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "batch/v1", "kind": "CronJob", + "metadata": map[string]any{"name": "nightly", "namespace": testNamespace, "uid": "nightly", "creationTimestamp": now.Add(-48 * time.Hour).UTC().Format(time.RFC3339)}, + "status": status, + }} + } + failedJob := func() { + require.NoError(t, s.SaveComponentAttributes(createComponentAttributes("nightly-1", new("nightly"), WithAttributesKind("Job"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("nightly-1")), nil, now.Add(-time.Hour).Unix(), nil)) + } + + never := cronJob(nil) + require.NoError(t, s.SaveComponent(never)) + failedJob() + assert.ElementsMatch(t, []string{"nightly"}, rootNames(t, s), "it never succeeded, so the failed job is its failure") + + afterFailure := now.Add(-30 * time.Minute) + require.NoError(t, s.SaveComponent(cronJob(&afterFailure))) + assert.Empty(t, rootNames(t, s), "a run succeeded after the retained failed job, which is only history") + + beforeFailure := now.Add(-2 * time.Hour) + require.NoError(t, s.SaveComponent(cronJob(&beforeFailure))) + assert.ElementsMatch(t, []string{"nightly"}, rootNames(t, s), "the failed job is newer than the last success, so it is a failure again") + }) + + t.Run("a cron job saved in bulk keeps its last successful run", func(t *testing.T) { + s := newTestStore(t) + now := time.Now() + cronJob := unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "batch/v1", "kind": "CronJob", + "metadata": map[string]any{"name": "nightly", "namespace": testNamespace, "uid": "nightly"}, + "status": map[string]any{"lastSuccessfulTime": now.Add(-30 * time.Minute).UTC().Format(time.RFC3339)}, + }} + require.NoError(t, s.SaveComponents([]unstructured.Unstructured{cronJob})) + require.NoError(t, s.SaveComponentAttributes(createComponentAttributes("nightly-1", new("nightly"), WithAttributesKind("Job"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("nightly-1")), nil, now.Add(-time.Hour).Unix(), nil)) + + assert.Empty(t, rootNames(t, s), "the last run succeeded after the retained failed job") + }) + + t.Run("a failed node is reported only after five minutes", func(t *testing.T) { + s := newTestStore(t) + + require.NoError(t, s.SaveComponentAttributes(createComponentAttributes("node-1", nil, WithAttributesKind("Node"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("node-1")))) + assert.Empty(t, rootNames(t, s), "the node just failed") + + node := unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "v1", "kind": "Node", + "metadata": map[string]any{"name": "node-2", "uid": "node-2"}, + "status": map[string]any{"conditions": []any{map[string]any{ + "type": "Ready", "status": "False", "lastTransitionTime": time.Now().Add(-10 * time.Minute).UTC().Format(time.RFC3339), + }}}, + }} + require.NoError(t, s.SaveComponent(node)) + + assert.ElementsMatch(t, []string{"node-2"}, rootNames(t, s)) + }) + + t.Run("a failure too deep to reach its root is ignored", func(t *testing.T) { + s := newTestStore(t) + + require.NoError(t, s.SaveComponentAttributes(createComponentAttributes("a", nil, WithAttributesKind("Widget"), WithAttributesState(client.ComponentStateRunning), WithAttributesName("a")))) + parent := "a" + for _, name := range []string{"b", "c", "d", "e"} { + require.NoError(t, s.SaveComponentAttributes(createComponentAttributes(name, new(parent), WithAttributesKind("Widget"), WithAttributesState(client.ComponentStateRunning), WithAttributesName(name)))) + parent = name + } + require.NoError(t, s.SaveComponentAttributes(createComponentAttributes("f", new(parent), WithAttributesKind("Widget"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("f")))) + + assert.Empty(t, rootNames(t, s)) + }) + + t.Run("a root inherits the service of its nearest owner", func(t *testing.T) { + s := newTestStore(t) + + require.NoError(t, s.SaveComponentAttributes(createComponentAttributes("operator-cr", nil, WithAttributesKind("Widget"), WithAttributesState(client.ComponentStateRunning), WithAttributesName("operator-cr")), nil, nil, "svc-1")) + require.NoError(t, s.SaveComponentAttributes(createComponentAttributes("created", new("operator-cr"), WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("created")))) + + roots, err := s.GetFailingRoots() + require.NoError(t, err) + require.Len(t, roots, 1) + assert.Equal(t, "svc-1", roots[0].ServiceID) + }) + + t.Run("replica counts and priority attributes are read from the object", func(t *testing.T) { + s := newTestStore(t) + + saveProgressingDeployment(t, s, "api", 5, 1) + require.NoError(t, s.SaveComponentAttributes(createComponentAttributes("api-pod", new("api"), WithAttributesKind("Pod"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("api-pod")))) + + roots, err := s.GetFailingRoots() + require.NoError(t, err) + require.Len(t, roots, 1) + assert.Equal(t, int64(5), *roots[0].DesiredReplicas) + assert.Equal(t, int64(1), *roots[0].ReadyReplicas) + }) +} + +func TestFailingRootsEdgeCases(t *testing.T) { + t.Run("an object served under several versions is one root", func(t *testing.T) { + s := newTestStore(t) + for _, version := range []string{"v1", "v2"} { + obj := unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "autoscaling/" + version, "kind": "HorizontalPodAutoscaler", + "metadata": map[string]any{"name": "web", "namespace": "apps", "uid": "hpa-uid"}, + }} + require.NoError(t, s.SaveComponent(obj)) + } + for _, version := range []string{"v1", "v2"} { + require.NoError(t, s.SaveComponentAttributes(client.ComponentChildAttributes{ + UID: "hpa-uid", Group: new("autoscaling"), Version: version, Kind: "HorizontalPodAutoscaler", + Namespace: new("apps"), Name: "web", State: new(client.ComponentStateFailed), + })) + } + + roots, err := s.GetFailingRoots() + require.NoError(t, err) + assert.Len(t, roots, 1) + + score, err := s.GetHealthScore() + require.NoError(t, err) + assert.Equal(t, int64(98), score, "one penalty, not one per served version") + }) + + t.Run("a static pod does not roll up to its node", func(t *testing.T) { + s := newTestStore(t) + require.NoError(t, s.SaveComponentAttributes(createComponentAttributes("node-1", nil, WithAttributesKind("Node"), WithAttributesState(client.ComponentStateRunning), WithAttributesName("node-1")))) + require.NoError(t, s.SaveComponentAttributes(createComponentAttributes("etcd", new("node-1"), WithAttributesKind("Pod"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("etcd-node-1"), WithAttributesNamespace("kube-system")))) + + roots, err := s.GetFailingRoots() + require.NoError(t, err) + require.Len(t, roots, 1) + assert.Equal(t, "etcd-node-1", roots[0].Name) + assert.Equal(t, insights.ScoreCore, roots[0].Criticality, "classified by its namespace even though the pod was running when it was stored") + }) + + t.Run("a healthy owner of any kind can be pinned by annotation", func(t *testing.T) { + s := newTestStore(t) + rollout := unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "argoproj.io/v1alpha1", "kind": "Rollout", + "metadata": map[string]any{"name": "web", "namespace": "apps", "uid": "rollout", "annotations": map[string]any{insights.AnnotationPriority: "IGNORE"}}, + }} + require.NoError(t, s.SaveComponent(rollout)) + require.NoError(t, s.SaveComponentAttributes(createComponentAttributes("rs", new("rollout"), WithAttributesKind("ReplicaSet"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("web-rs")))) + + roots, err := s.GetFailingRoots() + require.NoError(t, err) + require.Len(t, roots, 1) + assert.Equal(t, "IGNORE", roots[0].PriorityAnnotation) + }) +} + +func TestFailingRootDescendants(t *testing.T) { + s := newTestStore(t) + + api := deploymentExceedingDeadline("api", time.Now(), nil) + api.SetLabels(map[string]string{"tier": "core"}) + require.NoError(t, s.SaveComponent(api)) + for _, c := range []struct { + uid string + kind string + state client.ComponentState + }{ + {"api-rs", "ReplicaSet", client.ComponentStatePending}, + {"api-pod-1", "Pod", client.ComponentStateFailed}, + {"api-pod-2", "Pod", client.ComponentStateFailed}, + {"api-pod-3", "Pod", client.ComponentStatePending}, + {"api-pod-4", "Pod", client.ComponentStateRunning}, + } { + parent := "api" + if c.kind == "Pod" { + parent = "api-rs" + } + require.NoError(t, s.SaveComponentAttributes(createComponentAttributes(c.uid, new(parent), WithAttributesKind(c.kind), WithAttributesState(c.state), WithAttributesName(c.uid)))) + } + + roots, err := s.GetFailingRoots() + require.NoError(t, err) + require.Len(t, roots, 1) + assert.Equal(t, "failed", roots[0].State) + assert.Equal(t, map[string]string{"tier": "core"}, roots[0].Labels) + assert.Equal(t, map[string]insights.Descendants{ + "ReplicaSet": {Pending: 1}, + "Pod": {Failed: 2, Pending: 1}, + }, roots[0].Descendants) +} + +func TestFailingRootDescendantsServedUnderSeveralVersions(t *testing.T) { + s := newTestStore(t) + require.NoError(t, s.SaveComponent(deploymentExceedingDeadline("api", time.Now(), nil))) + + // the same objects are watched through two API versions, and so are stored twice + for _, version := range []string{"v1", "v2"} { + for _, name := range []string{"api-rs", "api-pod"} { + parent, kind := "api", "ReplicaSet" + if name == "api-pod" { + parent, kind = "api-rs", "Pod" + } + require.NoError(t, s.SaveComponentAttributes(client.ComponentChildAttributes{ + UID: name, ParentUID: new(parent), Group: new("example.com"), Version: version, Kind: kind, + Namespace: new(testNamespace), Name: name, State: new(client.ComponentStateFailed), + })) + } + } + + roots, err := s.GetFailingRoots() + require.NoError(t, err) + require.Len(t, roots, 1) + assert.Equal(t, map[string]insights.Descendants{ + "ReplicaSet": {Failed: 1}, + "Pod": {Failed: 1}, + }, roots[0].Descendants) +} + +func TestFailingRootDescendantsWithVersionsOwnedSeparately(t *testing.T) { + s := newTestStore(t) + require.NoError(t, s.SaveComponent(deploymentExceedingDeadline("api", time.Now(), nil))) + + // only the v2 row of the pod is attached to its owner, the v1 row has no parent + for _, c := range []struct{ version, parent string }{{"v1", ""}, {"v2", "api"}} { + attrs := client.ComponentChildAttributes{ + UID: "api-pod", Group: new("example.com"), Version: c.version, Kind: "Pod", + Namespace: new(testNamespace), Name: "api-pod", State: new(client.ComponentStateFailed), + } + if c.parent != "" { + attrs.ParentUID = new(c.parent) + } + require.NoError(t, s.SaveComponentAttributes(attrs)) + } + + roots, err := s.GetFailingRoots() + require.NoError(t, err) + require.NotEmpty(t, roots) + for _, r := range roots { + if r.Name == "api" { + assert.Equal(t, map[string]insights.Descendants{"Pod": {Failed: 1}}, r.Descendants) + return + } + } + t.Fatal("the deployment is not reported as a root") +} + +func TestFailingRootState(t *testing.T) { + t.Run("a healthy owner of a failed object is pending", func(t *testing.T) { + s := newTestStore(t) + require.NoError(t, s.SaveComponentAttributes(createComponentAttributes("api", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateRunning), WithAttributesName("api")))) + require.NoError(t, s.SaveComponentAttributes(createComponentAttributes("api-pod", new("api"), WithAttributesKind("Pod"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("api-pod")))) + + roots, err := s.GetFailingRoots() + require.NoError(t, err) + require.Len(t, roots, 1) + assert.Equal(t, "pending", roots[0].State) + }) +} + +func TestPriorityAnnotationLookup(t *testing.T) { + s := newTestStore(t) + require.NoError(t, s.SaveComponent(deploymentExceedingDeadline("api", time.Now(), map[string]any{insights.AnnotationPriority: "ignore"}))) + + annotation, found, err := s.GetPriorityAnnotation("api") + require.NoError(t, err) + assert.True(t, found) + assert.Equal(t, "IGNORE", annotation) + + _, found, err = s.GetPriorityAnnotation("missing") + require.NoError(t, err) + assert.False(t, found) +} + +func deploymentExceedingDeadline(name string, since time.Time, annotations map[string]any) unstructured.Unstructured { + meta := map[string]any{"name": name, "namespace": testNamespace, "uid": name, "generation": int64(1)} + if annotations != nil { + meta["annotations"] = annotations + } + return unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "apps/v1", "kind": "Deployment", "metadata": meta, + "spec": map[string]any{"replicas": int64(1)}, + "status": map[string]any{ + "observedGeneration": int64(1), + "conditions": []any{map[string]any{ + "type": "Progressing", "status": "False", "reason": "ProgressDeadlineExceeded", "lastTransitionTime": since.UTC().Format(time.RFC3339), + }}, + }, + }} +} + +func TestFailingSince(t *testing.T) { + t.Run("keeps the earliest failing time while the component keeps failing and resets on recovery", func(t *testing.T) { + s := newTestStore(t) + since := time.Now().Add(-3 * time.Hour).Truncate(time.Second) + + require.NoError(t, s.SaveComponent(deploymentExceedingDeadline("api", since, nil))) + require.NoError(t, s.SaveComponent(deploymentExceedingDeadline("api", time.Now(), nil))) + + roots, err := s.GetFailingRoots() + require.NoError(t, err) + require.Len(t, roots, 1) + assert.Equal(t, since.Unix(), roots[0].FailingSince.Unix()) + + healthy := deploymentExceedingDeadline("api", since, nil) + healthy.Object["status"] = map[string]any{"observedGeneration": int64(1), "replicas": int64(1), "updatedReplicas": int64(1), "availableReplicas": int64(1), "readyReplicas": int64(1)} + require.NoError(t, s.SaveComponent(healthy)) + assert.Empty(t, rootNames(t, s)) + + require.NoError(t, s.SaveComponent(deploymentExceedingDeadline("api", time.Now(), nil))) + roots, err = s.GetFailingRoots() + require.NoError(t, err) + require.Len(t, roots, 1) + assert.WithinDuration(t, time.Now(), roots[0].FailingSince, 5*time.Second, "a new failure starts from its own condition time") + }) + + t.Run("a replacement with the same name and another uid starts its own failing time", func(t *testing.T) { + s := newTestStore(t) + since := time.Now().Add(-3 * time.Hour).Truncate(time.Second) + require.NoError(t, s.SaveComponent(deploymentExceedingDeadline("api", since, nil))) + + replacement := deploymentExceedingDeadline("api", time.Now(), nil) + require.NoError(t, unstructured.SetNestedField(replacement.Object, "api-replacement", "metadata", "uid")) + require.NoError(t, s.SaveComponent(replacement)) + + roots, err := s.GetFailingRoots() + require.NoError(t, err) + require.Len(t, roots, 1) + assert.WithinDuration(t, time.Now(), roots[0].FailingSince, 5*time.Second, "the replaced object's failing time is not inherited") + }) +} + +func TestInsightPriorityAnnotation(t *testing.T) { + t.Run("is stored and ignored roots drop out of the score", func(t *testing.T) { + s := newTestStore(t) + + require.NoError(t, s.SaveComponent(deploymentExceedingDeadline("coredns", time.Now(), map[string]any{insights.AnnotationPriority: "ignore"}))) + + roots, err := s.GetFailingRoots() + require.NoError(t, err) + require.Len(t, roots, 1) + assert.Equal(t, "IGNORE", roots[0].PriorityAnnotation) + + score, err := s.GetHealthScore() + require.NoError(t, err) + assert.Equal(t, int64(100), score) + + list, err := s.GetComponentInsights() + require.NoError(t, err) + assert.Nil(t, list) + }) +} + +func TestComponentOwnership(t *testing.T) { + // the inventory is trusted only together with the tracking identifier of the object it was written for + inventory := func(serviceID string) map[string]any { + return map[string]any{ + "config.k8s.io/owning-inventory": serviceID, + "config.k8s.io/tracking-identifier": "apps/v1/Deployment/" + testNamespace + "/api", + } + } + + t.Run("a real loss of the inventory annotation clears the service", func(t *testing.T) { + s := newTestStore(t) + + require.NoError(t, s.SaveComponent(deploymentExceedingDeadline("api", time.Now(), inventory("svc-1")))) + roots, err := s.GetFailingRoots() + require.NoError(t, err) + require.Equal(t, "svc-1", roots[0].ServiceID) + + require.NoError(t, s.SaveComponent(deploymentExceedingDeadline("api", time.Now(), nil))) + roots, err = s.GetFailingRoots() + require.NoError(t, err) + require.Len(t, roots, 1) + assert.Empty(t, roots[0].ServiceID) + }) + + t.Run("a different inventory replaces the service", func(t *testing.T) { + s := newTestStore(t) + + require.NoError(t, s.SaveComponent(deploymentExceedingDeadline("api", time.Now(), inventory("svc-1")))) + require.NoError(t, s.SaveComponent(deploymentExceedingDeadline("api", time.Now(), inventory("svc-2")))) + + roots, err := s.GetFailingRoots() + require.NoError(t, err) + require.Len(t, roots, 1) + assert.Equal(t, "svc-2", roots[0].ServiceID) + }) +} + +func TestSchemaVersion(t *testing.T) { + t.Run("rebuilds the component table of an outdated database", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "store.db") + first := newTestStore(t, store.WithFilePath(path)) + require.NoError(t, first.SaveComponentAttributes(createComponentAttributes("api", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("api")))) + require.Equal(t, []string{"api"}, rootNames(t, first)) + + conn, err := sqlite.OpenConn(path, sqlite.OpenReadWrite) + require.NoError(t, err) + require.NoError(t, sqlitex.ExecuteTransient(conn, `PRAGMA user_version = 1`, nil)) + require.NoError(t, conn.Close()) + + second := newTestStore(t, store.WithFilePath(path)) + + assert.Empty(t, rootNames(t, second), "the cache is rebuilt from the informers") + require.NoError(t, second.SaveComponentAttributes(createComponentAttributes("api", nil, WithAttributesKind("Deployment"), WithAttributesState(client.ComponentStateFailed), WithAttributesName("api")))) + assert.Equal(t, []string{"api"}, rootNames(t, second)) + }) +} diff --git a/go/deployment-operator/pkg/streamline/store/db_store_test.go b/go/deployment-operator/pkg/streamline/store/db_store_test.go index 15524cb6b7..50a0dfdf80 100644 --- a/go/deployment-operator/pkg/streamline/store/db_store_test.go +++ b/go/deployment-operator/pkg/streamline/store/db_store_test.go @@ -1778,6 +1778,93 @@ func TestComponentCache_SetServiceChildren(t *testing.T) { require.Equal(t, componentBefore.ServiceID, "abc") require.Equal(t, componentBefore.ParentUID, "123") }) + + t.Run("should keep service ownership when a watch event updates the component", func(t *testing.T) { + storeInstance, err := store.NewDatabaseStore(context.Background(), store.WithStorage(api.StorageFile)) + require.NoError(t, err) + defer func(storeInstance store.Store) { + require.NoError(t, storeInstance.Shutdown(), "failed to shutdown store") + }(storeInstance) + + // Objects rendered by a Flux HelmRelease carry neither the owning inventory annotation + // nor an owner reference to the HelmRelease. Ownership comes only from SetServiceChildren. + obj := createUnstructuredResource("apps", "v1", "Deployment", "default", "helm-release-child") + require.NoError(t, storeInstance.SaveComponents([]unstructured.Unstructured{obj})) + + updated, err := storeInstance.SetServiceChildren("abc", "123", []common.StoreKey{ + { + GVK: obj.GroupVersionKind(), + Namespace: obj.GetNamespace(), + Name: obj.GetName(), + }, + }) + require.NoError(t, err) + require.Equal(t, 1, updated) + + // A status change, i.e. a pod restart, triggers a watch event for the same object + obj.Object["status"] = map[string]any{"observedGeneration": int64(2)} + require.NoError(t, storeInstance.SaveComponent(obj)) + + component, err := storeInstance.GetAppliedComponent(obj) + require.NoError(t, err) + require.NotNil(t, component) + assert.Equal(t, "abc", component.ServiceID, "Watch event should not drop the service ownership set by SetServiceChildren") + assert.Equal(t, "123", component.ParentUID, "Watch event should not drop the parent set by SetServiceChildren") + }) + + t.Run("should keep the ownership of an object that was reset to unsynced", func(t *testing.T) { + storeInstance, err := store.NewDatabaseStore(context.Background(), store.WithStorage(api.StorageFile)) + require.NoError(t, err) + defer func(storeInstance store.Store) { + require.NoError(t, storeInstance.Shutdown(), "failed to shutdown store") + }(storeInstance) + + obj := createUnstructuredResource("apps", "v1", "Deployment", "default", "helm-release-child") + require.NoError(t, storeInstance.SaveComponents([]unstructured.Unstructured{obj})) + _, err = storeInstance.SetServiceChildren("abc", "123", []common.StoreKey{ + {GVK: obj.GroupVersionKind(), Namespace: obj.GetNamespace(), Name: obj.GetName()}, + }) + require.NoError(t, err) + require.NoError(t, storeInstance.SetComponentUnsynced(obj)) + + // it is applied again, possibly with a new uid, the placeholder has none to compare with + again := obj.DeepCopy() + again.SetUID("another-uid") + require.NoError(t, storeInstance.SaveComponent(*again)) + + component, err := storeInstance.GetAppliedComponent(*again) + require.NoError(t, err) + require.NotNil(t, component) + assert.Equal(t, "abc", component.ServiceID) + assert.Equal(t, "123", component.ParentUID) + }) + + t.Run("should not hand the ownership to a replacement with another uid", func(t *testing.T) { + storeInstance, err := store.NewDatabaseStore(context.Background(), store.WithStorage(api.StorageFile)) + require.NoError(t, err) + defer func(storeInstance store.Store) { + require.NoError(t, storeInstance.Shutdown(), "failed to shutdown store") + }(storeInstance) + + obj := createUnstructuredResource("apps", "v1", "Deployment", "default", "helm-release-child") + require.NoError(t, storeInstance.SaveComponents([]unstructured.Unstructured{obj})) + _, err = storeInstance.SetServiceChildren("abc", "123", []common.StoreKey{ + {GVK: obj.GroupVersionKind(), Namespace: obj.GetNamespace(), Name: obj.GetName()}, + }) + require.NoError(t, err) + + // the delete event was missed, and the object was created again under the same name + replacement := obj.DeepCopy() + replacement.SetUID("another-uid") + require.NoError(t, storeInstance.SaveComponent(*replacement)) + + component, err := storeInstance.GetAppliedComponent(*replacement) + require.NoError(t, err) + require.NotNil(t, component) + assert.Equal(t, "another-uid", component.UID) + assert.Empty(t, component.ServiceID) + assert.Empty(t, component.ParentUID) + }) } func TestComponentCache_GetComponentAttributes(t *testing.T) { diff --git a/go/deployment-operator/pkg/streamline/store/facts.go b/go/deployment-operator/pkg/streamline/store/facts.go new file mode 100644 index 0000000000..a1b54170cb --- /dev/null +++ b/go/deployment-operator/pkg/streamline/store/facts.go @@ -0,0 +1,25 @@ +package store + +import "strconv" + +// sqlNull is the SQL literal of a missing value. +const sqlNull = "NULL" + +type integer interface{ ~int | ~int64 } + +// nullablePtr returns nil for a missing value, so the column stays NULL, and dereferences a present one. +// The SQL driver does not follow pointers, and a replica count of zero is a known value that must not become NULL. +func nullablePtr[T any](v *T) any { + if v == nil { + return nil + } + return *v +} + +// sqlPtr renders an optional integer as an SQL literal for statements built as strings, NULL when it is missing. +func sqlPtr[T integer](v *T) string { + if v == nil { + return sqlNull + } + return strconv.FormatInt(int64(*v), 10) +} diff --git a/go/deployment-operator/pkg/streamline/store/store.go b/go/deployment-operator/pkg/streamline/store/store.go index aedad721cf..3cf84b0e43 100644 --- a/go/deployment-operator/pkg/streamline/store/store.go +++ b/go/deployment-operator/pkg/streamline/store/store.go @@ -10,6 +10,7 @@ import ( "k8s.io/apimachinery/pkg/types" smcommon "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/common" + "github.com/pluralsh/console/go/deployment-operator/pkg/streamline/insights" ) type Store interface { @@ -38,6 +39,10 @@ type Store interface { GetAppliedComponentByUID(uid types.UID) (*client.ComponentChildAttributes, error) + // GetPriorityAnnotation returns the normalized insight priority annotation of an applied component, and whether + // the component is stored at all. + GetPriorityAnnotation(uid types.UID) (annotation string, found bool, err error) + // GetAppliedComponentsByGVK returns all applied components matching provided GVK. GetAppliedComponentsByGVK(gvk schema.GroupVersionKind) ([]smcommon.Component, error) @@ -60,6 +65,11 @@ type Store interface { // It uses a single query to fetch both components and their recursive children (up to 4 levels deep). GetServiceComponentsWithChildren(serviceID string, onlyApplied bool) ([]client.ComponentAttributes, error) + // GetFailingRoots returns the objects that represent the failures of the cluster. The health score and the + // insights are computed from them with insights.Summarize, so both always describe the same failures. + GetFailingRoots() ([]insights.Root, error) + + // GetComponentInsights returns the failing roots with their built-in priorities. GetComponentInsights() ([]client.ClusterInsightComponentAttributes, error) GetComponentCounts() (nodeCount, namespaceCount int64, err error) @@ -70,7 +80,7 @@ type Store interface { // Returns an error if the database operation fails or if the connection cannot be established. GetNodeStatistics() ([]*client.NodeStatisticAttributes, error) - // GetHealthScore calculates cluster health as a percentage using a base score minus deductions system. + // GetHealthScore calculates the cluster health, 100 minus a penalty for every failing root. // Returns the health score as an integer value (0-100) and any error encountered. GetHealthScore() (int64, error) diff --git a/go/deployment-operator/pkg/streamline/supervisor.go b/go/deployment-operator/pkg/streamline/supervisor.go index e143474e34..4b22ba2a81 100644 --- a/go/deployment-operator/pkg/streamline/supervisor.go +++ b/go/deployment-operator/pkg/streamline/supervisor.go @@ -3,6 +3,7 @@ package streamline import ( "context" "fmt" + "math/rand/v2" "sync" "time" @@ -36,6 +37,21 @@ func WithRestartDelay(d time.Duration) Option { } } +// WithMaxRestartBackoff caps the delay between restarts of a synchronizer that fails to start. A resource that +// keeps failing until its next restart waits for the cap no longer holds back WaitUntilSynced, it is still retried. +func WithMaxRestartBackoff(d time.Duration) Option { + return func(s *Supervisor) { + s.maxRestartBackoff = d + } +} + +// WithRestartJitter sets how far a restart is spread out at most, so resources that fail together do not retry together. +func WithRestartJitter(d time.Duration) Option { + return func(s *Supervisor) { + s.restartJitter = d + } +} + func WithCacheSyncTimeout(d time.Duration) Option { return func(s *Supervisor) { s.cacheSyncTimeout = d @@ -80,6 +96,8 @@ type Supervisor struct { dequeueJitter time.Duration updateJitter time.Duration restartDelay time.Duration + maxRestartBackoff time.Duration + restartJitter time.Duration cacheSyncTimeout time.Duration synchronizerResyncInterval time.Duration maxNotFoundRetries int @@ -95,13 +113,14 @@ func NewSupervisor(client dynamic.Interface, store store.Store, statusSynchroniz discoveryCache: discoveryCache, svcCache: svcCache, store: store, - registerQueue: workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[schema.GroupVersionResource]()), componentQueue: workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[string]()), synchronizers: cmap.NewStringer[schema.GroupVersionResource, Synchronizer](), restartAttemptsLeftMap: cmap.NewStringer[schema.GroupVersionResource, int](), eventSubscribers: cmap.NewStringer[schema.GroupVersionResource, []EventSubscriber](), maxNotFoundRetries: 3, restartDelay: 1 * time.Second, + maxRestartBackoff: 1 * time.Minute, + restartJitter: 10 * time.Second, cacheSyncTimeout: 5 * time.Second, synchronizerResyncInterval: 30 * time.Minute, dequeueJitter: 200 * time.Millisecond, @@ -114,9 +133,31 @@ func NewSupervisor(client dynamic.Interface, store store.Store, statusSynchroniz option(s) } + // The queue backs off per resource, so it is built once the delays are known. + s.registerQueue = workqueue.NewTypedRateLimitingQueue(jitteredRateLimiter{ + TypedRateLimiter: workqueue.NewTypedItemExponentialFailureRateLimiter[schema.GroupVersionResource](s.restartDelay, s.maxRestartBackoff), + jitter: s.restartJitter, + }) + return s } +// jitteredRateLimiter spreads out the delays of the wrapped limiter by up to jitter. The spread never exceeds the +// delay itself, so the first restarts stay quick. +type jitteredRateLimiter struct { + workqueue.TypedRateLimiter[schema.GroupVersionResource] + jitter time.Duration +} + +func (in jitteredRateLimiter) When(gvr schema.GroupVersionResource) time.Duration { + delay := in.TypedRateLimiter.When(gvr) + if spread := min(in.jitter, delay); spread > 0 { + delay += rand.N(spread) + } + + return delay +} + func (in *Supervisor) WaitForCacheSync(ctx context.Context) error { timeoutChan := time.After(in.cacheSyncTimeout) syncedCount := 0 @@ -142,6 +183,47 @@ func (in *Supervisor) WaitForCacheSync(ctx context.Context) error { } } +// WaitUntilSynced blocks until every synchronizer has loaded its resources into the store, or has been failing for so +// long that it is no longer waited for, and returns early only when the context is done. Unlike WaitForCacheSync it has no timeout, as a consumer that +// reports on the whole cluster has nothing correct to report from a partially loaded store. +func (in *Supervisor) WaitUntilSynced(ctx context.Context) error { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + + for { + if in.synced() { + return nil + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} + +func (in *Supervisor) synced() bool { + for gvr, s := range in.synchronizers.Items() { + if !s.Started() && !in.broken(gvr) { + return false + } + } + + return true +} + +// broken tells whether a resource failed to start often enough for its next restart to wait for the cap. It is still +// retried at that pace, but the store is not held back for it any longer. +func (in *Supervisor) broken(gvr schema.GroupVersionResource) bool { + failures := 0 + for delay := in.restartDelay; delay > 0 && delay < in.maxRestartBackoff; delay *= 2 { + failures++ + } + + return in.registerQueue.NumRequeues(gvr) >= max(failures, 1) +} + func (in *Supervisor) MaybeRegister(gvr schema.GroupVersionResource) { if smcommon.OptionalResourceVersionList.Has(fmt.Sprintf("%s/%s", gvr.Resource, gvr.Version)) { klog.V(log.LogLevelVerbose).InfoS("skipping resource to watch as it is optional", "gvr", gvr.String()) @@ -190,6 +272,7 @@ func (in *Supervisor) Unregister(gvr schema.GroupVersionResource) { in.synchronizers.Remove(gvr) } + in.registerQueue.Forget(gvr) in.clearAttempts(gvr) } @@ -235,10 +318,8 @@ func (in *Supervisor) processNextWorkItem(ctx context.Context) bool { return false } - defer func() { - in.registerQueue.Forget(gvr) - in.registerQueue.Done(gvr) - }() + // The failures of a resource are kept until it starts, that is what makes its restarts back off. + defer in.registerQueue.Done(gvr) go in.startSynchronizer(ctx, gvr) return true } @@ -273,9 +354,13 @@ func (in *Supervisor) startSynchronizer(ctx context.Context, gvr schema.GroupVer err := syn.Start(ctx) // start is a blocking operation metrics.Record().ResourceCacheWatchEnd(gvr.String()) - in.synchronizers.Remove(gvr) + if syn.Started() { + in.registerQueue.Forget(gvr) + } + // The synchronizer stays registered across a restart, so the store is not reported as loaded in between. if err == nil { + in.synchronizers.Remove(gvr) klog.V(log.LogLevelVerbose).InfoS("synchronizer stopped", "gvr", gvr.String()) metrics.Record().ResourceCacheWatchRemove(gvr.String()) return @@ -283,14 +368,18 @@ func (in *Supervisor) startSynchronizer(ctx context.Context, gvr schema.GroupVer if apierrors.IsMethodNotSupported(err) { klog.V(log.LogLevelVerbose).ErrorS(err, "skipping resource as it is not supported", "gvr", gvr.String()) + in.synchronizers.Remove(gvr) metrics.Record().ResourceCacheWatchRemove(gvr.String()) + in.registerQueue.Forget(gvr) return } gvk, discErr := in.discoveryCache.KindFor(gvr) if discErr != nil { klog.V(log.LogLevelVerbose).ErrorS(discErr, "failed to register resource, could not get gvk for resource", "gvr", gvr.String()) + in.synchronizers.Remove(gvr) metrics.Record().ResourceCacheWatchRemove(gvr.String()) + in.registerQueue.Forget(gvr) return } @@ -299,7 +388,9 @@ func (in *Supervisor) startSynchronizer(ctx context.Context, gvr schema.GroupVer if left == 0 { klog.V(log.LogLevelVerbose).ErrorS(err, "resource not found after retries, skipping", "gvr", gvr.String(), "attempts", used) in.clearAttempts(gvr) + in.synchronizers.Remove(gvr) metrics.Record().ResourceCacheWatchRemove(gvr.String()) + in.registerQueue.Forget(gvr) return } diff --git a/go/deployment-operator/pkg/streamline/supervisor_internal_test.go b/go/deployment-operator/pkg/streamline/supervisor_internal_test.go new file mode 100644 index 0000000000..6d7d2db762 --- /dev/null +++ b/go/deployment-operator/pkg/streamline/supervisor_internal_test.go @@ -0,0 +1,196 @@ +package streamline + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/util/workqueue" + + discoverycache "github.com/pluralsh/console/go/deployment-operator/pkg/cache/discovery" +) + +type fakeSynchronizer struct { + started atomic.Bool + err error +} + +func (f *fakeSynchronizer) Start(context.Context) error { return f.err } +func (f *fakeSynchronizer) Stop() {} +func (f *fakeSynchronizer) Started() bool { return f.started.Load() } + +// failingDiscovery makes startSynchronizer give up right after a failed start instead of re-registering. +type failingDiscovery struct{ discoverycache.Cache } + +func (failingDiscovery) KindFor(schema.GroupVersionResource) (schema.GroupVersionKind, error) { + return schema.GroupVersionKind{}, errors.New("unknown resource") +} + +// knownDiscovery makes startSynchronizer restart the synchronizer after a failed start. +type knownDiscovery struct{ discoverycache.Cache } + +func (knownDiscovery) KindFor(gvr schema.GroupVersionResource) (schema.GroupVersionKind, error) { + return schema.GroupVersionKind{Version: gvr.Version, Kind: "Pod"}, nil +} + +var ( + podsGVR = schema.GroupVersionResource{Version: "v1", Resource: "pods"} + deploymentGVR = schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} +) + +func TestSupervisorWaitUntilSynced(t *testing.T) { + t.Run("returns once every synchronizer has started", func(t *testing.T) { + s := NewSupervisor(nil, nil, StatusSynchronizer{}, nil, nil) + pods, deployments := &fakeSynchronizer{}, &fakeSynchronizer{} + s.synchronizers.Set(podsGVR, pods) + s.synchronizers.Set(deploymentGVR, deployments) + + done := make(chan error, 1) + go func() { done <- s.WaitUntilSynced(t.Context()) }() + + pods.started.Store(true) + select { + case <-done: + t.Fatal("returned while a synchronizer was still loading") + case <-time.After(1500 * time.Millisecond): + } + + deployments.started.Store(true) + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("did not return after all synchronizers started") + } + }) + + t.Run("waits for a resource that is still being retried", func(t *testing.T) { + s := NewSupervisor(nil, nil, StatusSynchronizer{}, nil, nil, + WithRestartDelay(10*time.Millisecond), WithMaxRestartBackoff(40*time.Millisecond)) + s.synchronizers.Set(podsGVR, &fakeSynchronizer{}) + s.registerQueue.AddRateLimited(podsGVR) + + assert.False(t, s.synced()) + }) + + t.Run("does not wait for a resource that keeps failing", func(t *testing.T) { + s := NewSupervisor(nil, nil, StatusSynchronizer{}, nil, nil, + WithRestartDelay(10*time.Millisecond), WithMaxRestartBackoff(40*time.Millisecond)) + s.synchronizers.Set(podsGVR, &fakeSynchronizer{}) + s.registerQueue.AddRateLimited(podsGVR) + s.registerQueue.AddRateLimited(podsGVR) + + require.NoError(t, s.WaitUntilSynced(t.Context())) + }) + + t.Run("a failed start is retried with a growing delay", func(t *testing.T) { + s := NewSupervisor(nil, nil, StatusSynchronizer{}, knownDiscovery{}, nil, + WithRestartDelay(time.Hour), WithRestartJitter(0)) + s.synchronizers.Set(podsGVR, &fakeSynchronizer{err: errors.New("forbidden")}) + + s.startSynchronizer(t.Context(), podsGVR) + assert.Equal(t, 1, s.registerQueue.NumRequeues(podsGVR)) + assert.True(t, s.synchronizers.Has(podsGVR), "the resource is registered again") + + s.synchronizers.Set(podsGVR, &fakeSynchronizer{err: errors.New("forbidden")}) + s.startSynchronizer(t.Context(), podsGVR) + assert.Equal(t, 2, s.registerQueue.NumRequeues(podsGVR)) + }) + + t.Run("a started synchronizer resets the failures", func(t *testing.T) { + s := NewSupervisor(nil, nil, StatusSynchronizer{}, knownDiscovery{}, nil, WithRestartDelay(time.Hour)) + s.registerQueue.AddRateLimited(podsGVR) + started := &fakeSynchronizer{err: errors.New("watch closed")} + started.started.Store(true) + s.synchronizers.Set(podsGVR, started) + + s.startSynchronizer(t.Context(), podsGVR) + + assert.Equal(t, 1, s.registerQueue.NumRequeues(podsGVR), "only the restart after the closed watch is counted") + }) + + t.Run("is broken only once the next restart waits for the cap", func(t *testing.T) { + s := NewSupervisor(nil, nil, StatusSynchronizer{}, nil, nil, + WithRestartDelay(10*time.Millisecond), WithMaxRestartBackoff(40*time.Millisecond)) + s.registerQueue.AddRateLimited(podsGVR) + assert.False(t, s.broken(podsGVR), "the next restart waits 20ms") + + s.registerQueue.AddRateLimited(podsGVR) + assert.True(t, s.broken(podsGVR), "the next restart waits 40ms") + }) + + t.Run("a resource that is not found is retried until its attempts are used", func(t *testing.T) { + s := NewSupervisor(nil, nil, StatusSynchronizer{}, knownDiscovery{}, nil, + WithRestartDelay(time.Hour), WithRestartJitter(0), WithMaxNotFoundRetries(2)) + notFound := apierrors.NewNotFound(schema.GroupResource{Resource: "pods"}, "pods") + + s.synchronizers.Set(podsGVR, &fakeSynchronizer{err: notFound}) + s.startSynchronizer(t.Context(), podsGVR) + assert.Equal(t, 1, s.registerQueue.NumRequeues(podsGVR)) + assert.True(t, s.synchronizers.Has(podsGVR), "it is waiting for its restart") + + s.synchronizers.Set(podsGVR, &fakeSynchronizer{err: notFound}) + s.startSynchronizer(t.Context(), podsGVR) + assert.Zero(t, s.registerQueue.NumRequeues(podsGVR)) + assert.False(t, s.synchronizers.Has(podsGVR), "it was given up on") + }) + + t.Run("a resource that is given up on is forgotten", func(t *testing.T) { + s := NewSupervisor(nil, nil, StatusSynchronizer{}, failingDiscovery{}, nil, WithRestartDelay(time.Hour)) + s.registerQueue.AddRateLimited(podsGVR) + s.synchronizers.Set(podsGVR, &fakeSynchronizer{err: errors.New("forbidden")}) + + s.startSynchronizer(t.Context(), podsGVR) + + assert.Zero(t, s.registerQueue.NumRequeues(podsGVR)) + assert.False(t, s.synchronizers.Has(podsGVR)) + }) + + t.Run("returns when the context is done", func(t *testing.T) { + s := NewSupervisor(nil, nil, StatusSynchronizer{}, nil, nil) + s.synchronizers.Set(podsGVR, &fakeSynchronizer{}) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + require.ErrorIs(t, s.WaitUntilSynced(ctx), context.Canceled) + }) +} + +func TestJitteredRateLimiter(t *testing.T) { + inner := workqueue.NewTypedItemExponentialFailureRateLimiter[schema.GroupVersionResource](time.Second, time.Minute) + + t.Run("spreads a delay by at most the jitter", func(t *testing.T) { + limiter := jitteredRateLimiter{TypedRateLimiter: inner, jitter: 10 * time.Second} + for range 50 { + inner.Forget(podsGVR) + for range 5 { + inner.When(podsGVR) // 1s, 2s, 4s, 8s, 16s + } + delay := limiter.When(podsGVR) // 32s + assert.GreaterOrEqual(t, delay, 32*time.Second) + assert.Less(t, delay, 42*time.Second) + } + }) + + t.Run("never spreads more than the delay itself", func(t *testing.T) { + limiter := jitteredRateLimiter{TypedRateLimiter: inner, jitter: 10 * time.Second} + inner.Forget(podsGVR) + + delay := limiter.When(podsGVR) // 1s + assert.GreaterOrEqual(t, delay, time.Second) + assert.Less(t, delay, 2*time.Second) + }) + + t.Run("leaves the delay alone without jitter", func(t *testing.T) { + limiter := jitteredRateLimiter{TypedRateLimiter: inner} + inner.Forget(podsGVR) + + assert.Equal(t, time.Second, limiter.When(podsGVR)) + }) +} diff --git a/js/console/src/components/cd/cluster/ClusterInsightComponent.tsx b/js/console/src/components/cd/cluster/ClusterInsightComponent.tsx index 6ff187864b..1d1f22545f 100644 --- a/js/console/src/components/cd/cluster/ClusterInsightComponent.tsx +++ b/js/console/src/components/cd/cluster/ClusterInsightComponent.tsx @@ -1,14 +1,19 @@ -import { Code, Divider, Flex, Switch } from '@pluralsh/design-system' +import { Button, Code, Divider, Flex, Switch } from '@pluralsh/design-system' import { InsightDisplay } from 'components/ai/insights/InsightDisplay.tsx' import { SendInsightToWorkbenchButton } from 'components/ai/insights/SendInsightToWorkbench' import { GqlError } from 'components/utils/Alert.tsx' import IconFrameRefreshButton from 'components/utils/RefreshIconFrame.tsx' -import { useClusterInsightComponentQuery } from 'generated/graphql.ts' +import { + useClusterInsightComponentQuery, + useServiceComponentIdentitiesQuery, +} from 'generated/graphql.ts' import { useMemo, useState } from 'react' -import { useMatch } from 'react-router-dom' +import { Link, useMatch } from 'react-router-dom' import { CLUSTER_ABS_PATH, CLUSTER_INSIGHTS_PATH, + getServiceComponentPath, + getServiceDetailsPath, } from 'routes/cdRoutesConsts.tsx' import { useTheme } from 'styled-components' import { stringify } from 'yaml' @@ -17,7 +22,10 @@ import { useSetActionContent, useSetNavigationContent, } from './ClusterInsights.tsx' -import { ClusterInsightComponentLabel } from './ClusterInsightsComponents.tsx' +import { + ClusterInsightComponentLabel, + isSameObject, +} from './ClusterInsightComponentParts.tsx' export function ClusterInsightComponent() { const theme = useTheme() @@ -32,6 +40,38 @@ export function ClusterInsightComponent() { const component = data?.clusterInsightComponent const [showRaw, setShowRaw] = useState(false) + // Goes to the component of the managing service that is this object, which has the insight with the context of the + // service, and to the service itself when it does not have that component in its manifests. + const service = component?.service + const { data: serviceData } = useServiceComponentIdentitiesQuery({ + skip: !service, + variables: { id: service?.id ?? '' }, + }) + const serviceComponentId = component + ? serviceData?.serviceDeployment?.components?.find( + (c) => c && isSameObject(c, component) + )?.id + : undefined + const serviceLink = useMemo(() => { + if (!service) return null + + const clusterId = service.cluster?.id + + return serviceComponentId + ? { + label: 'View Service Component', + to: `${getServiceComponentPath({ + clusterId, + serviceId: service.id, + componentId: serviceComponentId, + })}/insights`, + } + : { + label: 'View Service', + to: getServiceDetailsPath({ clusterId, serviceId: service.id }), + } + }, [service, serviceComponentId]) + useSetNavigationContent( useMemo( () => ( @@ -47,7 +87,10 @@ export function ClusterInsightComponent() { useSetActionContent( useMemo( () => ( - + setShowRaw(checked)} @@ -58,10 +101,19 @@ export function ClusterInsightComponent() { loading={clusterLoading} refetch={refetch} /> + {serviceLink && ( + + )} ), - [showRaw, clusterLoading, refetch, component?.insight] + [showRaw, clusterLoading, refetch, component?.insight, serviceLink] ) ) diff --git a/js/console/src/components/cd/cluster/ClusterInsightComponentCard.tsx b/js/console/src/components/cd/cluster/ClusterInsightComponentCard.tsx new file mode 100644 index 0000000000..033649e8f1 --- /dev/null +++ b/js/console/src/components/cd/cluster/ClusterInsightComponentCard.tsx @@ -0,0 +1,153 @@ +import { + Card, + CaretRightIcon, + Chip, + ComponentsIcon, + Flex, + IconFrame, +} from '@pluralsh/design-system' +import { capitalize } from 'lodash' +import { Link } from 'react-router-dom' +import styled from 'styled-components' + +import { AiInsightSummaryIcon } from 'components/utils/AiInsights.tsx' +import { Body2P, CaptionP } from 'components/utils/typography/Text' +import { + ClusterInsightComponentFragment, + InsightFreshness, +} from 'generated/graphql.ts' +import { fromNow } from 'utils/datetime' +import { + ClusterInsightComponentLabel, + ClusterInsightServiceChip, + priorityToSeverity, +} from './ClusterInsightComponentParts.tsx' + +export function ClusterInsightComponentCard({ + component, +}: { + component: ClusterInsightComponentFragment +}) { + const { insight, priority, service } = component + const summary = + insight?.freshness === InsightFreshness.Expired ? null : insight?.summary + + return ( + + ` : ''}${component.name} (${component.kind})`} + /> + + } + /> + } + /> + {(priority || service) && ( + + {priority && ( + + {capitalize(priority)} + + )} + + + )} + + {summary ?? 'No insight generated yet.'} + + + + + + {insight?.updatedAt && ( + + Updated {fromNow(insight.updatedAt)} + + )} + + + + + ) +} + +const ContentSC = styled.div(({ theme }) => ({ + display: 'flex', + flexDirection: 'column', + flex: 1, + minWidth: 0, + padding: theme.spacing.medium, + gap: theme.spacing.small, +})) + +const FooterSC = styled.div(({ theme }) => ({ + display: 'flex', + justifyContent: 'space-between', + alignItems: 'center', + padding: `${theme.spacing.small}px ${theme.spacing.medium}px`, + borderTop: theme.borders.default, +})) + +const CardLinkSC = styled(Link)({ + position: 'absolute', + inset: 0, + zIndex: 0, +}) + +const CardSC = styled(Card)(({ theme }) => ({ + position: 'relative', + display: 'flex', + flexDirection: 'column', + width: '100%', + overflow: 'hidden', + cursor: 'pointer', + textDecoration: 'none', + [`& button, & a:not(${CardLinkSC})`]: { + position: 'relative', + zIndex: 1, + }, + [`&:hover:not(:has(button:hover, a:not(${CardLinkSC}):hover))`]: { + backgroundColor: theme.colors['fill-one-hover'], + borderColor: theme.colors['border-fill-one'], + [`${FooterSC}`]: { + backgroundColor: theme.colors['fill-one-hover'], + borderTopColor: theme.colors['border-fill-one'], + }, + }, +})) diff --git a/js/console/src/components/cd/cluster/ClusterInsightComponentParts.test.ts b/js/console/src/components/cd/cluster/ClusterInsightComponentParts.test.ts new file mode 100644 index 0000000000..9ca3d4dc9a --- /dev/null +++ b/js/console/src/components/cd/cluster/ClusterInsightComponentParts.test.ts @@ -0,0 +1,45 @@ +import { InsightComponentPriority } from 'generated/graphql' +import { describe, expect, it } from 'vitest' + +import { sortInsightComponents } from './ClusterInsightComponentParts' + +const component = ( + name: string, + priority: InsightComponentPriority | null, + namespace: string | null = 'apps' +) => ({ name, priority, namespace }) + +describe('sortInsightComponents', () => { + it('lists the most urgent components first, then by namespace and name', () => { + const sorted = sortInsightComponents([ + component('low-a', InsightComponentPriority.Low), + component('crit-b', InsightComponentPriority.Critical), + component('none', null), + component('crit-a', InsightComponentPriority.Critical), + component('crit-kube', InsightComponentPriority.Critical, 'kube-system'), + component('med', InsightComponentPriority.Medium), + component('high', InsightComponentPriority.High), + ]) + + expect(sorted.map((c) => c.name)).toEqual([ + 'crit-a', + 'crit-b', + 'crit-kube', + 'high', + 'med', + 'low-a', + 'none', + ]) + }) + + it('does not change the input', () => { + const input = [ + component('b', InsightComponentPriority.Low), + component('a', InsightComponentPriority.Critical), + ] + + sortInsightComponents(input) + + expect(input.map((c) => c.name)).toEqual(['b', 'a']) + }) +}) diff --git a/js/console/src/components/cd/cluster/ClusterInsightComponentParts.tsx b/js/console/src/components/cd/cluster/ClusterInsightComponentParts.tsx new file mode 100644 index 0000000000..0315ac0081 --- /dev/null +++ b/js/console/src/components/cd/cluster/ClusterInsightComponentParts.tsx @@ -0,0 +1,122 @@ +import { Chip, ChipSeverity, ComponentsIcon } from '@pluralsh/design-system' +import { ComponentPropsWithRef } from 'react' + +import { + ClusterInsightComponentFragment, + InsightComponentPriority, +} from 'generated/graphql.ts' +import { StackedText } from '../../utils/table/StackedText.tsx' + +export function ClusterInsightComponentLabel({ + component, + icon = , + ...props +}: { + component: Nullable +} & Partial>) { + const title = component?.namespace + ? `${component.namespace} > ${component.name}` + : component?.name + + return ( + + ) +} + +export function priorityToSeverity( + priority: Nullable +): ChipSeverity { + switch (priority) { + case InsightComponentPriority.Medium: + return 'warning' + case InsightComponentPriority.High: + return 'danger' + case InsightComponentPriority.Critical: + return 'critical' + case InsightComponentPriority.Low: + default: + return 'neutral' + } +} + +const PRIORITY_RANK: Record = { + [InsightComponentPriority.Critical]: 4, + [InsightComponentPriority.High]: 3, + [InsightComponentPriority.Medium]: 2, + [InsightComponentPriority.Low]: 1, +} + +type SortableComponent = Pick< + ClusterInsightComponentFragment, + 'priority' | 'namespace' | 'name' +> + +// most urgent first, then a stable order by namespace and name, so both views list components the same way +export function compareInsightComponents( + a: SortableComponent, + b: SortableComponent +): number { + const byPriority = + (b.priority ? PRIORITY_RANK[b.priority] : 0) - + (a.priority ? PRIORITY_RANK[a.priority] : 0) + + if (byPriority !== 0) return byPriority + + return ( + (a.namespace ?? '').localeCompare(b.namespace ?? '') || + a.name.localeCompare(b.name) + ) +} + +export function sortInsightComponents( + components: T[] +): T[] { + return [...components].sort(compareInsightComponents) +} + +type ObjectRef = { + group?: Nullable + version?: Nullable + kind: string + namespace?: Nullable + name: string +} + +// the core group and cluster scoped objects are reported as either nil or an empty string +export function isSameObject(a: ObjectRef, b: ObjectRef): boolean { + return ( + (a.group ?? '') === (b.group ?? '') && + (a.version ?? '') === (b.version ?? '') && + a.kind === b.kind && + (a.namespace ?? '') === (b.namespace ?? '') && + a.name === b.name + ) +} + +export function ClusterInsightServiceChip({ + service, +}: { + service: Nullable +}) { + if (!service) return null + + return ( + + {service.name} + + ) +} diff --git a/js/console/src/components/cd/cluster/ClusterInsightComponentsTable.tsx b/js/console/src/components/cd/cluster/ClusterInsightComponentsTable.tsx new file mode 100644 index 0000000000..e5d5ee2c55 --- /dev/null +++ b/js/console/src/components/cd/cluster/ClusterInsightComponentsTable.tsx @@ -0,0 +1,116 @@ +import { + CaretRightIcon, + Chip, + ComponentsIcon, + IconFrame, + Table, +} from '@pluralsh/design-system' +import { createColumnHelper } from '@tanstack/react-table' +import { capitalize } from 'lodash' +import { Link } from 'react-router-dom' + +import { AiInsightSummaryIcon } from 'components/utils/AiInsights.tsx' +import { ClusterInsightComponentFragment } from 'generated/graphql.ts' +import { + ClusterInsightComponentLabel, + ClusterInsightServiceChip, + priorityToSeverity, +} from './ClusterInsightComponentParts.tsx' + +const columnHelper = createColumnHelper() + +const columns = [ + columnHelper.accessor((component) => component, { + id: 'component', + header: 'Component', + meta: { gridTemplate: 'minmax(240px, 3fr)' }, + cell: function Cell({ getValue }) { + return ( + } + /> + } + /> + ) + }, + }), + columnHelper.accessor((component) => component.priority, { + id: 'priority', + header: 'Priority', + meta: { gridTemplate: 'minmax(110px, 0.7fr)' }, + cell: function Cell({ getValue }) { + const priority = getValue() + + return priority ? ( + + {capitalize(priority)} + + ) : null + }, + }), + columnHelper.accessor((component) => component.service, { + id: 'service', + header: 'Service', + meta: { gridTemplate: 'minmax(140px, 1.5fr)' }, + cell: function Cell({ getValue }) { + return + }, + }), + columnHelper.accessor((component) => component, { + id: 'insight', + header: '', + meta: { gridTemplate: 'min-content' }, + cell: function Cell({ getValue }) { + const component = getValue() + + return ( + + ) + }, + }), + columnHelper.display({ + id: 'arrow', + header: '', + meta: { gridTemplate: 'min-content' }, + cell: () => ( + } + size="medium" + type="tertiary" + /> + ), + }), +] + +export function ClusterInsightComponentsTable({ + components, + loading = false, +}: { + components: ClusterInsightComponentFragment[] + loading?: boolean +}) { + return ( + ( + + )} + /> + ) +} diff --git a/js/console/src/components/cd/cluster/ClusterInsightsComponents.tsx b/js/console/src/components/cd/cluster/ClusterInsightsComponents.tsx index 3c49dbbc7e..47f44e6b2a 100644 --- a/js/console/src/components/cd/cluster/ClusterInsightsComponents.tsx +++ b/js/console/src/components/cd/cluster/ClusterInsightsComponents.tsx @@ -1,45 +1,71 @@ import { - Card, + AppsIcon, Chip, - ChipSeverity, ComponentsIcon, EmptyState, Flex, - IconFrame, + ListIcon, SubTab, } from '@pluralsh/design-system' -import { ComponentPropsWithRef, useMemo, useState } from 'react' +import { useMemo, useState } from 'react' +import usePersistedState from 'components/hooks/usePersistedState.tsx' +import { CardGrid } from 'components/self-service/catalog/CatalogsGrid.tsx' +import { ButtonGroup } from 'components/utils/ButtonGroup.tsx' import { RectangleSkeleton } from 'components/utils/SkeletonLoaders.tsx' -import { - ClusterInsightComponentFragment, - InsightComponentPriority, -} from 'generated/graphql.ts' +import { InsightComponentPriority } from 'generated/graphql.ts' import { capitalize, countBy, isEmpty } from 'lodash' -import { Link } from 'react-router-dom' -import styled, { useTheme } from 'styled-components' -import { AiInsightSummaryIcon } from '../../utils/AiInsights.tsx' +import { useTheme } from 'styled-components' import { StackedText } from '../../utils/table/StackedText.tsx' +import { useSetPageHeaderContent } from '../ContinuousDeployment.tsx' import { componentHasInsight } from '../clusters/info-flyover/health/ConfigurationIssuesSection.tsx' import { useClusterInsightsContext, useSetActionContent, useSetNavigationContent, } from './ClusterInsights.tsx' +import { ClusterInsightComponentCard } from './ClusterInsightComponentCard.tsx' +import { + priorityToSeverity, + sortInsightComponents, +} from './ClusterInsightComponentParts.tsx' +import { ClusterInsightComponentsTable } from './ClusterInsightComponentsTable.tsx' + +type InsightsView = 'list' | 'board' + +const VIEW_STORAGE_KEY = 'cluster-insights-view' + +const viewDirectory = [ + { path: 'list', icon: , tooltip: 'List view' }, + { path: 'board', icon: , tooltip: 'Board view' }, +] + +function parseView(value: unknown): InsightsView { + return value === 'board' ? 'board' : 'list' +} export function ClusterInsightsComponents() { const theme = useTheme() const { cluster, clusterLoading } = useClusterInsightsContext() const [priorityFilter, setPriorityFilter] = useState(null) + const [view, setView] = usePersistedState( + VIEW_STORAGE_KEY, + 'list', + 0, + parseView + ) const { total, filteredComponents, counts } = useMemo(() => { const components = cluster?.insightComponents?.filter(componentHasInsight) ?? [] return { total: components.length, - filteredComponents: components.filter( - (component) => !priorityFilter || component.priority === priorityFilter + filteredComponents: sortInsightComponents( + components.filter( + (component) => + !priorityFilter || component.priority === priorityFilter + ) ), counts: countBy(components, (component) => component.priority), } @@ -100,117 +126,70 @@ export function ClusterInsightsComponents() { ) ) - if (!loading && isEmpty(filteredComponents)) + // same place and look as the list/tree switch of the cluster services + useSetPageHeaderContent( + useMemo( + () => ( + setView(parseView(path))} + /> + ), + [view, setView] + ) + ) + + if (loading && view === 'list') + return ( + + ) + + if (loading) + return ( +
+ {Array.from({ length: 8 }).map((_, i) => ( + + ))} +
+ ) + + if (isEmpty(filteredComponents)) return + if (view === 'list') + return + return ( -
- {loading - ? Array.from({ length: 8 }).map((_, i) => ( - - )) - : filteredComponents.map((component) => ( - - } - /> - } - first={ - - {component.namespace && ( - - {component.namespace} >{' '} - - )} - {component.name} - - } - firstPartialType="body2" - secondPartialType="caption" - /> - - {component.priority && ( - - {capitalize(component.priority)} - - )} - - - - ))} -
- ) -} - -export function ClusterInsightComponentLabel({ - component, - icon = , - ...props -}: { - component: Nullable -} & Partial>) { - const title = component?.namespace - ? `${component.namespace} > ${component.name}` - : component?.name - - return ( - + {filteredComponents.map((component) => ( + + ))} + ) } -function priorityToSeverity( - priority: Nullable -): ChipSeverity { - switch (priority) { - case InsightComponentPriority.Medium: - return 'warning' - case InsightComponentPriority.High: - return 'danger' - case InsightComponentPriority.Critical: - return 'critical' - case InsightComponentPriority.Low: - default: - return 'neutral' - } -} - const priorityFilterValues: (InsightComponentPriority | 'All')[] = [ 'All', InsightComponentPriority.Critical, @@ -218,13 +197,3 @@ const priorityFilterValues: (InsightComponentPriority | 'All')[] = [ InsightComponentPriority.Medium, InsightComponentPriority.Low, ] - -const ComponentInsightCardSC = styled(Card)(({ theme }) => ({ - display: 'flex', - alignItems: 'center', - justifyContent: 'space-between', - padding: `${theme.spacing.small}px ${theme.spacing.medium}px`, - gap: theme.spacing.xsmall, - textDecoration: 'none', - '&:any-link': { textDecoration: 'none' }, -})) diff --git a/js/console/src/components/cd/clusters/info-flyover/health/ConfigurationIssuesSection.tsx b/js/console/src/components/cd/clusters/info-flyover/health/ConfigurationIssuesSection.tsx index 62feab7e30..767a21eea0 100644 --- a/js/console/src/components/cd/clusters/info-flyover/health/ConfigurationIssuesSection.tsx +++ b/js/console/src/components/cd/clusters/info-flyover/health/ConfigurationIssuesSection.tsx @@ -1,6 +1,6 @@ import { Chip, Flex, SubTab, Table } from '@pluralsh/design-system' import { createColumnHelper, Row } from '@tanstack/react-table' -import { ClusterInsightComponentLabel } from 'components/cd/cluster/ClusterInsightsComponents' +import { ClusterInsightComponentLabel } from 'components/cd/cluster/ClusterInsightComponentParts' import { AiInsightSummaryIcon } from 'components/utils/AiInsights' import { ClusterInsightComponentFragment, diff --git a/js/console/src/generated/graphql-plural.ts b/js/console/src/generated/graphql-plural.ts index ea8f963c46..76e36ec025 100644 --- a/js/console/src/generated/graphql-plural.ts +++ b/js/console/src/generated/graphql-plural.ts @@ -2591,6 +2591,15 @@ export type RedirectToUrl = { url?: Maybe; }; +export type RefreshToken = { + __typename?: 'RefreshToken'; + id: Scalars['ID']['output']; + insertedAt?: Maybe; + /** the token to use to request a refresh */ + token: Scalars['String']['output']; + updatedAt?: Maybe; +}; + /** The release status of a repository, defaults to ALPHA, GA if it is ready for general consumption */ export enum ReleaseStatus { Alpha = 'ALPHA', @@ -2979,6 +2988,7 @@ export type RootMutationType = { linkPublisher?: Maybe; login?: Maybe; loginToken?: Maybe; + logout?: Maybe; oauthCallback?: Maybe; oauthConsent?: Maybe; passwordlessLogin?: Maybe; @@ -3908,6 +3918,7 @@ export type RootQueryType = { publishers?: Maybe; recipe?: Maybe; recipes?: Maybe; + refresh?: Maybe; repositories?: Maybe; /** Get an application by its ID or name. */ repository?: Maybe; @@ -4311,6 +4322,11 @@ export type RootQueryTypeRecipesArgs = { }; +export type RootQueryTypeRefreshArgs = { + token: Scalars['String']['input']; +}; + + export type RootQueryTypeRepositoriesArgs = { after?: InputMaybe; before?: InputMaybe; @@ -5125,6 +5141,7 @@ export type User = { phone?: Maybe; provider?: Maybe; publisher?: Maybe; + refreshToken?: Maybe; roles?: Maybe; serviceAccount?: Maybe; trustRelationships?: Maybe>>; diff --git a/js/console/src/generated/graphql.ts b/js/console/src/generated/graphql.ts index 04ec9df051..3c2bc67680 100644 --- a/js/console/src/generated/graphql.ts +++ b/js/console/src/generated/graphql.ts @@ -2939,6 +2939,8 @@ export type ClusterInsightComponent = { priority?: Maybe; /** the raw kubernetes resource itself, this is an expensive fetch and should be used sparingly */ resource?: Maybe; + /** the plural service managing this component, if any */ + service?: Maybe; version: Scalars['String']['output']; }; @@ -2948,6 +2950,8 @@ export type ClusterInsightComponentAttributes = { name: Scalars['String']['input']; namespace?: InputMaybe; priority?: InputMaybe; + /** the plural service that manages this component, if any */ + serviceId?: InputMaybe; version: Scalars['String']['input']; }; @@ -19920,9 +19924,16 @@ export type AlertEvidenceFragment = { __typename?: 'AlertEvidence', alertId?: st export type KnowledgeEvidenceFragment = { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null }; -export type ClusterInsightComponentFragment = { __typename?: 'ClusterInsightComponent', id: string, kind: string, name: string, namespace?: string | null, group?: string | null, version: string, priority?: InsightComponentPriority | null, insight?: { __typename?: 'AiInsight', id: string, text?: string | null, summary?: string | null, sha?: string | null, freshness?: InsightFreshness | null, updatedAt?: string | null, insertedAt?: string | null, error?: Array<{ __typename?: 'ServiceError', message: string, source: string } | null> | null, evidence?: Array<{ __typename?: 'AiInsightEvidence', id: string, type: EvidenceType, insertedAt?: string | null, updatedAt?: string | null, logs?: { __typename?: 'LogsEvidence', clusterId?: string | null, serviceId?: string | null, line?: string | null, lines?: Array<{ __typename?: 'LogLine', log?: string | null, timestamp?: string | null, facets?: Array<{ __typename?: 'LogFacet', key: string, value?: string | null } | null> | null } | null> | null } | null, pullRequest?: { __typename?: 'PullRequestEvidence', contents?: string | null, filename?: string | null, patch?: string | null, repo?: string | null, sha?: string | null, title?: string | null, url?: string | null } | null, alert?: { __typename?: 'AlertEvidence', alertId?: string | null, title?: string | null, resolution?: string | null } | null, knowledge?: { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null } | null, clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, group?: string | null, version: string, kind: string, name: string, namespace?: string | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, serviceComponent?: { __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null } | null, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string, type: StackType } | null, stackRun?: { __typename?: 'StackRun', id: string, message?: string | null, type: StackType, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string } | null } | null, alert?: { __typename?: 'Alert', id: string, title?: string | null, message?: string | null } | null } | null }; +export type ClusterInsightComponentFragment = { __typename?: 'ClusterInsightComponent', id: string, kind: string, name: string, namespace?: string | null, group?: string | null, version: string, priority?: InsightComponentPriority | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string } | null } | null, insight?: { __typename?: 'AiInsight', id: string, text?: string | null, summary?: string | null, sha?: string | null, freshness?: InsightFreshness | null, updatedAt?: string | null, insertedAt?: string | null, error?: Array<{ __typename?: 'ServiceError', message: string, source: string } | null> | null, evidence?: Array<{ __typename?: 'AiInsightEvidence', id: string, type: EvidenceType, insertedAt?: string | null, updatedAt?: string | null, logs?: { __typename?: 'LogsEvidence', clusterId?: string | null, serviceId?: string | null, line?: string | null, lines?: Array<{ __typename?: 'LogLine', log?: string | null, timestamp?: string | null, facets?: Array<{ __typename?: 'LogFacet', key: string, value?: string | null } | null> | null } | null> | null } | null, pullRequest?: { __typename?: 'PullRequestEvidence', contents?: string | null, filename?: string | null, patch?: string | null, repo?: string | null, sha?: string | null, title?: string | null, url?: string | null } | null, alert?: { __typename?: 'AlertEvidence', alertId?: string | null, title?: string | null, resolution?: string | null } | null, knowledge?: { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null } | null, clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, group?: string | null, version: string, kind: string, name: string, namespace?: string | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, serviceComponent?: { __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null } | null, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string, type: StackType } | null, stackRun?: { __typename?: 'StackRun', id: string, message?: string | null, type: StackType, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string } | null } | null, alert?: { __typename?: 'Alert', id: string, title?: string | null, message?: string | null } | null } | null }; + +export type ClusterInsightComponentDetailFragment = { __typename?: 'ClusterInsightComponent', id: string, kind: string, name: string, namespace?: string | null, group?: string | null, version: string, priority?: InsightComponentPriority | null, resource?: { __typename?: 'KubernetesUnstructured', raw?: Record | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string } | null } | null, insight?: { __typename?: 'AiInsight', id: string, text?: string | null, summary?: string | null, sha?: string | null, freshness?: InsightFreshness | null, updatedAt?: string | null, insertedAt?: string | null, error?: Array<{ __typename?: 'ServiceError', message: string, source: string } | null> | null, evidence?: Array<{ __typename?: 'AiInsightEvidence', id: string, type: EvidenceType, insertedAt?: string | null, updatedAt?: string | null, logs?: { __typename?: 'LogsEvidence', clusterId?: string | null, serviceId?: string | null, line?: string | null, lines?: Array<{ __typename?: 'LogLine', log?: string | null, timestamp?: string | null, facets?: Array<{ __typename?: 'LogFacet', key: string, value?: string | null } | null> | null } | null> | null } | null, pullRequest?: { __typename?: 'PullRequestEvidence', contents?: string | null, filename?: string | null, patch?: string | null, repo?: string | null, sha?: string | null, title?: string | null, url?: string | null } | null, alert?: { __typename?: 'AlertEvidence', alertId?: string | null, title?: string | null, resolution?: string | null } | null, knowledge?: { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null } | null, clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, group?: string | null, version: string, kind: string, name: string, namespace?: string | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, serviceComponent?: { __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null } | null, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string, type: StackType } | null, stackRun?: { __typename?: 'StackRun', id: string, message?: string | null, type: StackType, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string } | null } | null, alert?: { __typename?: 'Alert', id: string, title?: string | null, message?: string | null } | null } | null }; + +export type ServiceComponentIdentitiesQueryVariables = Exact<{ + id: Scalars['ID']['input']; +}>; + -export type ClusterInsightComponentDetailFragment = { __typename?: 'ClusterInsightComponent', id: string, kind: string, name: string, namespace?: string | null, group?: string | null, version: string, priority?: InsightComponentPriority | null, resource?: { __typename?: 'KubernetesUnstructured', raw?: Record | null } | null, insight?: { __typename?: 'AiInsight', id: string, text?: string | null, summary?: string | null, sha?: string | null, freshness?: InsightFreshness | null, updatedAt?: string | null, insertedAt?: string | null, error?: Array<{ __typename?: 'ServiceError', message: string, source: string } | null> | null, evidence?: Array<{ __typename?: 'AiInsightEvidence', id: string, type: EvidenceType, insertedAt?: string | null, updatedAt?: string | null, logs?: { __typename?: 'LogsEvidence', clusterId?: string | null, serviceId?: string | null, line?: string | null, lines?: Array<{ __typename?: 'LogLine', log?: string | null, timestamp?: string | null, facets?: Array<{ __typename?: 'LogFacet', key: string, value?: string | null } | null> | null } | null> | null } | null, pullRequest?: { __typename?: 'PullRequestEvidence', contents?: string | null, filename?: string | null, patch?: string | null, repo?: string | null, sha?: string | null, title?: string | null, url?: string | null } | null, alert?: { __typename?: 'AlertEvidence', alertId?: string | null, title?: string | null, resolution?: string | null } | null, knowledge?: { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null } | null, clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, group?: string | null, version: string, kind: string, name: string, namespace?: string | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, serviceComponent?: { __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null } | null, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string, type: StackType } | null, stackRun?: { __typename?: 'StackRun', id: string, message?: string | null, type: StackType, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string } | null } | null, alert?: { __typename?: 'Alert', id: string, title?: string | null, message?: string | null } | null } | null }; +export type ServiceComponentIdentitiesQuery = { __typename?: 'RootQueryType', serviceDeployment?: { __typename?: 'ServiceDeployment', id: string, components?: Array<{ __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, namespace?: string | null, name: string } | null> | null } | null }; export type AiInsightQueryVariables = Exact<{ id: Scalars['ID']['input']; @@ -19953,7 +19964,7 @@ export type ClusterInsightComponentQueryVariables = Exact<{ }>; -export type ClusterInsightComponentQuery = { __typename?: 'RootQueryType', clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, kind: string, name: string, namespace?: string | null, group?: string | null, version: string, priority?: InsightComponentPriority | null, resource?: { __typename?: 'KubernetesUnstructured', raw?: Record | null } | null, insight?: { __typename?: 'AiInsight', id: string, text?: string | null, summary?: string | null, sha?: string | null, freshness?: InsightFreshness | null, updatedAt?: string | null, insertedAt?: string | null, error?: Array<{ __typename?: 'ServiceError', message: string, source: string } | null> | null, evidence?: Array<{ __typename?: 'AiInsightEvidence', id: string, type: EvidenceType, insertedAt?: string | null, updatedAt?: string | null, logs?: { __typename?: 'LogsEvidence', clusterId?: string | null, serviceId?: string | null, line?: string | null, lines?: Array<{ __typename?: 'LogLine', log?: string | null, timestamp?: string | null, facets?: Array<{ __typename?: 'LogFacet', key: string, value?: string | null } | null> | null } | null> | null } | null, pullRequest?: { __typename?: 'PullRequestEvidence', contents?: string | null, filename?: string | null, patch?: string | null, repo?: string | null, sha?: string | null, title?: string | null, url?: string | null } | null, alert?: { __typename?: 'AlertEvidence', alertId?: string | null, title?: string | null, resolution?: string | null } | null, knowledge?: { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null } | null, clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, group?: string | null, version: string, kind: string, name: string, namespace?: string | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, serviceComponent?: { __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null } | null, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string, type: StackType } | null, stackRun?: { __typename?: 'StackRun', id: string, message?: string | null, type: StackType, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string } | null } | null, alert?: { __typename?: 'Alert', id: string, title?: string | null, message?: string | null } | null } | null } | null }; +export type ClusterInsightComponentQuery = { __typename?: 'RootQueryType', clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, kind: string, name: string, namespace?: string | null, group?: string | null, version: string, priority?: InsightComponentPriority | null, resource?: { __typename?: 'KubernetesUnstructured', raw?: Record | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string } | null } | null, insight?: { __typename?: 'AiInsight', id: string, text?: string | null, summary?: string | null, sha?: string | null, freshness?: InsightFreshness | null, updatedAt?: string | null, insertedAt?: string | null, error?: Array<{ __typename?: 'ServiceError', message: string, source: string } | null> | null, evidence?: Array<{ __typename?: 'AiInsightEvidence', id: string, type: EvidenceType, insertedAt?: string | null, updatedAt?: string | null, logs?: { __typename?: 'LogsEvidence', clusterId?: string | null, serviceId?: string | null, line?: string | null, lines?: Array<{ __typename?: 'LogLine', log?: string | null, timestamp?: string | null, facets?: Array<{ __typename?: 'LogFacet', key: string, value?: string | null } | null> | null } | null> | null } | null, pullRequest?: { __typename?: 'PullRequestEvidence', contents?: string | null, filename?: string | null, patch?: string | null, repo?: string | null, sha?: string | null, title?: string | null, url?: string | null } | null, alert?: { __typename?: 'AlertEvidence', alertId?: string | null, title?: string | null, resolution?: string | null } | null, knowledge?: { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null } | null, clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, group?: string | null, version: string, kind: string, name: string, namespace?: string | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, serviceComponent?: { __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null } | null, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string, type: StackType } | null, stackRun?: { __typename?: 'StackRun', id: string, message?: string | null, type: StackType, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string } | null } | null, alert?: { __typename?: 'Alert', id: string, title?: string | null, message?: string | null } | null } | null } | null }; export type AiFixPrMutationVariables = Exact<{ insightId: Scalars['ID']['input']; @@ -20535,7 +20546,7 @@ export type ClusterNodeMetricsFragment = { __typename?: 'Cluster', id: string, c export type ClusterFragment = { __typename?: 'Cluster', currentVersion?: string | null, id: string, name: string, handle?: string | null, metadata?: Record | null, pingedAt?: string | null, self?: boolean | null, version?: string | null, healthy?: boolean | null, healthScore?: number | null, protect?: boolean | null, distro?: ClusterDistro | null, cpuTotal?: number | null, memoryTotal?: number | null, cpuUtil?: number | null, nodeCount?: number | null, namespaceCount?: number | null, availabilityZones?: Array | null, podCount?: number | null, memoryUtil?: number | null, installed?: boolean | null, deletedAt?: string | null, kubeletVersion?: string | null, virtual?: boolean | null, disableAi?: boolean | null, apiDeprecations?: Array<{ __typename?: 'ApiDeprecation', availableIn?: string | null, blocking?: boolean | null, deprecatedIn?: string | null, removedIn?: string | null, replacement?: string | null, component?: { __typename?: 'ServiceComponent', group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', git?: { __typename?: 'GitRef', ref: string, folder: string } | null, repository?: { __typename?: 'GitRepository', httpsPath?: string | null, urlFormat?: string | null } | null } | null } | null } | null> | null, nodePools?: Array<{ __typename?: 'NodePool', id: string, name: string, minSize: number, maxSize: number, instanceType: string, spot?: boolean | null, labels?: Record | null, taints?: Array<{ __typename?: 'Taint', effect: string, key: string, value: string } | null> | null } | null> | null, provider?: { __typename?: 'ClusterProvider', id: string, cloud: string, name: string, namespace: string, supportedVersions?: Array | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, repository?: { __typename?: 'GitRepository', url: string } | null } | null, tags?: Array<{ __typename?: 'Tag', name: string, value: string } | null> | null, parentCluster?: { __typename?: 'Cluster', currentVersion?: string | null, id: string, self?: boolean | null, healthy?: boolean | null, healthScore?: number | null, protect?: boolean | null, name: string, handle?: string | null, distro?: ClusterDistro | null, cpuTotal?: number | null, memoryTotal?: number | null, cpuUtil?: number | null, nodeCount?: number | null, namespaceCount?: number | null, availabilityZones?: Array | null, podCount?: number | null, memoryUtil?: number | null, installed?: boolean | null, pingedAt?: string | null, deletedAt?: string | null, version?: string | null, kubeletVersion?: string | null, virtual?: boolean | null, disableAi?: boolean | null, provider?: { __typename?: 'ClusterProvider', id: string, cloud: string, name: string, namespace: string, supportedVersions?: Array | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, repository?: { __typename?: 'GitRepository', url: string } | null } | null, tags?: Array<{ __typename?: 'Tag', name: string, value: string } | null> | null, upgradePlan?: { __typename?: 'ClusterUpgradePlan', compatibilities?: boolean | null, deprecations?: boolean | null, incompatibilities?: boolean | null, kubeletSkew?: boolean | null } | null, insight?: { __typename?: 'AiInsight', id: string, summary?: string | null, freshness?: InsightFreshness | null, insertedAt?: string | null, updatedAt?: string | null, evidence?: Array<{ __typename?: 'AiInsightEvidence', id: string, type: EvidenceType, insertedAt?: string | null, updatedAt?: string | null, logs?: { __typename?: 'LogsEvidence', clusterId?: string | null, serviceId?: string | null, line?: string | null, lines?: Array<{ __typename?: 'LogLine', log?: string | null, timestamp?: string | null, facets?: Array<{ __typename?: 'LogFacet', key: string, value?: string | null } | null> | null } | null> | null } | null, pullRequest?: { __typename?: 'PullRequestEvidence', contents?: string | null, filename?: string | null, patch?: string | null, repo?: string | null, sha?: string | null, title?: string | null, url?: string | null } | null, alert?: { __typename?: 'AlertEvidence', alertId?: string | null, title?: string | null, resolution?: string | null } | null, knowledge?: { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null } | null, clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, group?: string | null, version: string, kind: string, name: string, namespace?: string | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, serviceComponent?: { __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null } | null, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string, type: StackType } | null, stackRun?: { __typename?: 'StackRun', id: string, message?: string | null, type: StackType, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string } | null } | null, alert?: { __typename?: 'Alert', id: string, title?: string | null, message?: string | null } | null } | null, extendedSupport?: { __typename?: 'ExtendedSupportInfo', extended?: boolean | null, extendedFrom?: string | null } | null } | null, operationalLayout?: { __typename?: 'OperationalLayout', serviceMesh?: ServiceMesh | null } | null, alerts?: { __typename?: 'AlertConnection', edges?: Array<{ __typename?: 'AlertEdge', node?: { __typename?: 'Alert', id: string } | null } | null> | null } | null, upgradePlan?: { __typename?: 'ClusterUpgradePlan', compatibilities?: boolean | null, deprecations?: boolean | null, incompatibilities?: boolean | null, kubeletSkew?: boolean | null } | null, insight?: { __typename?: 'AiInsight', id: string, summary?: string | null, freshness?: InsightFreshness | null, insertedAt?: string | null, updatedAt?: string | null, evidence?: Array<{ __typename?: 'AiInsightEvidence', id: string, type: EvidenceType, insertedAt?: string | null, updatedAt?: string | null, logs?: { __typename?: 'LogsEvidence', clusterId?: string | null, serviceId?: string | null, line?: string | null, lines?: Array<{ __typename?: 'LogLine', log?: string | null, timestamp?: string | null, facets?: Array<{ __typename?: 'LogFacet', key: string, value?: string | null } | null> | null } | null> | null } | null, pullRequest?: { __typename?: 'PullRequestEvidence', contents?: string | null, filename?: string | null, patch?: string | null, repo?: string | null, sha?: string | null, title?: string | null, url?: string | null } | null, alert?: { __typename?: 'AlertEvidence', alertId?: string | null, title?: string | null, resolution?: string | null } | null, knowledge?: { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null } | null, clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, group?: string | null, version: string, kind: string, name: string, namespace?: string | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, serviceComponent?: { __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null } | null, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string, type: StackType } | null, stackRun?: { __typename?: 'StackRun', id: string, message?: string | null, type: StackType, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string } | null } | null, alert?: { __typename?: 'Alert', id: string, title?: string | null, message?: string | null } | null } | null, extendedSupport?: { __typename?: 'ExtendedSupportInfo', extended?: boolean | null, extendedFrom?: string | null } | null }; -export type ClusterInsightFragment = { __typename?: 'Cluster', id: string, insightComponents?: Array<{ __typename?: 'ClusterInsightComponent', id: string, kind: string, name: string, namespace?: string | null, group?: string | null, version: string, priority?: InsightComponentPriority | null, insight?: { __typename?: 'AiInsight', id: string, text?: string | null, summary?: string | null, sha?: string | null, freshness?: InsightFreshness | null, updatedAt?: string | null, insertedAt?: string | null, error?: Array<{ __typename?: 'ServiceError', message: string, source: string } | null> | null, evidence?: Array<{ __typename?: 'AiInsightEvidence', id: string, type: EvidenceType, insertedAt?: string | null, updatedAt?: string | null, logs?: { __typename?: 'LogsEvidence', clusterId?: string | null, serviceId?: string | null, line?: string | null, lines?: Array<{ __typename?: 'LogLine', log?: string | null, timestamp?: string | null, facets?: Array<{ __typename?: 'LogFacet', key: string, value?: string | null } | null> | null } | null> | null } | null, pullRequest?: { __typename?: 'PullRequestEvidence', contents?: string | null, filename?: string | null, patch?: string | null, repo?: string | null, sha?: string | null, title?: string | null, url?: string | null } | null, alert?: { __typename?: 'AlertEvidence', alertId?: string | null, title?: string | null, resolution?: string | null } | null, knowledge?: { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null } | null, clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, group?: string | null, version: string, kind: string, name: string, namespace?: string | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, serviceComponent?: { __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null } | null, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string, type: StackType } | null, stackRun?: { __typename?: 'StackRun', id: string, message?: string | null, type: StackType, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string } | null } | null, alert?: { __typename?: 'Alert', id: string, title?: string | null, message?: string | null } | null } | null } | null> | null }; +export type ClusterInsightFragment = { __typename?: 'Cluster', id: string, insightComponents?: Array<{ __typename?: 'ClusterInsightComponent', id: string, kind: string, name: string, namespace?: string | null, group?: string | null, version: string, priority?: InsightComponentPriority | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string } | null } | null, insight?: { __typename?: 'AiInsight', id: string, text?: string | null, summary?: string | null, sha?: string | null, freshness?: InsightFreshness | null, updatedAt?: string | null, insertedAt?: string | null, error?: Array<{ __typename?: 'ServiceError', message: string, source: string } | null> | null, evidence?: Array<{ __typename?: 'AiInsightEvidence', id: string, type: EvidenceType, insertedAt?: string | null, updatedAt?: string | null, logs?: { __typename?: 'LogsEvidence', clusterId?: string | null, serviceId?: string | null, line?: string | null, lines?: Array<{ __typename?: 'LogLine', log?: string | null, timestamp?: string | null, facets?: Array<{ __typename?: 'LogFacet', key: string, value?: string | null } | null> | null } | null> | null } | null, pullRequest?: { __typename?: 'PullRequestEvidence', contents?: string | null, filename?: string | null, patch?: string | null, repo?: string | null, sha?: string | null, title?: string | null, url?: string | null } | null, alert?: { __typename?: 'AlertEvidence', alertId?: string | null, title?: string | null, resolution?: string | null } | null, knowledge?: { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null } | null, clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, group?: string | null, version: string, kind: string, name: string, namespace?: string | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, serviceComponent?: { __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null } | null, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string, type: StackType } | null, stackRun?: { __typename?: 'StackRun', id: string, message?: string | null, type: StackType, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string } | null } | null, alert?: { __typename?: 'Alert', id: string, title?: string | null, message?: string | null } | null } | null } | null> | null }; export type CloudAddonFragment = { __typename?: 'CloudAddon', id: string, insertedAt?: string | null, updatedAt?: string | null, name: string, distro: ClusterDistro, version: string, info?: { __typename?: 'CloudAddonInformation', name?: string | null, publisher?: string | null, versions?: Array<{ __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null> | null } | null, versionInfo?: { __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null }; @@ -20620,7 +20631,7 @@ export type ClusterInsightQueryVariables = Exact<{ }>; -export type ClusterInsightQuery = { __typename?: 'RootQueryType', cluster?: { __typename?: 'Cluster', id: string, insightComponents?: Array<{ __typename?: 'ClusterInsightComponent', id: string, kind: string, name: string, namespace?: string | null, group?: string | null, version: string, priority?: InsightComponentPriority | null, insight?: { __typename?: 'AiInsight', id: string, text?: string | null, summary?: string | null, sha?: string | null, freshness?: InsightFreshness | null, updatedAt?: string | null, insertedAt?: string | null, error?: Array<{ __typename?: 'ServiceError', message: string, source: string } | null> | null, evidence?: Array<{ __typename?: 'AiInsightEvidence', id: string, type: EvidenceType, insertedAt?: string | null, updatedAt?: string | null, logs?: { __typename?: 'LogsEvidence', clusterId?: string | null, serviceId?: string | null, line?: string | null, lines?: Array<{ __typename?: 'LogLine', log?: string | null, timestamp?: string | null, facets?: Array<{ __typename?: 'LogFacet', key: string, value?: string | null } | null> | null } | null> | null } | null, pullRequest?: { __typename?: 'PullRequestEvidence', contents?: string | null, filename?: string | null, patch?: string | null, repo?: string | null, sha?: string | null, title?: string | null, url?: string | null } | null, alert?: { __typename?: 'AlertEvidence', alertId?: string | null, title?: string | null, resolution?: string | null } | null, knowledge?: { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null } | null, clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, group?: string | null, version: string, kind: string, name: string, namespace?: string | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, serviceComponent?: { __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null } | null, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string, type: StackType } | null, stackRun?: { __typename?: 'StackRun', id: string, message?: string | null, type: StackType, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string } | null } | null, alert?: { __typename?: 'Alert', id: string, title?: string | null, message?: string | null } | null } | null } | null> | null } | null }; +export type ClusterInsightQuery = { __typename?: 'RootQueryType', cluster?: { __typename?: 'Cluster', id: string, insightComponents?: Array<{ __typename?: 'ClusterInsightComponent', id: string, kind: string, name: string, namespace?: string | null, group?: string | null, version: string, priority?: InsightComponentPriority | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string } | null } | null, insight?: { __typename?: 'AiInsight', id: string, text?: string | null, summary?: string | null, sha?: string | null, freshness?: InsightFreshness | null, updatedAt?: string | null, insertedAt?: string | null, error?: Array<{ __typename?: 'ServiceError', message: string, source: string } | null> | null, evidence?: Array<{ __typename?: 'AiInsightEvidence', id: string, type: EvidenceType, insertedAt?: string | null, updatedAt?: string | null, logs?: { __typename?: 'LogsEvidence', clusterId?: string | null, serviceId?: string | null, line?: string | null, lines?: Array<{ __typename?: 'LogLine', log?: string | null, timestamp?: string | null, facets?: Array<{ __typename?: 'LogFacet', key: string, value?: string | null } | null> | null } | null> | null } | null, pullRequest?: { __typename?: 'PullRequestEvidence', contents?: string | null, filename?: string | null, patch?: string | null, repo?: string | null, sha?: string | null, title?: string | null, url?: string | null } | null, alert?: { __typename?: 'AlertEvidence', alertId?: string | null, title?: string | null, resolution?: string | null } | null, knowledge?: { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null } | null, clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, group?: string | null, version: string, kind: string, name: string, namespace?: string | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, serviceComponent?: { __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null } | null, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string, type: StackType } | null, stackRun?: { __typename?: 'StackRun', id: string, message?: string | null, type: StackType, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string } | null } | null, alert?: { __typename?: 'Alert', id: string, title?: string | null, message?: string | null } | null } | null } | null> | null } | null }; export type ClusterNodesQueryVariables = Exact<{ id: Scalars['ID']['input']; @@ -21922,7 +21933,7 @@ export type CloudAddonUpgradeFragment = { __typename?: 'CloudAddonUpgrade', addo export type ClusterWithUpgradeFragment = { __typename?: 'Cluster', id: string, name: string, currentVersion?: string | null, version?: string | null, deprecatedCustomResources?: Array<{ __typename?: 'DeprecatedCustomResource', name?: string | null, group: string, kind: string, namespace?: string | null, version: string, nextVersion: string } | null> | null, upgradePlanSummary?: { __typename?: 'UpgradePlanSummary', blockingAddons?: Array<{ __typename?: 'RuntimeAddonUpgrade', callout?: string | null, addon?: { __typename?: 'RuntimeAddon', name: string, icon?: string | null } | null, current?: { __typename?: 'AddonVersion', version?: string | null, chartVersion?: string | null, images?: Array | null, releaseUrl?: string | null } | null, fix?: { __typename?: 'AddonVersion', version?: string | null, chartVersion?: string | null, images?: Array | null, releaseUrl?: string | null, summary?: { __typename?: 'AddonVersionSummary', breakingChanges?: Array | null, chartUpdates?: Array | null, features?: Array | null, helmChanges?: string | null } | null } | null } | null> | null, blockingCloudAddons?: Array<{ __typename?: 'CloudAddonUpgrade', addon?: { __typename?: 'CloudAddon', id: string, insertedAt?: string | null, updatedAt?: string | null, name: string, distro: ClusterDistro, version: string, info?: { __typename?: 'CloudAddonInformation', name?: string | null, publisher?: string | null, versions?: Array<{ __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null> | null } | null, versionInfo?: { __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null } | null, current?: { __typename?: 'CloudAddonVersionInformation', version?: string | null } | null, fix?: { __typename?: 'CloudAddonVersionInformation', version?: string | null } | null } | null> | null } | null, runtimeServices?: Array<{ __typename?: 'RuntimeService', id: string, name: string, version: string, addon?: { __typename?: 'RuntimeAddon', icon?: string | null, versions?: Array<{ __typename?: 'AddonVersion', version?: string | null, kube?: Array | null, chartVersion?: string | null, incompatibilities?: Array<{ __typename?: 'VersionReference', version: string, name: string } | null> | null, requirements?: Array<{ __typename?: 'VersionReference', version: string, name: string } | null> | null } | null> | null } | null, service?: { __typename?: 'ServiceDeployment', git?: { __typename?: 'GitRef', ref: string, folder: string } | null, repository?: { __typename?: 'GitRepository', httpsPath?: string | null, urlFormat?: string | null } | null, helm?: { __typename?: 'HelmSpec', version?: string | null } | null } | null, addonVersion?: { __typename?: 'AddonVersion', blocking?: boolean | null, version?: string | null, kube?: Array | null, chartVersion?: string | null, incompatibilities?: Array<{ __typename?: 'VersionReference', version: string, name: string } | null> | null, requirements?: Array<{ __typename?: 'VersionReference', version: string, name: string } | null> | null } | null } | null> | null, apiDeprecations?: Array<{ __typename?: 'ApiDeprecation', availableIn?: string | null, blocking?: boolean | null, deprecatedIn?: string | null, removedIn?: string | null, replacement?: string | null, component?: { __typename?: 'ServiceComponent', group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', git?: { __typename?: 'GitRef', ref: string, folder: string } | null, repository?: { __typename?: 'GitRepository', httpsPath?: string | null, urlFormat?: string | null } | null } | null } | null } | null> | null, upgradeInsights?: Array<{ __typename?: 'UpgradeInsight', id: string, name: string, description?: string | null, refreshedAt?: string | null, transitionedAt?: string | null, version?: string | null, status?: UpgradeInsightStatus | null, details?: Array<{ __typename?: 'UpgradeInsightDetail', id: string, removedIn?: string | null, replacedIn?: string | null, replacement?: string | null, status?: UpgradeInsightStatus | null, used?: string | null, clientInfo?: Array<{ __typename?: 'InsightClientInfo', userAgent?: string | null, count?: string | null, lastRequestAt?: string | null } | null> | null } | null> | null } | null> | null, cloudAddons?: Array<{ __typename?: 'CloudAddon', id: string, insertedAt?: string | null, updatedAt?: string | null, name: string, distro: ClusterDistro, version: string, info?: { __typename?: 'CloudAddonInformation', name?: string | null, publisher?: string | null, versions?: Array<{ __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null> | null } | null, versionInfo?: { __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null } | null> | null }; -export type ClusterOverviewDetailsFragment = { __typename?: 'Cluster', currentVersion?: string | null, id: string, self?: boolean | null, healthy?: boolean | null, healthScore?: number | null, protect?: boolean | null, name: string, handle?: string | null, distro?: ClusterDistro | null, cpuTotal?: number | null, memoryTotal?: number | null, cpuUtil?: number | null, nodeCount?: number | null, namespaceCount?: number | null, availabilityZones?: Array | null, podCount?: number | null, memoryUtil?: number | null, installed?: boolean | null, pingedAt?: string | null, deletedAt?: string | null, version?: string | null, kubeletVersion?: string | null, virtual?: boolean | null, disableAi?: boolean | null, currentUpgrade?: { __typename?: 'ClusterUpgrade', id: string, status: ClusterUpgradeStatus, version?: string | null, steps?: Array<{ __typename?: 'ClusterUpgradeStep', id: string, name: string, prompt: string, status: ClusterUpgradeStatus, type: ClusterUpgradeStepType, error?: string | null, agentRun?: { __typename?: 'AgentRun', id: string, status: AgentRunStatus, mode: AgentRunMode, babysit?: boolean | null, approval?: boolean | null, approvedAt?: string | null, prompt: string, shared?: boolean | null, error?: string | null, repository: string, branch?: string | null, headBranch?: string | null, insertedAt?: string | null, updatedAt?: string | null, runtime?: { __typename?: 'AgentRuntime', id: string, name: string, type: AgentRuntimeType } | null, pullRequests?: Array<{ __typename?: 'PullRequest', id: string, url: string, title?: string | null, creator?: string | null, status?: PrStatus | null, insertedAt?: string | null, updatedAt?: string | null } | null> | null, podReference?: { __typename?: 'AgentPodReference', name: string, namespace: string } | null, usage?: { __typename?: 'AgentRunUsage', totalCost?: number | null, totalTokens?: number | null } | null, workbenchJob?: { __typename?: 'WorkbenchJob', id: string, workbench?: { __typename?: 'Workbench', id: string, name: string } | null } | null, upload?: { __typename?: 'AgentRunUpload', id: string, session?: string | null, patch?: string | null } | null, todos?: Array<{ __typename?: 'AgentTodo', title: string, description: string, done?: boolean | null } | null> | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null, runtime?: { __typename?: 'AgentRuntime', type: AgentRuntimeType, name: string } | null } | null, nodeStatistics?: Array<{ __typename?: 'NodeStatistic', id: string, name: string, pendingPods?: number | null, health?: NodeStatisticHealth | null, insertedAt?: string | null, updatedAt?: string | null, cluster?: { __typename?: 'Cluster', id: string } | null } | null> | null, provider?: { __typename?: 'ClusterProvider', id: string, cloud: string, name: string, namespace: string, supportedVersions?: Array | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, repository?: { __typename?: 'GitRepository', url: string } | null } | null, tags?: Array<{ __typename?: 'Tag', name: string, value: string } | null> | null, upgradePlan?: { __typename?: 'ClusterUpgradePlan', compatibilities?: boolean | null, deprecations?: boolean | null, incompatibilities?: boolean | null, kubeletSkew?: boolean | null } | null, insight?: { __typename?: 'AiInsight', id: string, summary?: string | null, freshness?: InsightFreshness | null, insertedAt?: string | null, updatedAt?: string | null, evidence?: Array<{ __typename?: 'AiInsightEvidence', id: string, type: EvidenceType, insertedAt?: string | null, updatedAt?: string | null, logs?: { __typename?: 'LogsEvidence', clusterId?: string | null, serviceId?: string | null, line?: string | null, lines?: Array<{ __typename?: 'LogLine', log?: string | null, timestamp?: string | null, facets?: Array<{ __typename?: 'LogFacet', key: string, value?: string | null } | null> | null } | null> | null } | null, pullRequest?: { __typename?: 'PullRequestEvidence', contents?: string | null, filename?: string | null, patch?: string | null, repo?: string | null, sha?: string | null, title?: string | null, url?: string | null } | null, alert?: { __typename?: 'AlertEvidence', alertId?: string | null, title?: string | null, resolution?: string | null } | null, knowledge?: { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null } | null, clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, group?: string | null, version: string, kind: string, name: string, namespace?: string | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, serviceComponent?: { __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null } | null, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string, type: StackType } | null, stackRun?: { __typename?: 'StackRun', id: string, message?: string | null, type: StackType, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string } | null } | null, alert?: { __typename?: 'Alert', id: string, title?: string | null, message?: string | null } | null } | null, extendedSupport?: { __typename?: 'ExtendedSupportInfo', extended?: boolean | null, extendedFrom?: string | null } | null, deprecatedCustomResources?: Array<{ __typename?: 'DeprecatedCustomResource', name?: string | null, group: string, kind: string, namespace?: string | null, version: string, nextVersion: string } | null> | null, upgradePlanSummary?: { __typename?: 'UpgradePlanSummary', blockingAddons?: Array<{ __typename?: 'RuntimeAddonUpgrade', callout?: string | null, addon?: { __typename?: 'RuntimeAddon', name: string, icon?: string | null } | null, current?: { __typename?: 'AddonVersion', version?: string | null, chartVersion?: string | null, images?: Array | null, releaseUrl?: string | null } | null, fix?: { __typename?: 'AddonVersion', version?: string | null, chartVersion?: string | null, images?: Array | null, releaseUrl?: string | null, summary?: { __typename?: 'AddonVersionSummary', breakingChanges?: Array | null, chartUpdates?: Array | null, features?: Array | null, helmChanges?: string | null } | null } | null } | null> | null, blockingCloudAddons?: Array<{ __typename?: 'CloudAddonUpgrade', addon?: { __typename?: 'CloudAddon', id: string, insertedAt?: string | null, updatedAt?: string | null, name: string, distro: ClusterDistro, version: string, info?: { __typename?: 'CloudAddonInformation', name?: string | null, publisher?: string | null, versions?: Array<{ __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null> | null } | null, versionInfo?: { __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null } | null, current?: { __typename?: 'CloudAddonVersionInformation', version?: string | null } | null, fix?: { __typename?: 'CloudAddonVersionInformation', version?: string | null } | null } | null> | null } | null, insightComponents?: Array<{ __typename?: 'ClusterInsightComponent', id: string, kind: string, name: string, namespace?: string | null, group?: string | null, version: string, priority?: InsightComponentPriority | null, insight?: { __typename?: 'AiInsight', id: string, text?: string | null, summary?: string | null, sha?: string | null, freshness?: InsightFreshness | null, updatedAt?: string | null, insertedAt?: string | null, error?: Array<{ __typename?: 'ServiceError', message: string, source: string } | null> | null, evidence?: Array<{ __typename?: 'AiInsightEvidence', id: string, type: EvidenceType, insertedAt?: string | null, updatedAt?: string | null, logs?: { __typename?: 'LogsEvidence', clusterId?: string | null, serviceId?: string | null, line?: string | null, lines?: Array<{ __typename?: 'LogLine', log?: string | null, timestamp?: string | null, facets?: Array<{ __typename?: 'LogFacet', key: string, value?: string | null } | null> | null } | null> | null } | null, pullRequest?: { __typename?: 'PullRequestEvidence', contents?: string | null, filename?: string | null, patch?: string | null, repo?: string | null, sha?: string | null, title?: string | null, url?: string | null } | null, alert?: { __typename?: 'AlertEvidence', alertId?: string | null, title?: string | null, resolution?: string | null } | null, knowledge?: { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null } | null, clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, group?: string | null, version: string, kind: string, name: string, namespace?: string | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, serviceComponent?: { __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null } | null, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string, type: StackType } | null, stackRun?: { __typename?: 'StackRun', id: string, message?: string | null, type: StackType, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string } | null } | null, alert?: { __typename?: 'Alert', id: string, title?: string | null, message?: string | null } | null } | null } | null> | null, runtimeServices?: Array<{ __typename?: 'RuntimeService', id: string, name: string, version: string, addon?: { __typename?: 'RuntimeAddon', icon?: string | null, versions?: Array<{ __typename?: 'AddonVersion', version?: string | null, kube?: Array | null, chartVersion?: string | null, incompatibilities?: Array<{ __typename?: 'VersionReference', version: string, name: string } | null> | null, requirements?: Array<{ __typename?: 'VersionReference', version: string, name: string } | null> | null } | null> | null } | null, service?: { __typename?: 'ServiceDeployment', git?: { __typename?: 'GitRef', ref: string, folder: string } | null, repository?: { __typename?: 'GitRepository', httpsPath?: string | null, urlFormat?: string | null } | null, helm?: { __typename?: 'HelmSpec', version?: string | null } | null } | null, addonVersion?: { __typename?: 'AddonVersion', blocking?: boolean | null, version?: string | null, kube?: Array | null, chartVersion?: string | null, incompatibilities?: Array<{ __typename?: 'VersionReference', version: string, name: string } | null> | null, requirements?: Array<{ __typename?: 'VersionReference', version: string, name: string } | null> | null } | null } | null> | null, apiDeprecations?: Array<{ __typename?: 'ApiDeprecation', availableIn?: string | null, blocking?: boolean | null, deprecatedIn?: string | null, removedIn?: string | null, replacement?: string | null, component?: { __typename?: 'ServiceComponent', group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', git?: { __typename?: 'GitRef', ref: string, folder: string } | null, repository?: { __typename?: 'GitRepository', httpsPath?: string | null, urlFormat?: string | null } | null } | null } | null } | null> | null, upgradeInsights?: Array<{ __typename?: 'UpgradeInsight', id: string, name: string, description?: string | null, refreshedAt?: string | null, transitionedAt?: string | null, version?: string | null, status?: UpgradeInsightStatus | null, details?: Array<{ __typename?: 'UpgradeInsightDetail', id: string, removedIn?: string | null, replacedIn?: string | null, replacement?: string | null, status?: UpgradeInsightStatus | null, used?: string | null, clientInfo?: Array<{ __typename?: 'InsightClientInfo', userAgent?: string | null, count?: string | null, lastRequestAt?: string | null } | null> | null } | null> | null } | null> | null, cloudAddons?: Array<{ __typename?: 'CloudAddon', id: string, insertedAt?: string | null, updatedAt?: string | null, name: string, distro: ClusterDistro, version: string, info?: { __typename?: 'CloudAddonInformation', name?: string | null, publisher?: string | null, versions?: Array<{ __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null> | null } | null, versionInfo?: { __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null } | null> | null }; +export type ClusterOverviewDetailsFragment = { __typename?: 'Cluster', currentVersion?: string | null, id: string, self?: boolean | null, healthy?: boolean | null, healthScore?: number | null, protect?: boolean | null, name: string, handle?: string | null, distro?: ClusterDistro | null, cpuTotal?: number | null, memoryTotal?: number | null, cpuUtil?: number | null, nodeCount?: number | null, namespaceCount?: number | null, availabilityZones?: Array | null, podCount?: number | null, memoryUtil?: number | null, installed?: boolean | null, pingedAt?: string | null, deletedAt?: string | null, version?: string | null, kubeletVersion?: string | null, virtual?: boolean | null, disableAi?: boolean | null, currentUpgrade?: { __typename?: 'ClusterUpgrade', id: string, status: ClusterUpgradeStatus, version?: string | null, steps?: Array<{ __typename?: 'ClusterUpgradeStep', id: string, name: string, prompt: string, status: ClusterUpgradeStatus, type: ClusterUpgradeStepType, error?: string | null, agentRun?: { __typename?: 'AgentRun', id: string, status: AgentRunStatus, mode: AgentRunMode, babysit?: boolean | null, approval?: boolean | null, approvedAt?: string | null, prompt: string, shared?: boolean | null, error?: string | null, repository: string, branch?: string | null, headBranch?: string | null, insertedAt?: string | null, updatedAt?: string | null, runtime?: { __typename?: 'AgentRuntime', id: string, name: string, type: AgentRuntimeType } | null, pullRequests?: Array<{ __typename?: 'PullRequest', id: string, url: string, title?: string | null, creator?: string | null, status?: PrStatus | null, insertedAt?: string | null, updatedAt?: string | null } | null> | null, podReference?: { __typename?: 'AgentPodReference', name: string, namespace: string } | null, usage?: { __typename?: 'AgentRunUsage', totalCost?: number | null, totalTokens?: number | null } | null, workbenchJob?: { __typename?: 'WorkbenchJob', id: string, workbench?: { __typename?: 'Workbench', id: string, name: string } | null } | null, upload?: { __typename?: 'AgentRunUpload', id: string, session?: string | null, patch?: string | null } | null, todos?: Array<{ __typename?: 'AgentTodo', title: string, description: string, done?: boolean | null } | null> | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null, runtime?: { __typename?: 'AgentRuntime', type: AgentRuntimeType, name: string } | null } | null, nodeStatistics?: Array<{ __typename?: 'NodeStatistic', id: string, name: string, pendingPods?: number | null, health?: NodeStatisticHealth | null, insertedAt?: string | null, updatedAt?: string | null, cluster?: { __typename?: 'Cluster', id: string } | null } | null> | null, provider?: { __typename?: 'ClusterProvider', id: string, cloud: string, name: string, namespace: string, supportedVersions?: Array | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, repository?: { __typename?: 'GitRepository', url: string } | null } | null, tags?: Array<{ __typename?: 'Tag', name: string, value: string } | null> | null, upgradePlan?: { __typename?: 'ClusterUpgradePlan', compatibilities?: boolean | null, deprecations?: boolean | null, incompatibilities?: boolean | null, kubeletSkew?: boolean | null } | null, insight?: { __typename?: 'AiInsight', id: string, summary?: string | null, freshness?: InsightFreshness | null, insertedAt?: string | null, updatedAt?: string | null, evidence?: Array<{ __typename?: 'AiInsightEvidence', id: string, type: EvidenceType, insertedAt?: string | null, updatedAt?: string | null, logs?: { __typename?: 'LogsEvidence', clusterId?: string | null, serviceId?: string | null, line?: string | null, lines?: Array<{ __typename?: 'LogLine', log?: string | null, timestamp?: string | null, facets?: Array<{ __typename?: 'LogFacet', key: string, value?: string | null } | null> | null } | null> | null } | null, pullRequest?: { __typename?: 'PullRequestEvidence', contents?: string | null, filename?: string | null, patch?: string | null, repo?: string | null, sha?: string | null, title?: string | null, url?: string | null } | null, alert?: { __typename?: 'AlertEvidence', alertId?: string | null, title?: string | null, resolution?: string | null } | null, knowledge?: { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null } | null, clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, group?: string | null, version: string, kind: string, name: string, namespace?: string | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, serviceComponent?: { __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null } | null, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string, type: StackType } | null, stackRun?: { __typename?: 'StackRun', id: string, message?: string | null, type: StackType, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string } | null } | null, alert?: { __typename?: 'Alert', id: string, title?: string | null, message?: string | null } | null } | null, extendedSupport?: { __typename?: 'ExtendedSupportInfo', extended?: boolean | null, extendedFrom?: string | null } | null, deprecatedCustomResources?: Array<{ __typename?: 'DeprecatedCustomResource', name?: string | null, group: string, kind: string, namespace?: string | null, version: string, nextVersion: string } | null> | null, upgradePlanSummary?: { __typename?: 'UpgradePlanSummary', blockingAddons?: Array<{ __typename?: 'RuntimeAddonUpgrade', callout?: string | null, addon?: { __typename?: 'RuntimeAddon', name: string, icon?: string | null } | null, current?: { __typename?: 'AddonVersion', version?: string | null, chartVersion?: string | null, images?: Array | null, releaseUrl?: string | null } | null, fix?: { __typename?: 'AddonVersion', version?: string | null, chartVersion?: string | null, images?: Array | null, releaseUrl?: string | null, summary?: { __typename?: 'AddonVersionSummary', breakingChanges?: Array | null, chartUpdates?: Array | null, features?: Array | null, helmChanges?: string | null } | null } | null } | null> | null, blockingCloudAddons?: Array<{ __typename?: 'CloudAddonUpgrade', addon?: { __typename?: 'CloudAddon', id: string, insertedAt?: string | null, updatedAt?: string | null, name: string, distro: ClusterDistro, version: string, info?: { __typename?: 'CloudAddonInformation', name?: string | null, publisher?: string | null, versions?: Array<{ __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null> | null } | null, versionInfo?: { __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null } | null, current?: { __typename?: 'CloudAddonVersionInformation', version?: string | null } | null, fix?: { __typename?: 'CloudAddonVersionInformation', version?: string | null } | null } | null> | null } | null, insightComponents?: Array<{ __typename?: 'ClusterInsightComponent', id: string, kind: string, name: string, namespace?: string | null, group?: string | null, version: string, priority?: InsightComponentPriority | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string } | null } | null, insight?: { __typename?: 'AiInsight', id: string, text?: string | null, summary?: string | null, sha?: string | null, freshness?: InsightFreshness | null, updatedAt?: string | null, insertedAt?: string | null, error?: Array<{ __typename?: 'ServiceError', message: string, source: string } | null> | null, evidence?: Array<{ __typename?: 'AiInsightEvidence', id: string, type: EvidenceType, insertedAt?: string | null, updatedAt?: string | null, logs?: { __typename?: 'LogsEvidence', clusterId?: string | null, serviceId?: string | null, line?: string | null, lines?: Array<{ __typename?: 'LogLine', log?: string | null, timestamp?: string | null, facets?: Array<{ __typename?: 'LogFacet', key: string, value?: string | null } | null> | null } | null> | null } | null, pullRequest?: { __typename?: 'PullRequestEvidence', contents?: string | null, filename?: string | null, patch?: string | null, repo?: string | null, sha?: string | null, title?: string | null, url?: string | null } | null, alert?: { __typename?: 'AlertEvidence', alertId?: string | null, title?: string | null, resolution?: string | null } | null, knowledge?: { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null } | null, clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, group?: string | null, version: string, kind: string, name: string, namespace?: string | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, serviceComponent?: { __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null } | null, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string, type: StackType } | null, stackRun?: { __typename?: 'StackRun', id: string, message?: string | null, type: StackType, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string } | null } | null, alert?: { __typename?: 'Alert', id: string, title?: string | null, message?: string | null } | null } | null } | null> | null, runtimeServices?: Array<{ __typename?: 'RuntimeService', id: string, name: string, version: string, addon?: { __typename?: 'RuntimeAddon', icon?: string | null, versions?: Array<{ __typename?: 'AddonVersion', version?: string | null, kube?: Array | null, chartVersion?: string | null, incompatibilities?: Array<{ __typename?: 'VersionReference', version: string, name: string } | null> | null, requirements?: Array<{ __typename?: 'VersionReference', version: string, name: string } | null> | null } | null> | null } | null, service?: { __typename?: 'ServiceDeployment', git?: { __typename?: 'GitRef', ref: string, folder: string } | null, repository?: { __typename?: 'GitRepository', httpsPath?: string | null, urlFormat?: string | null } | null, helm?: { __typename?: 'HelmSpec', version?: string | null } | null } | null, addonVersion?: { __typename?: 'AddonVersion', blocking?: boolean | null, version?: string | null, kube?: Array | null, chartVersion?: string | null, incompatibilities?: Array<{ __typename?: 'VersionReference', version: string, name: string } | null> | null, requirements?: Array<{ __typename?: 'VersionReference', version: string, name: string } | null> | null } | null } | null> | null, apiDeprecations?: Array<{ __typename?: 'ApiDeprecation', availableIn?: string | null, blocking?: boolean | null, deprecatedIn?: string | null, removedIn?: string | null, replacement?: string | null, component?: { __typename?: 'ServiceComponent', group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', git?: { __typename?: 'GitRef', ref: string, folder: string } | null, repository?: { __typename?: 'GitRepository', httpsPath?: string | null, urlFormat?: string | null } | null } | null } | null } | null> | null, upgradeInsights?: Array<{ __typename?: 'UpgradeInsight', id: string, name: string, description?: string | null, refreshedAt?: string | null, transitionedAt?: string | null, version?: string | null, status?: UpgradeInsightStatus | null, details?: Array<{ __typename?: 'UpgradeInsightDetail', id: string, removedIn?: string | null, replacedIn?: string | null, replacement?: string | null, status?: UpgradeInsightStatus | null, used?: string | null, clientInfo?: Array<{ __typename?: 'InsightClientInfo', userAgent?: string | null, count?: string | null, lastRequestAt?: string | null } | null> | null } | null> | null } | null> | null, cloudAddons?: Array<{ __typename?: 'CloudAddon', id: string, insertedAt?: string | null, updatedAt?: string | null, name: string, distro: ClusterDistro, version: string, info?: { __typename?: 'CloudAddonInformation', name?: string | null, publisher?: string | null, versions?: Array<{ __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null> | null } | null, versionInfo?: { __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null } | null> | null }; export type ClusterHealthScoreFragment = { __typename?: 'Cluster', id: string, name: string, healthScore?: number | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null }; @@ -21936,7 +21947,7 @@ export type ClusterOverviewDetailsQueryVariables = Exact<{ }>; -export type ClusterOverviewDetailsQuery = { __typename?: 'RootQueryType', cluster?: { __typename?: 'Cluster', currentVersion?: string | null, id: string, self?: boolean | null, healthy?: boolean | null, healthScore?: number | null, protect?: boolean | null, name: string, handle?: string | null, distro?: ClusterDistro | null, cpuTotal?: number | null, memoryTotal?: number | null, cpuUtil?: number | null, nodeCount?: number | null, namespaceCount?: number | null, availabilityZones?: Array | null, podCount?: number | null, memoryUtil?: number | null, installed?: boolean | null, pingedAt?: string | null, deletedAt?: string | null, version?: string | null, kubeletVersion?: string | null, virtual?: boolean | null, disableAi?: boolean | null, currentUpgrade?: { __typename?: 'ClusterUpgrade', id: string, status: ClusterUpgradeStatus, version?: string | null, steps?: Array<{ __typename?: 'ClusterUpgradeStep', id: string, name: string, prompt: string, status: ClusterUpgradeStatus, type: ClusterUpgradeStepType, error?: string | null, agentRun?: { __typename?: 'AgentRun', id: string, status: AgentRunStatus, mode: AgentRunMode, babysit?: boolean | null, approval?: boolean | null, approvedAt?: string | null, prompt: string, shared?: boolean | null, error?: string | null, repository: string, branch?: string | null, headBranch?: string | null, insertedAt?: string | null, updatedAt?: string | null, runtime?: { __typename?: 'AgentRuntime', id: string, name: string, type: AgentRuntimeType } | null, pullRequests?: Array<{ __typename?: 'PullRequest', id: string, url: string, title?: string | null, creator?: string | null, status?: PrStatus | null, insertedAt?: string | null, updatedAt?: string | null } | null> | null, podReference?: { __typename?: 'AgentPodReference', name: string, namespace: string } | null, usage?: { __typename?: 'AgentRunUsage', totalCost?: number | null, totalTokens?: number | null } | null, workbenchJob?: { __typename?: 'WorkbenchJob', id: string, workbench?: { __typename?: 'Workbench', id: string, name: string } | null } | null, upload?: { __typename?: 'AgentRunUpload', id: string, session?: string | null, patch?: string | null } | null, todos?: Array<{ __typename?: 'AgentTodo', title: string, description: string, done?: boolean | null } | null> | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null, runtime?: { __typename?: 'AgentRuntime', type: AgentRuntimeType, name: string } | null } | null, nodeStatistics?: Array<{ __typename?: 'NodeStatistic', id: string, name: string, pendingPods?: number | null, health?: NodeStatisticHealth | null, insertedAt?: string | null, updatedAt?: string | null, cluster?: { __typename?: 'Cluster', id: string } | null } | null> | null, provider?: { __typename?: 'ClusterProvider', id: string, cloud: string, name: string, namespace: string, supportedVersions?: Array | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, repository?: { __typename?: 'GitRepository', url: string } | null } | null, tags?: Array<{ __typename?: 'Tag', name: string, value: string } | null> | null, upgradePlan?: { __typename?: 'ClusterUpgradePlan', compatibilities?: boolean | null, deprecations?: boolean | null, incompatibilities?: boolean | null, kubeletSkew?: boolean | null } | null, insight?: { __typename?: 'AiInsight', id: string, summary?: string | null, freshness?: InsightFreshness | null, insertedAt?: string | null, updatedAt?: string | null, evidence?: Array<{ __typename?: 'AiInsightEvidence', id: string, type: EvidenceType, insertedAt?: string | null, updatedAt?: string | null, logs?: { __typename?: 'LogsEvidence', clusterId?: string | null, serviceId?: string | null, line?: string | null, lines?: Array<{ __typename?: 'LogLine', log?: string | null, timestamp?: string | null, facets?: Array<{ __typename?: 'LogFacet', key: string, value?: string | null } | null> | null } | null> | null } | null, pullRequest?: { __typename?: 'PullRequestEvidence', contents?: string | null, filename?: string | null, patch?: string | null, repo?: string | null, sha?: string | null, title?: string | null, url?: string | null } | null, alert?: { __typename?: 'AlertEvidence', alertId?: string | null, title?: string | null, resolution?: string | null } | null, knowledge?: { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null } | null, clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, group?: string | null, version: string, kind: string, name: string, namespace?: string | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, serviceComponent?: { __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null } | null, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string, type: StackType } | null, stackRun?: { __typename?: 'StackRun', id: string, message?: string | null, type: StackType, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string } | null } | null, alert?: { __typename?: 'Alert', id: string, title?: string | null, message?: string | null } | null } | null, extendedSupport?: { __typename?: 'ExtendedSupportInfo', extended?: boolean | null, extendedFrom?: string | null } | null, deprecatedCustomResources?: Array<{ __typename?: 'DeprecatedCustomResource', name?: string | null, group: string, kind: string, namespace?: string | null, version: string, nextVersion: string } | null> | null, upgradePlanSummary?: { __typename?: 'UpgradePlanSummary', blockingAddons?: Array<{ __typename?: 'RuntimeAddonUpgrade', callout?: string | null, addon?: { __typename?: 'RuntimeAddon', name: string, icon?: string | null } | null, current?: { __typename?: 'AddonVersion', version?: string | null, chartVersion?: string | null, images?: Array | null, releaseUrl?: string | null } | null, fix?: { __typename?: 'AddonVersion', version?: string | null, chartVersion?: string | null, images?: Array | null, releaseUrl?: string | null, summary?: { __typename?: 'AddonVersionSummary', breakingChanges?: Array | null, chartUpdates?: Array | null, features?: Array | null, helmChanges?: string | null } | null } | null } | null> | null, blockingCloudAddons?: Array<{ __typename?: 'CloudAddonUpgrade', addon?: { __typename?: 'CloudAddon', id: string, insertedAt?: string | null, updatedAt?: string | null, name: string, distro: ClusterDistro, version: string, info?: { __typename?: 'CloudAddonInformation', name?: string | null, publisher?: string | null, versions?: Array<{ __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null> | null } | null, versionInfo?: { __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null } | null, current?: { __typename?: 'CloudAddonVersionInformation', version?: string | null } | null, fix?: { __typename?: 'CloudAddonVersionInformation', version?: string | null } | null } | null> | null } | null, insightComponents?: Array<{ __typename?: 'ClusterInsightComponent', id: string, kind: string, name: string, namespace?: string | null, group?: string | null, version: string, priority?: InsightComponentPriority | null, insight?: { __typename?: 'AiInsight', id: string, text?: string | null, summary?: string | null, sha?: string | null, freshness?: InsightFreshness | null, updatedAt?: string | null, insertedAt?: string | null, error?: Array<{ __typename?: 'ServiceError', message: string, source: string } | null> | null, evidence?: Array<{ __typename?: 'AiInsightEvidence', id: string, type: EvidenceType, insertedAt?: string | null, updatedAt?: string | null, logs?: { __typename?: 'LogsEvidence', clusterId?: string | null, serviceId?: string | null, line?: string | null, lines?: Array<{ __typename?: 'LogLine', log?: string | null, timestamp?: string | null, facets?: Array<{ __typename?: 'LogFacet', key: string, value?: string | null } | null> | null } | null> | null } | null, pullRequest?: { __typename?: 'PullRequestEvidence', contents?: string | null, filename?: string | null, patch?: string | null, repo?: string | null, sha?: string | null, title?: string | null, url?: string | null } | null, alert?: { __typename?: 'AlertEvidence', alertId?: string | null, title?: string | null, resolution?: string | null } | null, knowledge?: { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null } | null, clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, group?: string | null, version: string, kind: string, name: string, namespace?: string | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, serviceComponent?: { __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null } | null, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string, type: StackType } | null, stackRun?: { __typename?: 'StackRun', id: string, message?: string | null, type: StackType, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string } | null } | null, alert?: { __typename?: 'Alert', id: string, title?: string | null, message?: string | null } | null } | null } | null> | null, runtimeServices?: Array<{ __typename?: 'RuntimeService', id: string, name: string, version: string, addon?: { __typename?: 'RuntimeAddon', icon?: string | null, versions?: Array<{ __typename?: 'AddonVersion', version?: string | null, kube?: Array | null, chartVersion?: string | null, incompatibilities?: Array<{ __typename?: 'VersionReference', version: string, name: string } | null> | null, requirements?: Array<{ __typename?: 'VersionReference', version: string, name: string } | null> | null } | null> | null } | null, service?: { __typename?: 'ServiceDeployment', git?: { __typename?: 'GitRef', ref: string, folder: string } | null, repository?: { __typename?: 'GitRepository', httpsPath?: string | null, urlFormat?: string | null } | null, helm?: { __typename?: 'HelmSpec', version?: string | null } | null } | null, addonVersion?: { __typename?: 'AddonVersion', blocking?: boolean | null, version?: string | null, kube?: Array | null, chartVersion?: string | null, incompatibilities?: Array<{ __typename?: 'VersionReference', version: string, name: string } | null> | null, requirements?: Array<{ __typename?: 'VersionReference', version: string, name: string } | null> | null } | null } | null> | null, apiDeprecations?: Array<{ __typename?: 'ApiDeprecation', availableIn?: string | null, blocking?: boolean | null, deprecatedIn?: string | null, removedIn?: string | null, replacement?: string | null, component?: { __typename?: 'ServiceComponent', group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', git?: { __typename?: 'GitRef', ref: string, folder: string } | null, repository?: { __typename?: 'GitRepository', httpsPath?: string | null, urlFormat?: string | null } | null } | null } | null } | null> | null, upgradeInsights?: Array<{ __typename?: 'UpgradeInsight', id: string, name: string, description?: string | null, refreshedAt?: string | null, transitionedAt?: string | null, version?: string | null, status?: UpgradeInsightStatus | null, details?: Array<{ __typename?: 'UpgradeInsightDetail', id: string, removedIn?: string | null, replacedIn?: string | null, replacement?: string | null, status?: UpgradeInsightStatus | null, used?: string | null, clientInfo?: Array<{ __typename?: 'InsightClientInfo', userAgent?: string | null, count?: string | null, lastRequestAt?: string | null } | null> | null } | null> | null } | null> | null, cloudAddons?: Array<{ __typename?: 'CloudAddon', id: string, insertedAt?: string | null, updatedAt?: string | null, name: string, distro: ClusterDistro, version: string, info?: { __typename?: 'CloudAddonInformation', name?: string | null, publisher?: string | null, versions?: Array<{ __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null> | null } | null, versionInfo?: { __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null } | null> | null } | null, kubernetesChangelog?: { __typename?: 'KubernetesChangelog', version?: string | null, apiUpdates?: Array | null, bugFixes?: Array | null, breakingChanges?: Array | null, deprecations?: Array | null, features?: Array | null, majorChanges?: Array | null, removals?: Array | null } | null }; +export type ClusterOverviewDetailsQuery = { __typename?: 'RootQueryType', cluster?: { __typename?: 'Cluster', currentVersion?: string | null, id: string, self?: boolean | null, healthy?: boolean | null, healthScore?: number | null, protect?: boolean | null, name: string, handle?: string | null, distro?: ClusterDistro | null, cpuTotal?: number | null, memoryTotal?: number | null, cpuUtil?: number | null, nodeCount?: number | null, namespaceCount?: number | null, availabilityZones?: Array | null, podCount?: number | null, memoryUtil?: number | null, installed?: boolean | null, pingedAt?: string | null, deletedAt?: string | null, version?: string | null, kubeletVersion?: string | null, virtual?: boolean | null, disableAi?: boolean | null, currentUpgrade?: { __typename?: 'ClusterUpgrade', id: string, status: ClusterUpgradeStatus, version?: string | null, steps?: Array<{ __typename?: 'ClusterUpgradeStep', id: string, name: string, prompt: string, status: ClusterUpgradeStatus, type: ClusterUpgradeStepType, error?: string | null, agentRun?: { __typename?: 'AgentRun', id: string, status: AgentRunStatus, mode: AgentRunMode, babysit?: boolean | null, approval?: boolean | null, approvedAt?: string | null, prompt: string, shared?: boolean | null, error?: string | null, repository: string, branch?: string | null, headBranch?: string | null, insertedAt?: string | null, updatedAt?: string | null, runtime?: { __typename?: 'AgentRuntime', id: string, name: string, type: AgentRuntimeType } | null, pullRequests?: Array<{ __typename?: 'PullRequest', id: string, url: string, title?: string | null, creator?: string | null, status?: PrStatus | null, insertedAt?: string | null, updatedAt?: string | null } | null> | null, podReference?: { __typename?: 'AgentPodReference', name: string, namespace: string } | null, usage?: { __typename?: 'AgentRunUsage', totalCost?: number | null, totalTokens?: number | null } | null, workbenchJob?: { __typename?: 'WorkbenchJob', id: string, workbench?: { __typename?: 'Workbench', id: string, name: string } | null } | null, upload?: { __typename?: 'AgentRunUpload', id: string, session?: string | null, patch?: string | null } | null, todos?: Array<{ __typename?: 'AgentTodo', title: string, description: string, done?: boolean | null } | null> | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null, runtime?: { __typename?: 'AgentRuntime', type: AgentRuntimeType, name: string } | null } | null, nodeStatistics?: Array<{ __typename?: 'NodeStatistic', id: string, name: string, pendingPods?: number | null, health?: NodeStatisticHealth | null, insertedAt?: string | null, updatedAt?: string | null, cluster?: { __typename?: 'Cluster', id: string } | null } | null> | null, provider?: { __typename?: 'ClusterProvider', id: string, cloud: string, name: string, namespace: string, supportedVersions?: Array | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, repository?: { __typename?: 'GitRepository', url: string } | null } | null, tags?: Array<{ __typename?: 'Tag', name: string, value: string } | null> | null, upgradePlan?: { __typename?: 'ClusterUpgradePlan', compatibilities?: boolean | null, deprecations?: boolean | null, incompatibilities?: boolean | null, kubeletSkew?: boolean | null } | null, insight?: { __typename?: 'AiInsight', id: string, summary?: string | null, freshness?: InsightFreshness | null, insertedAt?: string | null, updatedAt?: string | null, evidence?: Array<{ __typename?: 'AiInsightEvidence', id: string, type: EvidenceType, insertedAt?: string | null, updatedAt?: string | null, logs?: { __typename?: 'LogsEvidence', clusterId?: string | null, serviceId?: string | null, line?: string | null, lines?: Array<{ __typename?: 'LogLine', log?: string | null, timestamp?: string | null, facets?: Array<{ __typename?: 'LogFacet', key: string, value?: string | null } | null> | null } | null> | null } | null, pullRequest?: { __typename?: 'PullRequestEvidence', contents?: string | null, filename?: string | null, patch?: string | null, repo?: string | null, sha?: string | null, title?: string | null, url?: string | null } | null, alert?: { __typename?: 'AlertEvidence', alertId?: string | null, title?: string | null, resolution?: string | null } | null, knowledge?: { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null } | null, clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, group?: string | null, version: string, kind: string, name: string, namespace?: string | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, serviceComponent?: { __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null } | null, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string, type: StackType } | null, stackRun?: { __typename?: 'StackRun', id: string, message?: string | null, type: StackType, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string } | null } | null, alert?: { __typename?: 'Alert', id: string, title?: string | null, message?: string | null } | null } | null, extendedSupport?: { __typename?: 'ExtendedSupportInfo', extended?: boolean | null, extendedFrom?: string | null } | null, deprecatedCustomResources?: Array<{ __typename?: 'DeprecatedCustomResource', name?: string | null, group: string, kind: string, namespace?: string | null, version: string, nextVersion: string } | null> | null, upgradePlanSummary?: { __typename?: 'UpgradePlanSummary', blockingAddons?: Array<{ __typename?: 'RuntimeAddonUpgrade', callout?: string | null, addon?: { __typename?: 'RuntimeAddon', name: string, icon?: string | null } | null, current?: { __typename?: 'AddonVersion', version?: string | null, chartVersion?: string | null, images?: Array | null, releaseUrl?: string | null } | null, fix?: { __typename?: 'AddonVersion', version?: string | null, chartVersion?: string | null, images?: Array | null, releaseUrl?: string | null, summary?: { __typename?: 'AddonVersionSummary', breakingChanges?: Array | null, chartUpdates?: Array | null, features?: Array | null, helmChanges?: string | null } | null } | null } | null> | null, blockingCloudAddons?: Array<{ __typename?: 'CloudAddonUpgrade', addon?: { __typename?: 'CloudAddon', id: string, insertedAt?: string | null, updatedAt?: string | null, name: string, distro: ClusterDistro, version: string, info?: { __typename?: 'CloudAddonInformation', name?: string | null, publisher?: string | null, versions?: Array<{ __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null> | null } | null, versionInfo?: { __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null } | null, current?: { __typename?: 'CloudAddonVersionInformation', version?: string | null } | null, fix?: { __typename?: 'CloudAddonVersionInformation', version?: string | null } | null } | null> | null } | null, insightComponents?: Array<{ __typename?: 'ClusterInsightComponent', id: string, kind: string, name: string, namespace?: string | null, group?: string | null, version: string, priority?: InsightComponentPriority | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string } | null } | null, insight?: { __typename?: 'AiInsight', id: string, text?: string | null, summary?: string | null, sha?: string | null, freshness?: InsightFreshness | null, updatedAt?: string | null, insertedAt?: string | null, error?: Array<{ __typename?: 'ServiceError', message: string, source: string } | null> | null, evidence?: Array<{ __typename?: 'AiInsightEvidence', id: string, type: EvidenceType, insertedAt?: string | null, updatedAt?: string | null, logs?: { __typename?: 'LogsEvidence', clusterId?: string | null, serviceId?: string | null, line?: string | null, lines?: Array<{ __typename?: 'LogLine', log?: string | null, timestamp?: string | null, facets?: Array<{ __typename?: 'LogFacet', key: string, value?: string | null } | null> | null } | null> | null } | null, pullRequest?: { __typename?: 'PullRequestEvidence', contents?: string | null, filename?: string | null, patch?: string | null, repo?: string | null, sha?: string | null, title?: string | null, url?: string | null } | null, alert?: { __typename?: 'AlertEvidence', alertId?: string | null, title?: string | null, resolution?: string | null } | null, knowledge?: { __typename?: 'KnowledgeEvidence', name?: string | null, observations?: Array | null, type?: string | null } | null } | null> | null, cluster?: { __typename?: 'Cluster', id: string, name: string, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', cloud: string } | null } | null, clusterInsightComponent?: { __typename?: 'ClusterInsightComponent', id: string, group?: string | null, version: string, kind: string, name: string, namespace?: string | null, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null, serviceComponent?: { __typename?: 'ServiceComponent', id: string, group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', id: string, name: string, cluster?: { __typename?: 'Cluster', id: string, name: string, handle?: string | null, distro?: ClusterDistro | null, provider?: { __typename?: 'ClusterProvider', name: string, cloud: string } | null } | null } | null } | null, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string, type: StackType } | null, stackRun?: { __typename?: 'StackRun', id: string, message?: string | null, type: StackType, stack?: { __typename?: 'InfrastructureStack', id?: string | null, name: string } | null } | null, alert?: { __typename?: 'Alert', id: string, title?: string | null, message?: string | null } | null } | null } | null> | null, runtimeServices?: Array<{ __typename?: 'RuntimeService', id: string, name: string, version: string, addon?: { __typename?: 'RuntimeAddon', icon?: string | null, versions?: Array<{ __typename?: 'AddonVersion', version?: string | null, kube?: Array | null, chartVersion?: string | null, incompatibilities?: Array<{ __typename?: 'VersionReference', version: string, name: string } | null> | null, requirements?: Array<{ __typename?: 'VersionReference', version: string, name: string } | null> | null } | null> | null } | null, service?: { __typename?: 'ServiceDeployment', git?: { __typename?: 'GitRef', ref: string, folder: string } | null, repository?: { __typename?: 'GitRepository', httpsPath?: string | null, urlFormat?: string | null } | null, helm?: { __typename?: 'HelmSpec', version?: string | null } | null } | null, addonVersion?: { __typename?: 'AddonVersion', blocking?: boolean | null, version?: string | null, kube?: Array | null, chartVersion?: string | null, incompatibilities?: Array<{ __typename?: 'VersionReference', version: string, name: string } | null> | null, requirements?: Array<{ __typename?: 'VersionReference', version: string, name: string } | null> | null } | null } | null> | null, apiDeprecations?: Array<{ __typename?: 'ApiDeprecation', availableIn?: string | null, blocking?: boolean | null, deprecatedIn?: string | null, removedIn?: string | null, replacement?: string | null, component?: { __typename?: 'ServiceComponent', group?: string | null, version?: string | null, kind: string, name: string, namespace?: string | null, service?: { __typename?: 'ServiceDeployment', git?: { __typename?: 'GitRef', ref: string, folder: string } | null, repository?: { __typename?: 'GitRepository', httpsPath?: string | null, urlFormat?: string | null } | null } | null } | null } | null> | null, upgradeInsights?: Array<{ __typename?: 'UpgradeInsight', id: string, name: string, description?: string | null, refreshedAt?: string | null, transitionedAt?: string | null, version?: string | null, status?: UpgradeInsightStatus | null, details?: Array<{ __typename?: 'UpgradeInsightDetail', id: string, removedIn?: string | null, replacedIn?: string | null, replacement?: string | null, status?: UpgradeInsightStatus | null, used?: string | null, clientInfo?: Array<{ __typename?: 'InsightClientInfo', userAgent?: string | null, count?: string | null, lastRequestAt?: string | null } | null> | null } | null> | null } | null> | null, cloudAddons?: Array<{ __typename?: 'CloudAddon', id: string, insertedAt?: string | null, updatedAt?: string | null, name: string, distro: ClusterDistro, version: string, info?: { __typename?: 'CloudAddonInformation', name?: string | null, publisher?: string | null, versions?: Array<{ __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null> | null } | null, versionInfo?: { __typename?: 'CloudAddonVersionInformation', version?: string | null, compatibilities?: Array | null, blocking?: boolean | null } | null } | null> | null } | null, kubernetesChangelog?: { __typename?: 'KubernetesChangelog', version?: string | null, apiUpdates?: Array | null, bugFixes?: Array | null, breakingChanges?: Array | null, deprecations?: Array | null, features?: Array | null, majorChanges?: Array | null, removals?: Array | null } | null }; export type UpgradeStatisticsQueryVariables = Exact<{ projectId?: InputMaybe; @@ -25179,6 +25190,13 @@ export const ClusterInsightComponentFragmentDoc = gql` group version priority + service { + id + name + cluster { + id + } + } insight { ...AiInsight } @@ -32625,6 +32643,57 @@ export function useFixResearchDiagramMutation(baseOptions?: Apollo.MutationHookO export type FixResearchDiagramMutationHookResult = ReturnType; export type FixResearchDiagramMutationResult = Apollo.MutationResult; export type FixResearchDiagramMutationOptions = Apollo.BaseMutationOptions; +export const ServiceComponentIdentitiesDocument = gql` + query ServiceComponentIdentities($id: ID!) { + serviceDeployment(id: $id) { + id + components { + id + group + version + kind + namespace + name + } + } +} + `; + +/** + * __useServiceComponentIdentitiesQuery__ + * + * To run a query within a React component, call `useServiceComponentIdentitiesQuery` and pass it any options that fit your needs. + * When your component renders, `useServiceComponentIdentitiesQuery` returns an object from Apollo Client that contains loading, error, and data properties + * you can use to render your UI. + * + * @param baseOptions options that will be passed into the query, supported options are listed on: https://www.apollographql.com/docs/react/api/react-hooks/#options; + * + * @example + * const { data, loading, error } = useServiceComponentIdentitiesQuery({ + * variables: { + * id: // value for 'id' + * }, + * }); + */ +export function useServiceComponentIdentitiesQuery(baseOptions: Apollo.QueryHookOptions & ({ variables: ServiceComponentIdentitiesQueryVariables; skip?: boolean; } | { skip: boolean; }) ) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useQuery(ServiceComponentIdentitiesDocument, options); + } +export function useServiceComponentIdentitiesLazyQuery(baseOptions?: Apollo.LazyQueryHookOptions) { + const options = {...defaultOptions, ...baseOptions} + return Apollo.useLazyQuery(ServiceComponentIdentitiesDocument, options); + } +// @ts-ignore +export function useServiceComponentIdentitiesSuspenseQuery(baseOptions?: Apollo.SuspenseQueryHookOptions): Apollo.UseSuspenseQueryResult; +export function useServiceComponentIdentitiesSuspenseQuery(baseOptions?: Apollo.SkipToken | Apollo.SuspenseQueryHookOptions): Apollo.UseSuspenseQueryResult; +export function useServiceComponentIdentitiesSuspenseQuery(baseOptions?: Apollo.SkipToken | Apollo.SuspenseQueryHookOptions) { + const options = baseOptions === Apollo.skipToken ? baseOptions : {...defaultOptions, ...baseOptions} + return Apollo.useSuspenseQuery(ServiceComponentIdentitiesDocument, options); + } +export type ServiceComponentIdentitiesQueryHookResult = ReturnType; +export type ServiceComponentIdentitiesLazyQueryHookResult = ReturnType; +export type ServiceComponentIdentitiesSuspenseQueryHookResult = ReturnType; +export type ServiceComponentIdentitiesQueryResult = Apollo.QueryResult; export const AiInsightDocument = gql` query AiInsight($id: ID!) { aiInsight(id: $id) { @@ -52649,6 +52718,7 @@ export const namedOperations = { CloudConnection: 'CloudConnection', InfraResearch: 'InfraResearch', InfraResearches: 'InfraResearches', + ServiceComponentIdentities: 'ServiceComponentIdentities', AiInsight: 'AiInsight', AICompletion: 'AICompletion', AISuggestedFix: 'AISuggestedFix', diff --git a/js/console/src/generated/persisted-queries/client.json b/js/console/src/generated/persisted-queries/client.json index 7ca490a916..b28616d521 100644 --- a/js/console/src/generated/persisted-queries/client.json +++ b/js/console/src/generated/persisted-queries/client.json @@ -222,6 +222,11 @@ "name": "FixResearchDiagram", "body": "mutation FixResearchDiagram($id: ID!, $error: String!) {\n fixResearchDiagram(id: $id, error: $error) {\n ...InfraResearch\n __typename\n }\n}\n\nfragment InfraResearch on InfraResearch {\n id\n prompt\n status\n diagram\n published\n analysis {\n ...InfraResearchAnalysis\n __typename\n }\n threads {\n ...ChatThreadTiny\n __typename\n }\n associations {\n ...InfraResearchAssociation\n __typename\n }\n user {\n id\n name\n __typename\n }\n insertedAt\n updatedAt\n __typename\n}\n\nfragment InfraResearchAnalysis on InfraResearchAnalysis {\n notes\n summary\n __typename\n}\n\nfragment ChatThreadTiny on ChatThread {\n id\n default\n summary\n insertedAt\n updatedAt\n lastMessageAt\n settings {\n ...ChatThreadSettings\n __typename\n }\n insight {\n ...AiInsightSummary\n __typename\n }\n flow {\n id\n name\n icon\n __typename\n }\n session {\n ...AgentSession\n __typename\n }\n research {\n id\n prompt\n __typename\n }\n service {\n id\n name\n __typename\n }\n __typename\n}\n\nfragment ChatThreadSettings on ChatThreadSettings {\n memory\n __typename\n}\n\nfragment AiInsightSummary on AiInsight {\n id\n summary\n freshness\n insertedAt\n updatedAt\n ...AiInsightContext\n __typename\n}\n\nfragment AiInsightContext on AiInsight {\n evidence {\n ...AiInsightEvidence\n __typename\n }\n cluster {\n id\n name\n distro\n provider {\n cloud\n __typename\n }\n __typename\n }\n clusterInsightComponent {\n id\n group\n version\n kind\n name\n namespace\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n service {\n id\n name\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n serviceComponent {\n id\n group\n version\n kind\n name\n namespace\n service {\n id\n name\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n __typename\n }\n stack {\n id\n name\n type\n __typename\n }\n stackRun {\n id\n message\n type\n stack {\n id\n name\n __typename\n }\n __typename\n }\n alert {\n id\n title\n message\n __typename\n }\n __typename\n}\n\nfragment AiInsightEvidence on AiInsightEvidence {\n id\n type\n logs {\n ...LogsEvidence\n __typename\n }\n pullRequest {\n ...PullRequestEvidence\n __typename\n }\n alert {\n ...AlertEvidence\n __typename\n }\n knowledge {\n ...KnowledgeEvidence\n __typename\n }\n insertedAt\n updatedAt\n __typename\n}\n\nfragment LogsEvidence on LogsEvidence {\n clusterId\n serviceId\n line\n lines {\n ...LogLine\n __typename\n }\n __typename\n}\n\nfragment LogLine on LogLine {\n facets {\n ...LogFacet\n __typename\n }\n log\n timestamp\n __typename\n}\n\nfragment LogFacet on LogFacet {\n key\n value\n __typename\n}\n\nfragment PullRequestEvidence on PullRequestEvidence {\n contents\n filename\n patch\n repo\n sha\n title\n url\n __typename\n}\n\nfragment AlertEvidence on AlertEvidence {\n alertId\n title\n resolution\n __typename\n}\n\nfragment KnowledgeEvidence on KnowledgeEvidence {\n name\n observations\n type\n __typename\n}\n\nfragment ClusterMinimal on Cluster {\n id\n name\n handle\n provider {\n name\n cloud\n __typename\n }\n distro\n __typename\n}\n\nfragment AgentSession on AgentSession {\n id\n type\n done\n connection {\n id\n name\n provider\n __typename\n }\n cluster {\n id\n __typename\n }\n runtime {\n id\n name\n __typename\n }\n pullRequest {\n id\n url\n __typename\n }\n __typename\n}\n\nfragment InfraResearchAssociation on InfraResearchAssociation {\n id\n stack {\n ...StackMinimal\n __typename\n }\n service {\n ...ServiceDeploymentTiny\n __typename\n }\n __typename\n}\n\nfragment StackMinimal on InfrastructureStack {\n id\n name\n type\n status\n insertedAt\n updatedAt\n deletedAt\n __typename\n}\n\nfragment ServiceDeploymentTiny on ServiceDeployment {\n id\n name\n cluster {\n ...ClusterMinimal\n __typename\n }\n componentStatus\n status\n errors {\n message\n source\n __typename\n }\n __typename\n}" }, + "sha256:b0bc7ffe49d473e855662da9bb902e6962855783b79e2b11a82fe76a71d12d56": { + "type": "query", + "name": "ServiceComponentIdentities", + "body": "query ServiceComponentIdentities($id: ID!) {\n serviceDeployment(id: $id) {\n id\n components {\n id\n group\n version\n kind\n namespace\n name\n __typename\n }\n __typename\n }\n}" + }, "sha256:06b312e69898f6e6253f452effb8e9301fcc8ccb324c12495a86060021c14276": { "type": "query", "name": "AiInsight", @@ -237,10 +242,10 @@ "name": "AISuggestedFix", "body": "query AISuggestedFix($insightID: ID!) {\n aiSuggestedFix(insightId: $insightID)\n}" }, - "sha256:ccb7b3dfa92dda7203535da60b53764b18d204b26c3d098216bb44e29925e518": { + "sha256:f979adfe319e825b6afdd4274760655899371bef9c2b3553822cc603037e343f": { "type": "query", "name": "ClusterInsightComponent", - "body": "query ClusterInsightComponent($id: ID!) {\n clusterInsightComponent(id: $id) {\n ...ClusterInsightComponentDetail\n __typename\n }\n}\n\nfragment ClusterInsightComponentDetail on ClusterInsightComponent {\n ...ClusterInsightComponent\n resource {\n raw\n __typename\n }\n __typename\n}\n\nfragment ClusterInsightComponent on ClusterInsightComponent {\n id\n kind\n name\n namespace\n group\n version\n priority\n insight {\n ...AiInsight\n __typename\n }\n __typename\n}\n\nfragment AiInsight on AiInsight {\n id\n text\n summary\n sha\n freshness\n updatedAt\n insertedAt\n error {\n message\n source\n __typename\n }\n ...AiInsightContext\n __typename\n}\n\nfragment AiInsightContext on AiInsight {\n evidence {\n ...AiInsightEvidence\n __typename\n }\n cluster {\n id\n name\n distro\n provider {\n cloud\n __typename\n }\n __typename\n }\n clusterInsightComponent {\n id\n group\n version\n kind\n name\n namespace\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n service {\n id\n name\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n serviceComponent {\n id\n group\n version\n kind\n name\n namespace\n service {\n id\n name\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n __typename\n }\n stack {\n id\n name\n type\n __typename\n }\n stackRun {\n id\n message\n type\n stack {\n id\n name\n __typename\n }\n __typename\n }\n alert {\n id\n title\n message\n __typename\n }\n __typename\n}\n\nfragment AiInsightEvidence on AiInsightEvidence {\n id\n type\n logs {\n ...LogsEvidence\n __typename\n }\n pullRequest {\n ...PullRequestEvidence\n __typename\n }\n alert {\n ...AlertEvidence\n __typename\n }\n knowledge {\n ...KnowledgeEvidence\n __typename\n }\n insertedAt\n updatedAt\n __typename\n}\n\nfragment LogsEvidence on LogsEvidence {\n clusterId\n serviceId\n line\n lines {\n ...LogLine\n __typename\n }\n __typename\n}\n\nfragment LogLine on LogLine {\n facets {\n ...LogFacet\n __typename\n }\n log\n timestamp\n __typename\n}\n\nfragment LogFacet on LogFacet {\n key\n value\n __typename\n}\n\nfragment PullRequestEvidence on PullRequestEvidence {\n contents\n filename\n patch\n repo\n sha\n title\n url\n __typename\n}\n\nfragment AlertEvidence on AlertEvidence {\n alertId\n title\n resolution\n __typename\n}\n\nfragment KnowledgeEvidence on KnowledgeEvidence {\n name\n observations\n type\n __typename\n}\n\nfragment ClusterMinimal on Cluster {\n id\n name\n handle\n provider {\n name\n cloud\n __typename\n }\n distro\n __typename\n}" + "body": "query ClusterInsightComponent($id: ID!) {\n clusterInsightComponent(id: $id) {\n ...ClusterInsightComponentDetail\n __typename\n }\n}\n\nfragment ClusterInsightComponentDetail on ClusterInsightComponent {\n ...ClusterInsightComponent\n resource {\n raw\n __typename\n }\n __typename\n}\n\nfragment ClusterInsightComponent on ClusterInsightComponent {\n id\n kind\n name\n namespace\n group\n version\n priority\n service {\n id\n name\n cluster {\n id\n __typename\n }\n __typename\n }\n insight {\n ...AiInsight\n __typename\n }\n __typename\n}\n\nfragment AiInsight on AiInsight {\n id\n text\n summary\n sha\n freshness\n updatedAt\n insertedAt\n error {\n message\n source\n __typename\n }\n ...AiInsightContext\n __typename\n}\n\nfragment AiInsightContext on AiInsight {\n evidence {\n ...AiInsightEvidence\n __typename\n }\n cluster {\n id\n name\n distro\n provider {\n cloud\n __typename\n }\n __typename\n }\n clusterInsightComponent {\n id\n group\n version\n kind\n name\n namespace\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n service {\n id\n name\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n serviceComponent {\n id\n group\n version\n kind\n name\n namespace\n service {\n id\n name\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n __typename\n }\n stack {\n id\n name\n type\n __typename\n }\n stackRun {\n id\n message\n type\n stack {\n id\n name\n __typename\n }\n __typename\n }\n alert {\n id\n title\n message\n __typename\n }\n __typename\n}\n\nfragment AiInsightEvidence on AiInsightEvidence {\n id\n type\n logs {\n ...LogsEvidence\n __typename\n }\n pullRequest {\n ...PullRequestEvidence\n __typename\n }\n alert {\n ...AlertEvidence\n __typename\n }\n knowledge {\n ...KnowledgeEvidence\n __typename\n }\n insertedAt\n updatedAt\n __typename\n}\n\nfragment LogsEvidence on LogsEvidence {\n clusterId\n serviceId\n line\n lines {\n ...LogLine\n __typename\n }\n __typename\n}\n\nfragment LogLine on LogLine {\n facets {\n ...LogFacet\n __typename\n }\n log\n timestamp\n __typename\n}\n\nfragment LogFacet on LogFacet {\n key\n value\n __typename\n}\n\nfragment PullRequestEvidence on PullRequestEvidence {\n contents\n filename\n patch\n repo\n sha\n title\n url\n __typename\n}\n\nfragment AlertEvidence on AlertEvidence {\n alertId\n title\n resolution\n __typename\n}\n\nfragment KnowledgeEvidence on KnowledgeEvidence {\n name\n observations\n type\n __typename\n}\n\nfragment ClusterMinimal on Cluster {\n id\n name\n handle\n provider {\n name\n cloud\n __typename\n }\n distro\n __typename\n}" }, "sha256:3c1871cecbb1d714f8e403a40acab706e72e791f5136151a41dd11dfbbb6fd9c": { "type": "mutation", @@ -567,10 +572,10 @@ "name": "ClusterBasic", "body": "query ClusterBasic($id: ID!) {\n cluster(id: $id) {\n ...ClusterBasic\n __typename\n }\n}\n\nfragment ClusterBasic on Cluster {\n ...ClusterTiny\n protect\n deletedAt\n version\n currentVersion\n __typename\n}\n\nfragment ClusterTiny on Cluster {\n ...ClusterMinimal\n self\n upgradePlan {\n compatibilities\n deprecations\n incompatibilities\n __typename\n }\n virtual\n __typename\n}\n\nfragment ClusterMinimal on Cluster {\n id\n name\n handle\n provider {\n name\n cloud\n __typename\n }\n distro\n __typename\n}" }, - "sha256:309b837fb6516698f3899becea8a56c52bae867c2c064834c210e59210275aa2": { + "sha256:da4cf1857da3e5e113bbcf315b63a62b1d562b84d671dd03b5650998536db4fc": { "type": "query", "name": "ClusterInsight", - "body": "query ClusterInsight($id: ID!) {\n cluster(id: $id) {\n ...ClusterInsight\n __typename\n }\n}\n\nfragment ClusterInsight on Cluster {\n id\n insightComponents {\n ...ClusterInsightComponent\n __typename\n }\n __typename\n}\n\nfragment ClusterInsightComponent on ClusterInsightComponent {\n id\n kind\n name\n namespace\n group\n version\n priority\n insight {\n ...AiInsight\n __typename\n }\n __typename\n}\n\nfragment AiInsight on AiInsight {\n id\n text\n summary\n sha\n freshness\n updatedAt\n insertedAt\n error {\n message\n source\n __typename\n }\n ...AiInsightContext\n __typename\n}\n\nfragment AiInsightContext on AiInsight {\n evidence {\n ...AiInsightEvidence\n __typename\n }\n cluster {\n id\n name\n distro\n provider {\n cloud\n __typename\n }\n __typename\n }\n clusterInsightComponent {\n id\n group\n version\n kind\n name\n namespace\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n service {\n id\n name\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n serviceComponent {\n id\n group\n version\n kind\n name\n namespace\n service {\n id\n name\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n __typename\n }\n stack {\n id\n name\n type\n __typename\n }\n stackRun {\n id\n message\n type\n stack {\n id\n name\n __typename\n }\n __typename\n }\n alert {\n id\n title\n message\n __typename\n }\n __typename\n}\n\nfragment AiInsightEvidence on AiInsightEvidence {\n id\n type\n logs {\n ...LogsEvidence\n __typename\n }\n pullRequest {\n ...PullRequestEvidence\n __typename\n }\n alert {\n ...AlertEvidence\n __typename\n }\n knowledge {\n ...KnowledgeEvidence\n __typename\n }\n insertedAt\n updatedAt\n __typename\n}\n\nfragment LogsEvidence on LogsEvidence {\n clusterId\n serviceId\n line\n lines {\n ...LogLine\n __typename\n }\n __typename\n}\n\nfragment LogLine on LogLine {\n facets {\n ...LogFacet\n __typename\n }\n log\n timestamp\n __typename\n}\n\nfragment LogFacet on LogFacet {\n key\n value\n __typename\n}\n\nfragment PullRequestEvidence on PullRequestEvidence {\n contents\n filename\n patch\n repo\n sha\n title\n url\n __typename\n}\n\nfragment AlertEvidence on AlertEvidence {\n alertId\n title\n resolution\n __typename\n}\n\nfragment KnowledgeEvidence on KnowledgeEvidence {\n name\n observations\n type\n __typename\n}\n\nfragment ClusterMinimal on Cluster {\n id\n name\n handle\n provider {\n name\n cloud\n __typename\n }\n distro\n __typename\n}" + "body": "query ClusterInsight($id: ID!) {\n cluster(id: $id) {\n ...ClusterInsight\n __typename\n }\n}\n\nfragment ClusterInsight on Cluster {\n id\n insightComponents {\n ...ClusterInsightComponent\n __typename\n }\n __typename\n}\n\nfragment ClusterInsightComponent on ClusterInsightComponent {\n id\n kind\n name\n namespace\n group\n version\n priority\n service {\n id\n name\n cluster {\n id\n __typename\n }\n __typename\n }\n insight {\n ...AiInsight\n __typename\n }\n __typename\n}\n\nfragment AiInsight on AiInsight {\n id\n text\n summary\n sha\n freshness\n updatedAt\n insertedAt\n error {\n message\n source\n __typename\n }\n ...AiInsightContext\n __typename\n}\n\nfragment AiInsightContext on AiInsight {\n evidence {\n ...AiInsightEvidence\n __typename\n }\n cluster {\n id\n name\n distro\n provider {\n cloud\n __typename\n }\n __typename\n }\n clusterInsightComponent {\n id\n group\n version\n kind\n name\n namespace\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n service {\n id\n name\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n serviceComponent {\n id\n group\n version\n kind\n name\n namespace\n service {\n id\n name\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n __typename\n }\n stack {\n id\n name\n type\n __typename\n }\n stackRun {\n id\n message\n type\n stack {\n id\n name\n __typename\n }\n __typename\n }\n alert {\n id\n title\n message\n __typename\n }\n __typename\n}\n\nfragment AiInsightEvidence on AiInsightEvidence {\n id\n type\n logs {\n ...LogsEvidence\n __typename\n }\n pullRequest {\n ...PullRequestEvidence\n __typename\n }\n alert {\n ...AlertEvidence\n __typename\n }\n knowledge {\n ...KnowledgeEvidence\n __typename\n }\n insertedAt\n updatedAt\n __typename\n}\n\nfragment LogsEvidence on LogsEvidence {\n clusterId\n serviceId\n line\n lines {\n ...LogLine\n __typename\n }\n __typename\n}\n\nfragment LogLine on LogLine {\n facets {\n ...LogFacet\n __typename\n }\n log\n timestamp\n __typename\n}\n\nfragment LogFacet on LogFacet {\n key\n value\n __typename\n}\n\nfragment PullRequestEvidence on PullRequestEvidence {\n contents\n filename\n patch\n repo\n sha\n title\n url\n __typename\n}\n\nfragment AlertEvidence on AlertEvidence {\n alertId\n title\n resolution\n __typename\n}\n\nfragment KnowledgeEvidence on KnowledgeEvidence {\n name\n observations\n type\n __typename\n}\n\nfragment ClusterMinimal on Cluster {\n id\n name\n handle\n provider {\n name\n cloud\n __typename\n }\n distro\n __typename\n}" }, "sha256:9febd582e186643926517f42f504798f4bc1e59e91427e70f4f2425ca0b212b8": { "type": "query", @@ -1217,10 +1222,10 @@ "name": "DeleteGroup", "body": "mutation DeleteGroup($id: ID!) {\n deleteGroup(groupId: $id) {\n ...Group\n __typename\n }\n}\n\nfragment Group on Group {\n id\n name\n description\n global\n memberCount\n insertedAt\n updatedAt\n __typename\n}" }, - "sha256:a2a8c9641fff469e17c5a216c9882da4a92e7c32d830cd55eee1822f82ee2145": { + "sha256:112893d2df99852fea88f58a2bddf576c1fd5a8e5385f97075cb6e1c8604c20a": { "type": "query", "name": "ClusterOverviewDetails", - "body": "query ClusterOverviewDetails($id: ID!, $kubeVersion: String!, $nextKubeVersion: String!, $hasKubeVersion: Boolean!) {\n cluster(id: $id) {\n ...ClusterOverviewDetails\n __typename\n }\n kubernetesChangelog(version: $nextKubeVersion) @include(if: $hasKubeVersion) {\n ...KubernetesChangelog\n __typename\n }\n}\n\nfragment ClusterOverviewDetails on Cluster {\n ...ClustersRow\n ...ClusterWithUpgrade\n ...ClusterInsight\n currentUpgrade {\n ...ClusterUpgrade\n __typename\n }\n nodeStatistics {\n ...NodeStatistic\n __typename\n }\n __typename\n}\n\nfragment ClustersRow on Cluster {\n currentVersion\n id\n self\n healthy\n healthScore\n protect\n name\n handle\n distro\n cpuTotal\n memoryTotal\n cpuUtil\n nodeCount\n namespaceCount\n availabilityZones\n podCount\n memoryUtil\n installed\n pingedAt\n deletedAt\n provider {\n id\n cloud\n name\n namespace\n supportedVersions\n __typename\n }\n service {\n id\n repository {\n url\n __typename\n }\n __typename\n }\n version\n kubeletVersion\n tags {\n name\n value\n __typename\n }\n upgradePlan {\n ...ClusterUpgradePlan\n __typename\n }\n virtual\n insight {\n ...AiInsightSummary\n __typename\n }\n extendedSupport {\n extended\n extendedFrom\n __typename\n }\n disableAi\n __typename\n}\n\nfragment ClusterUpgradePlan on ClusterUpgradePlan {\n compatibilities\n deprecations\n incompatibilities\n kubeletSkew\n __typename\n}\n\nfragment AiInsightSummary on AiInsight {\n id\n summary\n freshness\n insertedAt\n updatedAt\n ...AiInsightContext\n __typename\n}\n\nfragment AiInsightContext on AiInsight {\n evidence {\n ...AiInsightEvidence\n __typename\n }\n cluster {\n id\n name\n distro\n provider {\n cloud\n __typename\n }\n __typename\n }\n clusterInsightComponent {\n id\n group\n version\n kind\n name\n namespace\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n service {\n id\n name\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n serviceComponent {\n id\n group\n version\n kind\n name\n namespace\n service {\n id\n name\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n __typename\n }\n stack {\n id\n name\n type\n __typename\n }\n stackRun {\n id\n message\n type\n stack {\n id\n name\n __typename\n }\n __typename\n }\n alert {\n id\n title\n message\n __typename\n }\n __typename\n}\n\nfragment AiInsightEvidence on AiInsightEvidence {\n id\n type\n logs {\n ...LogsEvidence\n __typename\n }\n pullRequest {\n ...PullRequestEvidence\n __typename\n }\n alert {\n ...AlertEvidence\n __typename\n }\n knowledge {\n ...KnowledgeEvidence\n __typename\n }\n insertedAt\n updatedAt\n __typename\n}\n\nfragment LogsEvidence on LogsEvidence {\n clusterId\n serviceId\n line\n lines {\n ...LogLine\n __typename\n }\n __typename\n}\n\nfragment LogLine on LogLine {\n facets {\n ...LogFacet\n __typename\n }\n log\n timestamp\n __typename\n}\n\nfragment LogFacet on LogFacet {\n key\n value\n __typename\n}\n\nfragment PullRequestEvidence on PullRequestEvidence {\n contents\n filename\n patch\n repo\n sha\n title\n url\n __typename\n}\n\nfragment AlertEvidence on AlertEvidence {\n alertId\n title\n resolution\n __typename\n}\n\nfragment KnowledgeEvidence on KnowledgeEvidence {\n name\n observations\n type\n __typename\n}\n\nfragment ClusterMinimal on Cluster {\n id\n name\n handle\n provider {\n name\n cloud\n __typename\n }\n distro\n __typename\n}\n\nfragment ClusterWithUpgrade on Cluster {\n ...ClusterRuntimeServices\n deprecatedCustomResources {\n ...ClusterUpgradeDeprecatedCustomResource\n __typename\n }\n upgradePlanSummary {\n blockingAddons {\n ...RuntimeAddonUpgrade\n __typename\n }\n blockingCloudAddons {\n ...CloudAddonUpgrade\n __typename\n }\n __typename\n }\n __typename\n}\n\nfragment ClusterRuntimeServices on Cluster {\n id\n name\n currentVersion\n version\n runtimeServices {\n ...RuntimeService\n __typename\n }\n apiDeprecations {\n ...ApiDeprecation\n __typename\n }\n upgradeInsights {\n ...UpgradeInsight\n __typename\n }\n cloudAddons {\n ...CloudAddon\n __typename\n }\n __typename\n}\n\nfragment RuntimeService on RuntimeService {\n id\n name\n version\n addon {\n icon\n versions {\n ...AddonVersion\n __typename\n }\n __typename\n }\n service {\n git {\n ref\n folder\n __typename\n }\n repository {\n httpsPath\n urlFormat\n __typename\n }\n helm {\n version\n __typename\n }\n __typename\n }\n addonVersion {\n ...AddonVersionBlocking @include(if: $hasKubeVersion)\n ...AddonVersion\n __typename\n }\n __typename\n}\n\nfragment AddonVersion on AddonVersion {\n version\n kube\n chartVersion\n incompatibilities {\n version\n name\n __typename\n }\n requirements {\n version\n name\n __typename\n }\n __typename\n}\n\nfragment AddonVersionBlocking on AddonVersion {\n blocking(kubeVersion: $kubeVersion)\n __typename\n}\n\nfragment ApiDeprecation on ApiDeprecation {\n availableIn\n blocking\n component {\n group\n version\n kind\n name\n namespace\n service {\n git {\n ref\n folder\n __typename\n }\n repository {\n httpsPath\n urlFormat\n __typename\n }\n __typename\n }\n __typename\n }\n deprecatedIn\n removedIn\n replacement\n __typename\n}\n\nfragment UpgradeInsight on UpgradeInsight {\n id\n name\n description\n details {\n ...UpgradeInsightDetail\n __typename\n }\n refreshedAt\n transitionedAt\n version\n status\n __typename\n}\n\nfragment UpgradeInsightDetail on UpgradeInsightDetail {\n id\n removedIn\n replacedIn\n replacement\n status\n used\n clientInfo {\n ...InsightClientInfo\n __typename\n }\n __typename\n}\n\nfragment InsightClientInfo on InsightClientInfo {\n userAgent\n count\n lastRequestAt\n __typename\n}\n\nfragment CloudAddon on CloudAddon {\n id\n insertedAt\n updatedAt\n name\n distro\n info {\n name\n publisher\n versions {\n version\n compatibilities\n ...CloudAddonVersionInformation @include(if: $hasKubeVersion)\n __typename\n }\n __typename\n }\n version\n versionInfo {\n version\n compatibilities\n ...CloudAddonVersionInformation @include(if: $hasKubeVersion)\n __typename\n }\n __typename\n}\n\nfragment CloudAddonVersionInformation on CloudAddonVersionInformation {\n blocking(kubeVersion: $kubeVersion)\n __typename\n}\n\nfragment ClusterUpgradeDeprecatedCustomResource on DeprecatedCustomResource {\n name\n group\n kind\n namespace\n version\n nextVersion\n __typename\n}\n\nfragment RuntimeAddonUpgrade on RuntimeAddonUpgrade {\n addon {\n name\n icon\n __typename\n }\n callout\n current {\n ...RuntimeAddonVersion\n __typename\n }\n fix {\n ...RuntimeAddonVersionWithSummary\n __typename\n }\n __typename\n}\n\nfragment RuntimeAddonVersion on AddonVersion {\n version\n chartVersion\n images\n releaseUrl\n __typename\n}\n\nfragment RuntimeAddonVersionWithSummary on AddonVersion {\n ...RuntimeAddonVersion\n summary {\n ...AddonVersionSummary\n __typename\n }\n __typename\n}\n\nfragment AddonVersionSummary on AddonVersionSummary {\n breakingChanges\n chartUpdates\n features\n helmChanges\n __typename\n}\n\nfragment CloudAddonUpgrade on CloudAddonUpgrade {\n addon {\n ...CloudAddon\n __typename\n }\n current {\n ...CloudAddonVersion\n __typename\n }\n fix {\n ...CloudAddonVersion\n __typename\n }\n __typename\n}\n\nfragment CloudAddonVersion on CloudAddonVersionInformation {\n version\n __typename\n}\n\nfragment ClusterInsight on Cluster {\n id\n insightComponents {\n ...ClusterInsightComponent\n __typename\n }\n __typename\n}\n\nfragment ClusterInsightComponent on ClusterInsightComponent {\n id\n kind\n name\n namespace\n group\n version\n priority\n insight {\n ...AiInsight\n __typename\n }\n __typename\n}\n\nfragment AiInsight on AiInsight {\n id\n text\n summary\n sha\n freshness\n updatedAt\n insertedAt\n error {\n message\n source\n __typename\n }\n ...AiInsightContext\n __typename\n}\n\nfragment ClusterUpgrade on ClusterUpgrade {\n id\n status\n steps {\n ...ClusterUpgradeStep\n __typename\n }\n cluster {\n ...ClusterMinimal\n __typename\n }\n runtime {\n type\n name\n __typename\n }\n version\n __typename\n}\n\nfragment ClusterUpgradeStep on ClusterUpgradeStep {\n id\n name\n prompt\n status\n type\n error\n agentRun {\n ...AgentRunTiny\n __typename\n }\n __typename\n}\n\nfragment AgentRunTiny on AgentRun {\n id\n status\n mode\n babysit\n approval\n approvedAt\n prompt\n shared\n error\n runtime {\n id\n name\n type\n __typename\n }\n repository\n branch\n headBranch\n pullRequests {\n ...PullRequestBasic\n __typename\n }\n podReference {\n name\n namespace\n __typename\n }\n usage {\n totalCost\n totalTokens\n __typename\n }\n workbenchJob {\n id\n workbench {\n id\n name\n __typename\n }\n __typename\n }\n upload {\n id\n session\n patch\n __typename\n }\n todos {\n ...AgentTodo\n __typename\n }\n insertedAt\n updatedAt\n __typename\n}\n\nfragment PullRequestBasic on PullRequest {\n id\n url\n title\n creator\n status\n insertedAt\n updatedAt\n __typename\n}\n\nfragment AgentTodo on AgentTodo {\n title\n description\n done\n __typename\n}\n\nfragment NodeStatistic on NodeStatistic {\n id\n name\n pendingPods\n health\n cluster {\n id\n __typename\n }\n insertedAt\n updatedAt\n __typename\n}\n\nfragment KubernetesChangelog on KubernetesChangelog {\n version\n apiUpdates\n bugFixes\n breakingChanges\n deprecations\n features\n majorChanges\n removals\n __typename\n}" + "body": "query ClusterOverviewDetails($id: ID!, $kubeVersion: String!, $nextKubeVersion: String!, $hasKubeVersion: Boolean!) {\n cluster(id: $id) {\n ...ClusterOverviewDetails\n __typename\n }\n kubernetesChangelog(version: $nextKubeVersion) @include(if: $hasKubeVersion) {\n ...KubernetesChangelog\n __typename\n }\n}\n\nfragment ClusterOverviewDetails on Cluster {\n ...ClustersRow\n ...ClusterWithUpgrade\n ...ClusterInsight\n currentUpgrade {\n ...ClusterUpgrade\n __typename\n }\n nodeStatistics {\n ...NodeStatistic\n __typename\n }\n __typename\n}\n\nfragment ClustersRow on Cluster {\n currentVersion\n id\n self\n healthy\n healthScore\n protect\n name\n handle\n distro\n cpuTotal\n memoryTotal\n cpuUtil\n nodeCount\n namespaceCount\n availabilityZones\n podCount\n memoryUtil\n installed\n pingedAt\n deletedAt\n provider {\n id\n cloud\n name\n namespace\n supportedVersions\n __typename\n }\n service {\n id\n repository {\n url\n __typename\n }\n __typename\n }\n version\n kubeletVersion\n tags {\n name\n value\n __typename\n }\n upgradePlan {\n ...ClusterUpgradePlan\n __typename\n }\n virtual\n insight {\n ...AiInsightSummary\n __typename\n }\n extendedSupport {\n extended\n extendedFrom\n __typename\n }\n disableAi\n __typename\n}\n\nfragment ClusterUpgradePlan on ClusterUpgradePlan {\n compatibilities\n deprecations\n incompatibilities\n kubeletSkew\n __typename\n}\n\nfragment AiInsightSummary on AiInsight {\n id\n summary\n freshness\n insertedAt\n updatedAt\n ...AiInsightContext\n __typename\n}\n\nfragment AiInsightContext on AiInsight {\n evidence {\n ...AiInsightEvidence\n __typename\n }\n cluster {\n id\n name\n distro\n provider {\n cloud\n __typename\n }\n __typename\n }\n clusterInsightComponent {\n id\n group\n version\n kind\n name\n namespace\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n service {\n id\n name\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n serviceComponent {\n id\n group\n version\n kind\n name\n namespace\n service {\n id\n name\n cluster {\n ...ClusterMinimal\n __typename\n }\n __typename\n }\n __typename\n }\n stack {\n id\n name\n type\n __typename\n }\n stackRun {\n id\n message\n type\n stack {\n id\n name\n __typename\n }\n __typename\n }\n alert {\n id\n title\n message\n __typename\n }\n __typename\n}\n\nfragment AiInsightEvidence on AiInsightEvidence {\n id\n type\n logs {\n ...LogsEvidence\n __typename\n }\n pullRequest {\n ...PullRequestEvidence\n __typename\n }\n alert {\n ...AlertEvidence\n __typename\n }\n knowledge {\n ...KnowledgeEvidence\n __typename\n }\n insertedAt\n updatedAt\n __typename\n}\n\nfragment LogsEvidence on LogsEvidence {\n clusterId\n serviceId\n line\n lines {\n ...LogLine\n __typename\n }\n __typename\n}\n\nfragment LogLine on LogLine {\n facets {\n ...LogFacet\n __typename\n }\n log\n timestamp\n __typename\n}\n\nfragment LogFacet on LogFacet {\n key\n value\n __typename\n}\n\nfragment PullRequestEvidence on PullRequestEvidence {\n contents\n filename\n patch\n repo\n sha\n title\n url\n __typename\n}\n\nfragment AlertEvidence on AlertEvidence {\n alertId\n title\n resolution\n __typename\n}\n\nfragment KnowledgeEvidence on KnowledgeEvidence {\n name\n observations\n type\n __typename\n}\n\nfragment ClusterMinimal on Cluster {\n id\n name\n handle\n provider {\n name\n cloud\n __typename\n }\n distro\n __typename\n}\n\nfragment ClusterWithUpgrade on Cluster {\n ...ClusterRuntimeServices\n deprecatedCustomResources {\n ...ClusterUpgradeDeprecatedCustomResource\n __typename\n }\n upgradePlanSummary {\n blockingAddons {\n ...RuntimeAddonUpgrade\n __typename\n }\n blockingCloudAddons {\n ...CloudAddonUpgrade\n __typename\n }\n __typename\n }\n __typename\n}\n\nfragment ClusterRuntimeServices on Cluster {\n id\n name\n currentVersion\n version\n runtimeServices {\n ...RuntimeService\n __typename\n }\n apiDeprecations {\n ...ApiDeprecation\n __typename\n }\n upgradeInsights {\n ...UpgradeInsight\n __typename\n }\n cloudAddons {\n ...CloudAddon\n __typename\n }\n __typename\n}\n\nfragment RuntimeService on RuntimeService {\n id\n name\n version\n addon {\n icon\n versions {\n ...AddonVersion\n __typename\n }\n __typename\n }\n service {\n git {\n ref\n folder\n __typename\n }\n repository {\n httpsPath\n urlFormat\n __typename\n }\n helm {\n version\n __typename\n }\n __typename\n }\n addonVersion {\n ...AddonVersionBlocking @include(if: $hasKubeVersion)\n ...AddonVersion\n __typename\n }\n __typename\n}\n\nfragment AddonVersion on AddonVersion {\n version\n kube\n chartVersion\n incompatibilities {\n version\n name\n __typename\n }\n requirements {\n version\n name\n __typename\n }\n __typename\n}\n\nfragment AddonVersionBlocking on AddonVersion {\n blocking(kubeVersion: $kubeVersion)\n __typename\n}\n\nfragment ApiDeprecation on ApiDeprecation {\n availableIn\n blocking\n component {\n group\n version\n kind\n name\n namespace\n service {\n git {\n ref\n folder\n __typename\n }\n repository {\n httpsPath\n urlFormat\n __typename\n }\n __typename\n }\n __typename\n }\n deprecatedIn\n removedIn\n replacement\n __typename\n}\n\nfragment UpgradeInsight on UpgradeInsight {\n id\n name\n description\n details {\n ...UpgradeInsightDetail\n __typename\n }\n refreshedAt\n transitionedAt\n version\n status\n __typename\n}\n\nfragment UpgradeInsightDetail on UpgradeInsightDetail {\n id\n removedIn\n replacedIn\n replacement\n status\n used\n clientInfo {\n ...InsightClientInfo\n __typename\n }\n __typename\n}\n\nfragment InsightClientInfo on InsightClientInfo {\n userAgent\n count\n lastRequestAt\n __typename\n}\n\nfragment CloudAddon on CloudAddon {\n id\n insertedAt\n updatedAt\n name\n distro\n info {\n name\n publisher\n versions {\n version\n compatibilities\n ...CloudAddonVersionInformation @include(if: $hasKubeVersion)\n __typename\n }\n __typename\n }\n version\n versionInfo {\n version\n compatibilities\n ...CloudAddonVersionInformation @include(if: $hasKubeVersion)\n __typename\n }\n __typename\n}\n\nfragment CloudAddonVersionInformation on CloudAddonVersionInformation {\n blocking(kubeVersion: $kubeVersion)\n __typename\n}\n\nfragment ClusterUpgradeDeprecatedCustomResource on DeprecatedCustomResource {\n name\n group\n kind\n namespace\n version\n nextVersion\n __typename\n}\n\nfragment RuntimeAddonUpgrade on RuntimeAddonUpgrade {\n addon {\n name\n icon\n __typename\n }\n callout\n current {\n ...RuntimeAddonVersion\n __typename\n }\n fix {\n ...RuntimeAddonVersionWithSummary\n __typename\n }\n __typename\n}\n\nfragment RuntimeAddonVersion on AddonVersion {\n version\n chartVersion\n images\n releaseUrl\n __typename\n}\n\nfragment RuntimeAddonVersionWithSummary on AddonVersion {\n ...RuntimeAddonVersion\n summary {\n ...AddonVersionSummary\n __typename\n }\n __typename\n}\n\nfragment AddonVersionSummary on AddonVersionSummary {\n breakingChanges\n chartUpdates\n features\n helmChanges\n __typename\n}\n\nfragment CloudAddonUpgrade on CloudAddonUpgrade {\n addon {\n ...CloudAddon\n __typename\n }\n current {\n ...CloudAddonVersion\n __typename\n }\n fix {\n ...CloudAddonVersion\n __typename\n }\n __typename\n}\n\nfragment CloudAddonVersion on CloudAddonVersionInformation {\n version\n __typename\n}\n\nfragment ClusterInsight on Cluster {\n id\n insightComponents {\n ...ClusterInsightComponent\n __typename\n }\n __typename\n}\n\nfragment ClusterInsightComponent on ClusterInsightComponent {\n id\n kind\n name\n namespace\n group\n version\n priority\n service {\n id\n name\n cluster {\n id\n __typename\n }\n __typename\n }\n insight {\n ...AiInsight\n __typename\n }\n __typename\n}\n\nfragment AiInsight on AiInsight {\n id\n text\n summary\n sha\n freshness\n updatedAt\n insertedAt\n error {\n message\n source\n __typename\n }\n ...AiInsightContext\n __typename\n}\n\nfragment ClusterUpgrade on ClusterUpgrade {\n id\n status\n steps {\n ...ClusterUpgradeStep\n __typename\n }\n cluster {\n ...ClusterMinimal\n __typename\n }\n runtime {\n type\n name\n __typename\n }\n version\n __typename\n}\n\nfragment ClusterUpgradeStep on ClusterUpgradeStep {\n id\n name\n prompt\n status\n type\n error\n agentRun {\n ...AgentRunTiny\n __typename\n }\n __typename\n}\n\nfragment AgentRunTiny on AgentRun {\n id\n status\n mode\n babysit\n approval\n approvedAt\n prompt\n shared\n error\n runtime {\n id\n name\n type\n __typename\n }\n repository\n branch\n headBranch\n pullRequests {\n ...PullRequestBasic\n __typename\n }\n podReference {\n name\n namespace\n __typename\n }\n usage {\n totalCost\n totalTokens\n __typename\n }\n workbenchJob {\n id\n workbench {\n id\n name\n __typename\n }\n __typename\n }\n upload {\n id\n session\n patch\n __typename\n }\n todos {\n ...AgentTodo\n __typename\n }\n insertedAt\n updatedAt\n __typename\n}\n\nfragment PullRequestBasic on PullRequest {\n id\n url\n title\n creator\n status\n insertedAt\n updatedAt\n __typename\n}\n\nfragment AgentTodo on AgentTodo {\n title\n description\n done\n __typename\n}\n\nfragment NodeStatistic on NodeStatistic {\n id\n name\n pendingPods\n health\n cluster {\n id\n __typename\n }\n insertedAt\n updatedAt\n __typename\n}\n\nfragment KubernetesChangelog on KubernetesChangelog {\n version\n apiUpdates\n bugFixes\n breakingChanges\n deprecations\n features\n majorChanges\n removals\n __typename\n}" }, "sha256:1225b5ef3c2d595c46bacc27710f9a3231a853da068a706edf1074998e1d52a1": { "type": "query", diff --git a/js/console/src/graph/ai/insights.graphql b/js/console/src/graph/ai/insights.graphql index a57fcc3644..7445cb2e69 100644 --- a/js/console/src/graph/ai/insights.graphql +++ b/js/console/src/graph/ai/insights.graphql @@ -146,6 +146,13 @@ fragment ClusterInsightComponent on ClusterInsightComponent { group version priority + service { + id + name + cluster { + id + } + } insight { ...AiInsight } @@ -158,6 +165,21 @@ fragment ClusterInsightComponentDetail on ClusterInsightComponent { } } +# the components of a service by identity, to find the one that is a cluster insight component +query ServiceComponentIdentities($id: ID!) { + serviceDeployment(id: $id) { + id + components { + id + group + version + kind + namespace + name + } + } +} + query AiInsight($id: ID!) { aiInsight(id: $id) { ...AiInsight diff --git a/js/documentation/generated/routes.json b/js/documentation/generated/routes.json index 972982d9e3..d1d530ff11 100644 --- a/js/documentation/generated/routes.json +++ b/js/documentation/generated/routes.json @@ -159,6 +159,10 @@ "relPath": "/plural-features/continuous-deployment/deployment-operator/custom-health.md", "lastmod": "2026-09-02T10:37:22.000Z" }, + "/plural-features/continuous-deployment/deployment-operator/insight-priority": { + "relPath": "/plural-features/continuous-deployment/deployment-operator/insight-priority.md", + "lastmod": "2026-10-09T13:11:56.743Z" + }, "/plural-features/continuous-deployment/git-service": { "relPath": "/plural-features/continuous-deployment/git-service.md", "lastmod": "2026-09-02T10:37:22.000Z" diff --git a/js/documentation/pages/api-reference/kubernetes/agent-api-reference.md b/js/documentation/pages/api-reference/kubernetes/agent-api-reference.md index c38721f91c..b48be5d669 100644 --- a/js/documentation/pages/api-reference/kubernetes/agent-api-reference.md +++ b/js/documentation/pages/api-reference/kubernetes/agent-api-reference.md @@ -14,6 +14,7 @@ Package v1alpha1 contains API Schema definitions for the deployments v1alpha1 AP - [AgentRuntime](#agentruntime) - [ClusterDrain](#clusterdrain) - [CustomHealth](#customhealth) +- [CustomInsightPriority](#custominsightpriority) - [ImageWarmer](#imagewarmer) - [IngressReplica](#ingressreplica) - [KubecostExtractor](#kubecostextractor) @@ -604,6 +605,45 @@ _Appears in:_ +#### CustomInsightPriority + + + +CustomInsightPriority sets the priority of failing objects in the cluster insights and the cluster health score. + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `deployments.plural.sh/v1alpha1` | | | +| `kind` _string_ | `CustomInsightPriority` | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[CustomInsightPrioritySpec](#custominsightpriorityspec)_ | | | | + + +#### CustomInsightPrioritySpec + + + +CustomInsightPrioritySpec defines how the insight priority of failing objects is decided. + + + +_Appears in:_ +- [CustomInsightPriority](#custominsightpriority) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `target` _[InsightTarget](#insighttarget)_ | Target selects the failing objects this script sets the priority of. | | Optional: \{\}
| +| `precedence` _integer_ | Precedence decides which script owns an object that several targets match: the highest wins and ties are
broken by name. A script never sees objects it does not own, and scripts are not chained. | | Optional: \{\}
| +| `engine` _[InsightEngine](#insightengine)_ | Engine the script is written for. | python | Enum: [python]
Optional: \{\}
| +| `script` _string_ | Script defines `evaluate(input)`, called with the objects this script owns in `input["rows"]`, and returns a
dictionary from the `uid` of a row to `\{"priority": "LOW\|MEDIUM\|HIGH\|CRITICAL\|IGNORE", "reason": "..."\}`.
Rows that are left out keep their built-in priority, IGNORE removes a row from the insights and the cluster health score.
Objects annotated with deployments.plural.sh/insight-priority are decided by the annotation and never reach the script.
See https://docs.plural.sh/plural-features/continuous-deployment/deployment-operator/insight-priority for the full guide. | | | + + + + #### ExaConnection @@ -863,6 +903,80 @@ _Appears in:_ | `hostMappings` _object (keys:string, values:string)_ | | | Required: \{\}
| +#### InsightEngine + +_Underlying type:_ _string_ + +InsightEngine is the language an insight priority script is written in. + +_Validation:_ +- Enum: [python] + +_Appears in:_ +- [CustomInsightPrioritySpec](#custominsightpriorityspec) + +| Field | Description | +| --- | --- | +| `python` | InsightEnginePython evaluates the script as sandboxed Python.
| + + +#### InsightEvaluation + + + +InsightEvaluation is the outcome of the last time the script ran. + + + +_Appears in:_ +- [CustomInsightPriorityStatus](#custominsightprioritystatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `time` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#time-v1-meta)_ | Time the script last ran with a different outcome than the one before. | | | +| `matched` _integer_ | Matched is the number of failing objects the script owned. | | | +| `overridden` _integer_ | Overridden is the number of objects the script gave a priority. | | | +| `ignored` _integer_ | Ignored is the number of objects the script ignored. | | | +| `durationMs` _integer_ | DurationMs is how long the script took. | | | + + +#### InsightTarget + + + +InsightTarget selects the failing objects a script is responsible for. An object has to match every +non-empty field, so an empty target matches everything. + + + +_Appears in:_ +- [CustomInsightPrioritySpec](#custominsightpriorityspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `kinds` _[InsightTargetKind](#insighttargetkind) array_ | Kinds the script is responsible for. | | Optional: \{\}
| +| `namespaces` _string array_ | Namespaces the script is responsible for. Cluster scoped objects only match when this is empty. | | Optional: \{\}
| +| `labelSelector` _[LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#labelselector-v1-meta)_ | LabelSelector selects objects by their labels. | | Optional: \{\}
| + + +#### InsightTargetKind + + + +InsightTargetKind selects objects by kind. + + + +_Appears in:_ +- [InsightTarget](#insighttarget) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `group` _string_ | Group of the kind, empty for the core group. | | Optional: \{\}
| +| `version` _string_ | Version of the kind, any version matches when empty. | | Optional: \{\}
| +| `kind` _string_ | Kind of the objects, for example StatefulSet. | | MinLength: 1
| + + #### KubecostExtractor diff --git a/js/documentation/pages/plural-features/continuous-deployment/deployment-operator/insight-priority.md b/js/documentation/pages/plural-features/continuous-deployment/deployment-operator/insight-priority.md new file mode 100644 index 0000000000..59759ad198 --- /dev/null +++ b/js/documentation/pages/plural-features/continuous-deployment/deployment-operator/insight-priority.md @@ -0,0 +1,193 @@ +--- +title: Insight priority and cluster health score +description: How the deployment operator prioritizes failing workloads, scores cluster health, and how to customize both +--- + +Every few minutes the deployment operator reports two things about its cluster to Console: a list of failing components, each with a priority from `LOW` to `CRITICAL`, and a single cluster health score from 0 to 100. Console generates an AI insight for every listed component, shows the priority in the UI, and uses it to decide which failures deserve a response. The health score drives the cluster heatmap, the health chip, and the `healthScore` filters of the API. + +Both are computed by the operator itself from the objects it watches, with fixed rules. The same failure always gets the same priority, and the priority is never decided by an LLM. You can pin or ignore single objects with an annotation, or change the rules for whole groups of objects with a `CustomInsightPriority`. + +## What counts as a failure + +The operator reports one **root** per failure rather than one row per failing pod: + +- Every failed object is followed up its owners, at most four levels, to the first Deployment, StatefulSet, DaemonSet, Ingress, Certificate or CronJob. That object is the root. A Deployment with ten crashlooping pods is one root. +- If the owners end without such an object, the top-most owner is the root. This is how failed nodes, persistent volume claims, standalone pods and jobs, Argo Rollouts, Flagger canaries, and custom resources that a [CustomHealth](/plural-features/continuous-deployment/deployment-operator/custom-health) script marks as degraded are reported. +- A failure that is more than four levels below the nearest owner is ignored. +- A CronJob is a failure only while its newest job failed, and a node only after it has been failing for five minutes. + +Failures of workloads deployed by a Plural service are reported too. They carry the service that manages them, taken from the object or its nearest owner, and Console links the component to that service. + +## How the priority is decided + +The first source that applies wins: + +1. The `deployments.plural.sh/insight-priority` annotation on the object. +2. The `CustomInsightPriority` script that owns the object. +3. The built-in score. + +### The built-in score + +The score is the criticality of the component plus the severity of its failure, and it maps to a priority at 25, 50 and 75: + +| Score | Priority | +| ----- | -------- | +| below 25 | `LOW` | +| 25 to 49 | `MEDIUM` | +| 50 to 74 | `HIGH` | +| 75 and above | `CRITICAL` | + +Every failing object is either a **core component** of the cluster or an **app**, and that decides the range it can reach on its own: + +| Class | Starts at | Highest it can get | Typical result | +| ----- | --------- | ------------------ | -------------- | +| App | 10, or 35 for Ingresses, Certificates and persistent volumes | `MEDIUM` | partly down is `LOW`, down or with a broken Ingress or Certificate is `MEDIUM` | +| Core | 60 | `CRITICAL` | partly down is `HIGH`, mostly down is `CRITICAL` | + +Only workloads with replicas can reach `CRITICAL` on their own, since that comes from the share of replicas that are down. A failed node, or an Ingress or Certificate in a core namespace, is `HIGH`. + +`HIGH` and `CRITICAL` are therefore reserved for what keeps the cluster and the apps running, and a broken third-party workload never turns the cluster red on its own. To treat one of your own apps as important, pin its priority with an [annotation](#annotations) or a [script](#custominsightpriority), neither is limited by this. + +**Criticality** says whether the component is core. The first rule that matches decides: + +1. A `system-*` priority class, such as `system-cluster-critical`, is core. +2. A well-known cluster component is matched by name and is core. These are DNS (`coredns`, `kube-dns`), `kube-proxy`, CNI plugins (`aws-node`, `cilium`, `calico`, `flannel` and others), CSI drivers, service mesh control planes (`istio`, `linkerd`), `cert-manager`, ingress controllers, `metrics-server`, the cluster autoscaler and Karpenter. +3. A system namespace is core: `kube-system`, `istio-system`, `linkerd`, `cert-manager`, `ingress-nginx`, `traefik`, `calico-system`, `tigera-operator`, `cilium`, `plrl-console` and `plrl-deploy-operator`. +4. A node is core. +5. Everything else is an app. A Deployment, StatefulSet, DaemonSet, Job or CronJob starts at 10. An Ingress, Certificate or persistent volume starts at 35. + +Names are matched on whole words only. The name is split on `-`, `_` and `.`, and any run of consecutive words, joined together, can match a known component, so `core-dns` and `rke2-ingress-nginx-controller` match, while `concert-manager` and `core-banking-dns-proxy` do not. + +**Severity** says how bad the failure is, and adds to the criticality: + +- Up to 20 points for the share of replicas that are not ready. A Deployment with none of its three replicas ready adds the full 20, one with two of three ready adds 7. A core component with three quarters of its replicas down is `CRITICAL`. +- 3 points when the failure has lasted an hour, 5 points when it has lasted a day. + +The score of an app stops at 49 however severe the failure is. + +## Annotations + +Set the annotation on any object, usually the Deployment, StatefulSet or DaemonSet itself: + +```yaml +metadata: + annotations: + deployments.plural.sh/insight-priority: HIGH +``` + +`LOW`, `MEDIUM`, `HIGH` and `CRITICAL` pin the priority. `IGNORE` removes the object from the insights and from the health score. Scripts never see an object that has the annotation. + +## CustomInsightPriority + +A `CustomInsightPriority` runs a Python script over the failing objects it targets and lets it set their priority. Like [CustomHealth](/plural-features/continuous-deployment/deployment-operator/custom-health), it is a deployment-operator API, so apply it to each cluster where it should be used, or distribute it with a [GlobalService](/plural-features/continuous-deployment/global-service). + +The built-in rules rank a third-party workload at most `MEDIUM`, because the operator cannot know which of your apps matter. A script is how you tell it. The example below treats the StatefulSets labelled `tier: core` in two namespaces as important and raises them to `CRITICAL` when none of their replicas is ready: + +```yaml +apiVersion: deployments.plural.sh/v1alpha1 +kind: CustomInsightPriority +metadata: + name: payments + namespace: plrl-deploy-operator +spec: + target: + kinds: + - group: apps + kind: StatefulSet + namespaces: [payments, ledger] + labelSelector: + matchLabels: + tier: core + precedence: 100 + script: | + def evaluate(input): + out = {} + for r in input["rows"]: + if r["ready_replicas"] == 0: + out[r["uid"]] = {"priority": "CRITICAL", "reason": "no ready replicas"} + return out +``` + +The CRD is installed by the `deployment-operator` chart. Helm does not upgrade CRDs of an existing release, so on a cluster that was installed before this API existed, apply the CRD from the chart's `crds/` directory first. An agent that finds no CRD starts without the feature and logs it. + +{% callout severity="warn" %} +A `CustomInsightPriority` has a cluster-wide effect whatever namespace it lives in. Its script receives the failing objects of every namespace it targets, with their labels and health messages, plus the names of all failing roots in the cluster, and it can hide failures with `IGNORE`. Only allow people you would trust with CustomHealth, or cluster-admin, to create it. +{% /callout %} + +### Targeting + +`spec.target` selects the failing objects the script is responsible for. An object has to match every field that is set, so an empty target matches everything. + +- `kinds` is a list of `group`, optional `version`, and `kind`. +- `namespaces` is a list of namespaces. Cluster-scoped objects only match when this is empty. +- `labelSelector` is a standard label selector on the labels of the object. + +When several scripts match an object, the one with the highest `spec.precedence` owns it, and ties are broken by name. A script only receives the objects it owns, and scripts are not chained. A script that lost objects to another one gets the `Overlapping` condition naming the winners. + +### The script contract + +The script defines `evaluate(input)` and returns a dictionary from the `uid` of a row to `{"priority": ..., "reason": ...}`. `priority` is `LOW`, `MEDIUM`, `HIGH`, `CRITICAL` or `IGNORE`, and `reason` is optional. Rows that are left out keep their built-in priority. Returning nothing is fine. + +`input` contains: + +```json +{ + "now": "2026-10-09T12:00:00Z", + "cluster": { + "nodes": 12, + "namespaces": 41, + "failing_roots": [{ "kind": "DaemonSet", "namespace": "kube-system", "name": "aws-node", "builtin_priority": "CRITICAL" }] + }, + "rows": [ + { + "uid": "6c1f...", + "group": "apps", "version": "v1", "kind": "StatefulSet", + "namespace": "payments", "name": "ledger-db", + "labels": { "tier": "core" }, + "managed": true, "service_id": "a1b2...", + "priority_class": "", + "state": "failed", + "health_message": "Deployment \"ledger-db\" exceeded its progress deadline", + "desired_replicas": 3, "ready_replicas": 0, + "descendants": { "Pod": { "failed": 3, "pending": 0 } }, + "created_at": "2026-10-01T08:00:00Z", "failing_since": "2026-10-09T11:00:00Z", + "criticality": 10, "builtin_score": 30, + "builtin_priority": "MEDIUM", "builtin_reason": "kind default: StatefulSet" + } + ] +} +``` + +`state` is `failed` when the object itself is failing, and `pending` when only objects it owns are. `desired_replicas` and `ready_replicas` are `None` for kinds without replicas. Manifests, events, logs, metrics and annotations are not part of the input. + +The script runs in a sandbox. It has no access to the network, the file system, the environment, the clock or the cluster, so the same input always gives the same result. Loading the script and calling `evaluate` are each stopped after two seconds. + +### Errors + +- A script that does not compile, or does not define `evaluate`, is never used. The resource gets `Ready=False` with the reason `InvalidScript`. +- A script that raises, times out, or returns something that does not follow the contract, such as an unknown `uid` or priority, only affects its own objects: they keep their built-in priority. The resource gets `Ready=False` with the reason `EvaluationFailed`, `status.lastError` has the type of the exception and the line of the script, never its message, the operator emits a Warning event, and the `agent_insight_priority_failures_total` metric counts it. Other scripts are not affected. + +`status.lastEvaluation` shows how many objects the script owned, how many it gave a priority, how many it ignored, and how long it took. + +## Cluster health score + +The score starts at 100 and loses a penalty for every failing root that is not ignored, as decided by the priority above: + +| Priority | Penalty per root | Most that one priority can take | +| -------- | ---------------- | -------------------------------- | +| `LOW` | 2 | 10 | +| `MEDIUM` | 8 | 30 | +| `HIGH` | 25 | no limit | +| `CRITICAL` | 50 | no limit | + +The score is never below 0. Low and medium failures are capped, so any number of unimportant failures leaves a cluster in the green band, while a single critical failure moves it to 50 and two high failures to 50 as well. Raising an object with an annotation or a script moves it out of the capped tiers. The score and the insights list are computed from the same roots in the same moment, so everything that lowers the score is in the list, unless it is ignored. + +{% callout severity="info" %} +This changes what the score means. It no longer scales with the number of pods or the size of the cluster, pending objects no longer lower it, and a failing DNS, CNI or ingress controller now lowers it. Expect scores to move after upgrading, and review any API `healthScore` filters or automations that depend on exact values. +{% /callout %} + +## Recovering components + +A component that stops failing stays in the list with its last priority for ten minutes, so a workload that flaps does not lose its AI insight every time it recovers. It does not count towards the health score during that time. Set the period with the `--insight-grace-period` argument of the operator, `0` turns it off. Objects that are deleted or annotated `IGNORE` leave the list immediately. + +After the operator starts, the first report waits until it has loaded the state of the whole cluster, so a restart never reports a partial list. diff --git a/js/documentation/src/routing/docs-structure.ts b/js/documentation/src/routing/docs-structure.ts index 90cb4f2934..2dd6ebf4f0 100644 --- a/js/documentation/src/routing/docs-structure.ts +++ b/js/documentation/src/routing/docs-structure.ts @@ -135,6 +135,10 @@ export const docsStructure: DocSection[] = [ path: 'custom-health', title: 'CustomHealth Lua authoring', }, + { + path: 'insight-priority', + title: 'Insight priority and health score', + }, ], }, { path: 'git-service', title: 'Git-sourced services' }, diff --git a/lib/console/ai/memoizer.ex b/lib/console/ai/memoizer.ex index b5c350604f..7a5574471d 100644 --- a/lib/console/ai/memoizer.ex +++ b/lib/console/ai/memoizer.ex @@ -5,7 +5,7 @@ defmodule Console.AI.Memoizer do @spec generate(struct) :: {:ok, AiInsight.t} | {:error, binary} def generate(model) do - model = Evidence.preload(model) + model = model |> Evidence.preload() |> adopt() insight = Evidence.insight(model) case AiInsight.memoized?(insight, nil) do true -> @@ -15,6 +15,80 @@ defmodule Console.AI.Memoizer do end end + @doc """ + The same kubernetes object can be a component of a service and a component of its cluster, and each of them has an + insight of its own. They are made to share one row, so the object is paid for once and what they show cannot drift apart. + + The service component has the richer context, so it owns the row: a cluster component always points at the fresh + insight of its service component, even when it has a fresh one of its own, and is detached from a stale one instead of + regenerating it. A service component only adopts the insight of a cluster component when it has no fresh one itself. + """ + @spec adopt(struct) :: struct + def adopt(%ClusterInsightComponent{insight: own} = model) do + case find_peer(model) do + %{insight: %AiInsight{} = shared} -> + own_id = with %AiInsight{id: id} <- own, do: id + fresh? = AiInsight.memoized?(shared, nil) + + cond do + fresh? and own_id != shared.id -> link(model, shared) + fresh? -> %{model | insight: shared} + # a forced row is most likely owned by a service refresh, so it is detached instead of overwritten + match?(%AiInsight{force: true}, own) -> link(model, nil) + own_id == shared.id -> link(model, nil) + true -> model + end + _ -> model + end + end + def adopt(model) do + case Evidence.insight(model) do + # a forced refresh must regenerate, not fall back to a peer's existing diagnosis + %AiInsight{force: true} -> model + insight -> + case AiInsight.memoized?(insight, nil) do + true -> model + false -> find_peer(model) |> adopt(model) + end + end + end + + defp find_peer(%ClusterInsightComponent{cluster_id: cid, group: g, version: v, kind: k, namespace: ns, name: n}) do + ServiceComponent.for_cluster(cid) + |> ServiceComponent.for_object({g, v, k, ns, n}) + |> ServiceComponent.with_insight() + |> Repo.one() + end + defp find_peer(%ServiceComponent{group: g, version: v, kind: k, namespace: ns, name: n, service: %{cluster_id: cid}}) do + ClusterInsightComponent.for_cluster(cid) + |> ClusterInsightComponent.for_object({g, v, k, ns, n}) + |> ClusterInsightComponent.with_insight() + |> Repo.one() + end + defp find_peer(_), do: nil + + defp adopt(%{insight: %AiInsight{} = shared}, model) do + cond do + AiInsight.memoized?(shared, nil) -> link(model, shared) + detach?(model, shared) -> link(model, nil) + true -> model + end + end + defp adopt(_, model), do: model + + defp detach?(%ClusterInsightComponent{insight: %AiInsight{id: id}}, %AiInsight{id: id}), do: true + defp detach?(_, _), do: false + + defp link(%{} = model, insight) do + model + |> Ecto.Changeset.change(insight_id: insight && insight.id) + |> Repo.update(allow_stale: true) + |> case do + {:ok, updated} -> %{updated | insight: insight} + _ -> model + end + end + defp try_generate(model, insight) do set_context(model) with {:ok, [_ | _] = history, attrs} <- Evidence.generate(model), diff --git a/lib/console/deployments/clusters.ex b/lib/console/deployments/clusters.ex index ba289d6a4a..3438ff87cc 100644 --- a/lib/console/deployments/clusters.ex +++ b/lib/console/deployments/clusters.ex @@ -860,13 +860,36 @@ defmodule Console.Deployments.Clusters do |> notify(:ping) end - defp stabilize_insight_components(%{insight_components: [_ | _] = new} = attrs, %Cluster{insight_components: existing}) do + defp stabilize_insight_components(%{insight_components: [_ | _] = new} = attrs, %Cluster{id: cluster_id, insight_components: existing}) do key = fn %{group: g, version: v, kind: k, name: n} = attrs -> {g, v, k, Map.get(attrs, :namespace), n} end by_key = Map.new(existing, & {key.(&1), &1.id}) - Map.put(attrs, :insight_components, Enum.map(new, &Map.put(&1, :id, by_key[key.(&1)]))) + known = known_service_ids(cluster_id, new) + Map.put(attrs, :insight_components, Enum.map(new, fn comp -> + comp + |> Map.put(:id, by_key[key.(comp)]) + |> Map.put(:service_id, if(MapSet.member?(known, comp[:service_id]), do: comp[:service_id], else: nil)) + end)) end defp stabilize_insight_components(attrs, _), do: Map.put(attrs, :insight_components, []) + # a service can be deleted between the agent's scrape and its ping, and the agent is only trusted for its own cluster's services, + # so unknown, malformed or foreign ids are dropped rather than failing the whole ping + defp known_service_ids(cluster_id, components) do + components + |> Enum.flat_map(&List.wrap(&1[:service_id])) + |> Enum.filter(&match?({:ok, _}, Ecto.UUID.cast(&1))) + |> Enum.uniq() + |> case do + [] -> MapSet.new() + ids -> + Service.for_cluster(cluster_id) + |> Service.for_ids(ids) + |> Service.ids() + |> Repo.all() + |> MapSet.new() + end + end + defp stabilize_operational_layout(attrs, cluster) do Console.put_path(attrs, [:operational_layout, :id], case cluster do %Cluster{operational_layout: %{id: id}} -> id diff --git a/lib/console/graphql/ai.ex b/lib/console/graphql/ai.ex index cb1ae30c68..38ae44ad40 100644 --- a/lib/console/graphql/ai.ex +++ b/lib/console/graphql/ai.ex @@ -306,6 +306,7 @@ defmodule Console.GraphQl.AI do field :name, non_null(:string) field :priority, :insight_component_priority + field :service, :service_deployment, resolve: dataloader(Deployments), description: "the plural service managing this component, if any" field :cluster, :cluster, resolve: dataloader(Deployments) field :insight, :ai_insight, resolve: dataloader(AI) diff --git a/lib/console/graphql/deployments/cluster.ex b/lib/console/graphql/deployments/cluster.ex index a184d716ef..42971858f8 100644 --- a/lib/console/graphql/deployments/cluster.ex +++ b/lib/console/graphql/deployments/cluster.ex @@ -97,6 +97,7 @@ defmodule Console.GraphQl.Deployments.Cluster do field :namespace, :string field :name, non_null(:string) field :priority, :insight_component_priority + field :service_id, :id, description: "the plural service that manages this component, if any" end input_object :node_statistic_attributes do diff --git a/lib/console/schema/cluster_insight_component.ex b/lib/console/schema/cluster_insight_component.ex index 08ac23d6fc..d36f6620e3 100644 --- a/lib/console/schema/cluster_insight_component.ex +++ b/lib/console/schema/cluster_insight_component.ex @@ -1,6 +1,6 @@ defmodule Console.Schema.ClusterInsightComponent do use Piazza.Ecto.Schema - alias Console.Schema.{Cluster, AiInsight} + alias Console.Schema.{Cluster, AiInsight, Service} @expiry [hours: -4] @@ -15,27 +15,54 @@ defmodule Console.Schema.ClusterInsightComponent do field :priority, Priority belongs_to :cluster, Cluster + belongs_to :service, Service belongs_to :insight, AiInsight, on_replace: :update timestamps() end + # A component that stays in the agent's pings is not touched, so it can only be told apart from a stale one by its cluster: + # components leave a cluster's list with the ping that no longer reports them, the ones that are left are those of clusters that went quiet. def expired(query \\ __MODULE__) do expiry = Timex.now() |> Timex.shift(@expiry) - from(cic in query, where: coalesce(cic.updated_at, cic.inserted_at) < ^expiry) + from(cic in query, + join: c in assoc(cic, :cluster), + where: coalesce(c.pinged_at, c.inserted_at) < ^expiry + ) end def for_cluster(query \\ __MODULE__, cluster_id) do from(cic in query, where: cic.cluster_id == ^cluster_id) end - @valid ~w(group version kind namespace name priority cluster_id)a + # the core group and cluster scoped objects are stored as either nil or an empty string depending on who reported them + def for_object(query \\ __MODULE__, {g, v, k, ns, n}) do + from(cic in query, + where: coalesce(cic.group, "") == ^(g || "") + and cic.version == ^v + and cic.kind == ^k + and coalesce(cic.namespace, "") == ^(ns || "") + and cic.name == ^n + ) + end + + def with_insight(query \\ __MODULE__) do + from(cic in query, + where: not is_nil(cic.insight_id), + order_by: [desc: coalesce(cic.updated_at, cic.inserted_at)], + preload: [insight: :evidence], + limit: 1 + ) + end + + @valid ~w(group version kind namespace name priority cluster_id service_id)a def changeset(model, attrs \\ %{}) do model |> cast(attrs, @valid) |> cast_assoc(:insight) |> foreign_key_constraint(:cluster_id) + |> foreign_key_constraint(:service_id) |> validate_required([:kind, :name]) end end diff --git a/lib/console/schema/service.ex b/lib/console/schema/service.ex index 328b845b5e..d77d9a8842 100644 --- a/lib/console/schema/service.ex +++ b/lib/console/schema/service.ex @@ -340,6 +340,10 @@ defmodule Console.Schema.Service do ) end + def ids(query \\ __MODULE__) do + from(s in query, select: s.id) + end + def for_clusters(query \\ __MODULE__, cluster_ids) do from(s in query, where: s.cluster_id in ^cluster_ids) end diff --git a/lib/console/schema/service_component.ex b/lib/console/schema/service_component.ex index 0884fa5d41..60ac13af09 100644 --- a/lib/console/schema/service_component.ex +++ b/lib/console/schema/service_component.ex @@ -43,6 +43,33 @@ defmodule Console.Schema.ServiceComponent do from(sc in query, where: sc.service_id == ^service_id) end + def for_cluster(query \\ __MODULE__, cluster_id) do + from(sc in query, + join: s in assoc(sc, :service), + where: s.cluster_id == ^cluster_id + ) + end + + # the core group and cluster scoped objects are stored as either nil or an empty string depending on who reported them + def for_object(query \\ __MODULE__, {g, v, k, ns, n}) do + from(sc in query, + where: coalesce(sc.group, "") == ^(g || "") + and sc.version == ^v + and sc.kind == ^k + and coalesce(sc.namespace, "") == ^(ns || "") + and sc.name == ^n + ) + end + + def with_insight(query \\ __MODULE__) do + from(sc in query, + where: not is_nil(sc.insight_id), + order_by: [desc: coalesce(sc.updated_at, sc.inserted_at)], + preload: [insight: :evidence], + limit: 1 + ) + end + def count_by_flow_state(ids) do from(sc in __MODULE__, join: s in assoc(sc, :service), diff --git a/priv/repo/migrations/20261012140000_add_service_id_to_cluster_insight_components.exs b/priv/repo/migrations/20261012140000_add_service_id_to_cluster_insight_components.exs new file mode 100644 index 0000000000..5b0c7895dd --- /dev/null +++ b/priv/repo/migrations/20261012140000_add_service_id_to_cluster_insight_components.exs @@ -0,0 +1,11 @@ +defmodule Console.Repo.Migrations.AddServiceIdToClusterInsightComponents do + use Ecto.Migration + + def change do + alter table(:cluster_insight_components) do + add :service_id, references(:services, type: :uuid, on_delete: :nilify_all) + end + + create index(:cluster_insight_components, [:service_id]) + end +end diff --git a/schema/schema.graphql b/schema/schema.graphql index c01fdcc87c..02967f06a2 100644 --- a/schema/schema.graphql +++ b/schema/schema.graphql @@ -13099,11 +13099,19 @@ input ClusterPing { input ClusterInsightComponentAttributes { group: String + version: String! + kind: String! + namespace: String + name: String! + priority: InsightComponentPriority + + "the plural service that manages this component, if any" + serviceId: ID } input NodeStatisticAttributes { @@ -17029,6 +17037,9 @@ type ClusterInsightComponent { priority: InsightComponentPriority + "the plural service managing this component, if any" + service: ServiceDeployment + cluster: Cluster insight: AiInsight diff --git a/test/console/ai/memoizer_test.exs b/test/console/ai/memoizer_test.exs index c90678d292..b37d1e32e9 100644 --- a/test/console/ai/memoizer_test.exs +++ b/test/console/ai/memoizer_test.exs @@ -4,7 +4,7 @@ defmodule Console.AI.MemoizerTest do alias Console.AI.Memoizer alias Console.Repo - alias Console.Schema.Service + alias Console.Schema.{Service, ClusterInsightComponent, ServiceComponent} alias Console.AI.Tools.Insight @base_model "gpt-5.4" @@ -63,6 +63,135 @@ defmodule Console.AI.MemoizerTest do end end + describe "adopt/1" do + @object %{group: "apps", version: "v1", kind: "Deployment", namespace: "app", name: "api"} + + defp stale(), do: Timex.now() |> Timex.shift(days: -1) + + defp cluster_comp(cluster, attrs \\ []) do + insert(:cluster_insight_component, Keyword.merge([cluster: cluster] ++ Map.to_list(@object), attrs)) + |> Repo.preload([:cluster, insight: :evidence]) + end + + defp service_comp(svc, attrs \\ []) do + insert(:service_component, Keyword.merge([service: svc] ++ Map.to_list(@object), attrs)) + |> Repo.preload([insight: :evidence, service: :cluster]) + end + + test "a cluster component adopts the fresh insight of the service component of the same object" do + cluster = insert(:cluster) + svc = insert(:service, cluster: cluster) + insight = insert(:ai_insight, text: "the service insight") + service_comp(svc, insight: insight) + comp = cluster_comp(cluster) + + adopted = Memoizer.adopt(comp) + + assert adopted.insight_id == insight.id + assert adopted.insight.id == insight.id + assert refetch(comp).insight_id == insight.id + end + + test "a service component adopts the fresh insight of the cluster component of the same object" do + cluster = insert(:cluster) + svc = insert(:service, cluster: cluster) + insight = insert(:ai_insight, text: "the cluster insight") + cluster_comp(cluster, insight: insight) + comp = service_comp(svc) + + adopted = Memoizer.adopt(comp) + + assert adopted.insight_id == insight.id + assert refetch(comp).insight_id == insight.id + end + + test "a stale insight of the other component is not adopted" do + cluster = insert(:cluster) + svc = insert(:service, cluster: cluster) + service_comp(svc, insight: insert(:ai_insight, updated_at: stale())) + comp = cluster_comp(cluster) + + assert Memoizer.adopt(comp).insight_id == nil + end + + test "an insight of a service on another cluster is not adopted" do + cluster = insert(:cluster) + other = insert(:service, cluster: insert(:cluster)) + service_comp(other, insight: insert(:ai_insight)) + comp = cluster_comp(cluster) + + assert Memoizer.adopt(comp).insight_id == nil + end + + test "a cluster component switches to the fresh insight of its service component even if it has one of its own" do + cluster = insert(:cluster) + svc = insert(:service, cluster: cluster) + shared = insert(:ai_insight, text: "the service insight") + service_comp(svc, insight: shared) + comp = cluster_comp(cluster, insight: insert(:ai_insight, text: "its own insight")) + + adopted = Memoizer.adopt(comp) + + assert adopted.insight_id == shared.id + assert refetch(comp).insight_id == shared.id + end + + test "a service component keeps a fresh insight of its own" do + cluster = insert(:cluster) + svc = insert(:service, cluster: cluster) + cluster_comp(cluster, insight: insert(:ai_insight, text: "the cluster insight")) + own = insert(:ai_insight, text: "its own insight") + comp = service_comp(svc, insight: own) + + assert Memoizer.adopt(comp).insight_id == own.id + end + + test "the core group and namespaces match whether they are nil or empty" do + cluster = insert(:cluster) + svc = insert(:service, cluster: cluster) + insight = insert(:ai_insight) + service_comp(svc, group: nil, kind: "Node", namespace: nil, name: "node-1", insight: insight) + comp = cluster_comp(cluster, group: "", kind: "Node", namespace: "", name: "node-1") + + assert Memoizer.adopt(comp).insight_id == insight.id + end + + test "a cluster component is detached from a stale insight it shares, so it does not overwrite it" do + cluster = insert(:cluster) + svc = insert(:service, cluster: cluster) + shared = insert(:ai_insight, text: "service context", updated_at: stale()) + service = service_comp(svc, insight: shared) + comp = cluster_comp(cluster, insight: shared) + + detached = Memoizer.adopt(comp) + + assert detached.insight_id == nil + assert refetch(comp).insight_id == nil + assert refetch(service).insight_id == shared.id + end + end + + describe "generate/1 with a peer" do + test "does not generate an insight the other component already has" do + reject(&ReqLLM.generate_text/3) + cluster = insert(:cluster) + svc = insert(:service, cluster: cluster) + insight = insert(:ai_insight, text: "the service insight") + + insert(:service_component, + service: svc, group: "apps", version: "v1", kind: "Deployment", namespace: "app", name: "api", insight: insight + ) + comp = insert(:cluster_insight_component, + cluster: cluster, group: "apps", version: "v1", kind: "Deployment", namespace: "app", name: "api" + ) + + assert {:ok, %{id: id}} = Memoizer.generate(comp) + assert id == insight.id + assert %ClusterInsightComponent{insight_id: ^id} = Repo.get(ClusterInsightComponent, comp.id) + assert %ServiceComponent{insight_id: ^id} = Repo.get_by(ServiceComponent, service_id: svc.id) + end + end + defp insight_tool_response(model) do Jason.encode!(%{ object: "response", diff --git a/test/console/deployments/clusters_test.exs b/test/console/deployments/clusters_test.exs index 1dbf31f5df..835326f0a5 100644 --- a/test/console/deployments/clusters_test.exs +++ b/test/console/deployments/clusters_test.exs @@ -2,7 +2,7 @@ defmodule Console.Deployments.ClustersTest do use Console.DataCase, async: true use Mimic alias Console.PubSub - alias Console.Deployments.{Clusters, Services, Settings, Compatibilities} + alias Console.Deployments.{Clusters, Services, Settings, Compatibilities, Cron} alias Kazan.Apis.Core.V1, as: CoreV1 import KubernetesScaffolds @@ -1317,6 +1317,67 @@ defmodule Console.Deployments.ClustersTest do assert_receive {:event, %PubSub.ClusterPinged{item: ^pinged}} end + + test "it keeps insight components that are still reported when pruning" do + cluster = insert(:cluster) + stale = Timex.now() |> Timex.shift(hours: -5) + component = insert(:cluster_insight_component, + cluster: cluster, + group: "apps", + version: "v1", + kind: "Deployment", + namespace: "default", + name: "test", + insight: build(:ai_insight), + inserted_at: stale, + updated_at: stale + ) + + # the agent keeps reporting the same failing component on every ping + {:ok, _} = Clusters.ping(%{ + insight_components: [ + %{group: "apps", version: "v1", kind: "Deployment", namespace: "default", name: "test"} + ] + }, cluster) + + Cron.prune_insight_components() + + assert refetched = refetch(component) + assert refetched.insight_id == component.insight_id + end + + test "it prunes the insight components of clusters that stopped pinging" do + quiet = insert(:cluster, pinged_at: Timex.now() |> Timex.shift(hours: -5)) + active = insert(:cluster, pinged_at: Timex.now()) + stale = Timex.now() |> Timex.shift(hours: -5) + gone = insert(:cluster_insight_component, cluster: quiet, inserted_at: stale, updated_at: stale) + kept = insert(:cluster_insight_component, cluster: active, inserted_at: stale, updated_at: stale) + + Cron.prune_insight_components() + + refute refetch(gone) + assert refetch(kept) + end + + test "it stores the service managing an insight component, but only if it runs on the cluster" do + cluster = insert(:cluster) + svc = insert(:service, cluster: cluster) + foreign = insert(:service) + + {:ok, _} = Clusters.ping(%{ + insight_components: [ + %{group: "apps", version: "v1", kind: "Deployment", namespace: "default", name: "managed", service_id: svc.id}, + %{group: "apps", version: "v1", kind: "Deployment", namespace: "default", name: "foreign", service_id: foreign.id}, + %{group: "apps", version: "v1", kind: "Deployment", namespace: "default", name: "malformed", service_id: "not-a-uuid"}, + %{group: "apps", version: "v1", kind: "Deployment", namespace: "default", name: "unmanaged"} + ] + }, cluster) + + by_name = Console.Repo.all(Console.Schema.ClusterInsightComponent.for_cluster(cluster.id)) + |> Map.new(& {&1.name, &1.service_id}) + + assert by_name == %{"managed" => svc.id, "foreign" => nil, "malformed" => nil, "unmanaged" => nil} + end end describe "#install/1" do