From 10db28fc11a02edf9414342db54bc82f2d664729 Mon Sep 17 00:00:00 2001 From: Alexey Gorovenko Date: Sat, 25 Jul 2026 06:37:53 +0300 Subject: [PATCH 1/5] fix: fail Deployment tracking on ReplicaSet pod-create errors Read FailedCreate from the durable ReplicaSetReplicaFailure condition instead of transient Warning events, so failures that predate informer startup terminate tracking immediately. Only consider ReplicaSets controlled by the tracked Deployment to avoid selector collisions. Signed-off-by: Alexey Gorovenko (cherry picked from commit a2e5c1f654f951faff3944ec1d489d3c621e5fb3) --- pkg/tracker/deployment/export_test.go | 11 + pkg/tracker/deployment/harness_test.go | 326 ++++++++++++++++++ pkg/tracker/deployment/tracker.go | 152 ++++++-- .../deployment/tracker_replicaset_test.go | 189 ++++++++++ pkg/tracker/deployment/tracker_test.go | 273 +++++++++++++++ pkg/utils/deployment_utils.go | 31 +- 6 files changed, 952 insertions(+), 30 deletions(-) create mode 100644 pkg/tracker/deployment/export_test.go create mode 100644 pkg/tracker/deployment/harness_test.go create mode 100644 pkg/tracker/deployment/tracker_replicaset_test.go create mode 100644 pkg/tracker/deployment/tracker_test.go diff --git a/pkg/tracker/deployment/export_test.go b/pkg/tracker/deployment/export_test.go new file mode 100644 index 00000000..188cdb36 --- /dev/null +++ b/pkg/tracker/deployment/export_test.go @@ -0,0 +1,11 @@ +package deployment + +import appsv1 "k8s.io/api/apps/v1" + +func (d *Tracker) TestOnlyInjectReplicaSetModified(rs *appsv1.ReplicaSet) { + d.replicaSetModified <- rs +} + +func (d *Tracker) TestOnlyInjectReplicaSetDeleted(rs *appsv1.ReplicaSet) { + d.replicaSetDeleted <- rs +} diff --git a/pkg/tracker/deployment/harness_test.go b/pkg/tracker/deployment/harness_test.go new file mode 100644 index 00000000..e79a723e --- /dev/null +++ b/pkg/tracker/deployment/harness_test.go @@ -0,0 +1,326 @@ +package deployment_test + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + dynamicfake "k8s.io/client-go/dynamic/fake" + k8sfake "k8s.io/client-go/kubernetes/fake" + k8sscheme "k8s.io/client-go/kubernetes/scheme" + + "github.com/werf/kubedog/pkg/informer" + "github.com/werf/kubedog/pkg/tracker" + "github.com/werf/kubedog/pkg/tracker/deployment" +) + +const ( + testNamespace = "default" + testDeploymentName = "demo" +) + +var ( + gvrDeployments = schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} + gvrReplicaSets = schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "replicasets"} + + depUID = types.UID("dep-uid-1") + + errTimeout = errors.New("timeout") + errTrackDone = errors.New("Track() exited") +) + +func ptrTo[T any](v T) *T { return &v } + +type observation struct { + Kind string + Data any +} + +type harnessConfig struct { + dynamicSeed []runtime.Object +} + +type harness struct { + ctx context.Context + dynamic dynamic.Interface + tracker *deployment.Tracker + + events chan observation + backlog []observation + + done chan struct{} + wg sync.WaitGroup + trackErrMu sync.Mutex + trackErr error +} + +func newHarness(t *testing.T, cfg harnessConfig) *harness { + t.Helper() + + ctx, cancel := context.WithCancel(context.Background()) + + dynClient := dynamicfake.NewSimpleDynamicClient(k8sscheme.Scheme, cfg.dynamicSeed...) + kubeClient := k8sfake.NewSimpleClientset() + + watchErrCh := make(chan error, 100) + factory := informer.NewConcurrentInformerFactory(ctx.Done(), watchErrCh, dynClient, informer.ConcurrentInformerFactoryOptions{}) + + trk := deployment.NewTracker(testDeploymentName, testNamespace, kubeClient, factory, tracker.Options{IgnoreLogs: true}) + + h := &harness{ + ctx: ctx, + dynamic: dynClient, + tracker: trk, + events: make(chan observation, 4096), + done: make(chan struct{}), + } + + forwardChan(h, "Added", trk.Added) + forwardChan(h, "Ready", trk.Ready) + forwardChan(h, "Failed", trk.Failed) + forwardChan(h, "Status", trk.Status) + forwardChan(h, "EventMsg", trk.EventMsg) + forwardChan(h, "AddedReplicaSet", trk.AddedReplicaSet) + forwardChan(h, "AddedPod", trk.AddedPod) + forwardChan(h, "PodError", trk.PodError) + forwardChan(h, "WatchErr", watchErrCh) + h.wg.Add(1) + go func() { + defer h.wg.Done() + for { + select { + case <-trk.PodLogChunk: + case <-h.ctx.Done(): + return + } + } + }() + + go func() { + defer close(h.done) + defer func() { + if r := recover(); r != nil { + h.setTrackErr(fmt.Errorf("panic in Track(): %v", r)) + } + }() + h.setTrackErr(trk.Track(ctx)) + }() + + t.Cleanup(func() { + cancel() + select { + case <-h.done: + case <-time.After(10 * time.Second): + t.Error("Track() did not exit within 10s after cancel") + } + + forwardersDone := make(chan struct{}) + go func() { + h.wg.Wait() + close(forwardersDone) + }() + select { + case <-forwardersDone: + case <-time.After(5 * time.Second): + t.Error("forwarder goroutines did not exit within 5s after cancel") + } + }) + + return h +} + +func forwardChan[T any](h *harness, kind string, ch <-chan T) { + h.wg.Add(1) + go func() { + defer h.wg.Done() + for { + select { + case v := <-ch: + select { + case h.events <- observation{kind, v}: + case <-h.ctx.Done(): + return + } + case <-h.ctx.Done(): + return + } + } + }() +} + +func (h *harness) setTrackErr(err error) { + h.trackErrMu.Lock() + defer h.trackErrMu.Unlock() + if h.trackErr == nil { + h.trackErr = err + } +} + +func (h *harness) getTrackErr() error { + h.trackErrMu.Lock() + defer h.trackErrMu.Unlock() + return h.trackErr +} + +func (h *harness) waitFor(kind string, timeout time.Duration, pred func(observation) bool) (observation, error) { + for i, ev := range h.backlog { + if ev.Kind == kind && (pred == nil || pred(ev)) { + h.backlog = append(h.backlog[:i:i], h.backlog[i+1:]...) + return ev, nil + } + } + + deadline := time.NewTimer(timeout) + defer deadline.Stop() + for { + select { + case ev := <-h.events: + if ev.Kind == kind && (pred == nil || pred(ev)) { + return ev, nil + } + h.backlog = append(h.backlog, ev) + case <-h.done: + return observation{}, fmt.Errorf("%w (err=%v) while waiting for kind=%s", errTrackDone, h.getTrackErr(), kind) + case <-deadline.C: + return observation{}, fmt.Errorf("%w after %s waiting for kind=%s", errTimeout, timeout, kind) + } + } +} + +func (h *harness) waitForNone(kind string, quiet time.Duration, pred func(observation) bool) error { + ev, err := h.waitFor(kind, quiet, pred) + if err == nil { + return fmt.Errorf("expected no %s within %s, but got: %+v", kind, quiet, ev) + } + if errors.Is(err, errTimeout) { + return nil + } + return err +} + +func mustToUnstructured(t *testing.T, obj runtime.Object) *unstructured.Unstructured { + t.Helper() + m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + t.Fatalf("to unstructured: %v", err) + } + return &unstructured.Unstructured{Object: m} +} + +func (h *harness) createObject(t *testing.T, gvr schema.GroupVersionResource, obj runtime.Object) { + t.Helper() + if _, err := h.dynamic.Resource(gvr).Namespace(testNamespace).Create(h.ctx, mustToUnstructured(t, obj), metav1.CreateOptions{}); err != nil { + t.Fatalf("create %s: %v", gvr.Resource, err) + } +} + +func (h *harness) updateReplicaSet(t *testing.T, rs *appsv1.ReplicaSet) { + t.Helper() + cur, err := h.dynamic.Resource(gvrReplicaSets).Namespace(testNamespace).Get(h.ctx, rs.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("get current replicaset: %v", err) + } + rs.ResourceVersion = cur.GetResourceVersion() + if _, err := h.dynamic.Resource(gvrReplicaSets).Namespace(testNamespace).Update(h.ctx, mustToUnstructured(t, rs), metav1.UpdateOptions{}); err != nil { + t.Fatalf("update replicaset: %v", err) + } +} + +func (h *harness) updateDeployment(t *testing.T, dep *appsv1.Deployment) { + t.Helper() + cur, err := h.dynamic.Resource(gvrDeployments).Namespace(testNamespace).Get(h.ctx, dep.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("get current deployment: %v", err) + } + dep.ResourceVersion = cur.GetResourceVersion() + if _, err := h.dynamic.Resource(gvrDeployments).Namespace(testNamespace).Update(h.ctx, mustToUnstructured(t, dep), metav1.UpdateOptions{}); err != nil { + t.Fatalf("update deployment: %v", err) + } +} + +func (h *harness) deleteObject(t *testing.T, gvr schema.GroupVersionResource, name string) { + t.Helper() + if err := h.dynamic.Resource(gvr).Namespace(testNamespace).Delete(h.ctx, name, metav1.DeleteOptions{}); err != nil { + t.Fatalf("delete %s/%s: %v", gvr.Resource, name, err) + } +} + +func baseSelector() *metav1.LabelSelector { + return &metav1.LabelSelector{MatchLabels: map[string]string{"app": "demo"}} +} + +func basePodTemplate() corev1.PodTemplateSpec { + return corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "demo"}}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "c", Image: "busybox:1"}}, + }, + } +} + +func newNotReadyDeployment() *appsv1.Deployment { + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: testDeploymentName, Namespace: testNamespace, UID: depUID, Generation: 1}, + Spec: appsv1.DeploymentSpec{ + Replicas: ptrTo(int32(1)), + Selector: baseSelector(), + Template: basePodTemplate(), + }, + Status: appsv1.DeploymentStatus{ObservedGeneration: 1}, + } +} + +func newReplicaSet(name string, uid types.UID) *appsv1.ReplicaSet { + return &appsv1.ReplicaSet{ + TypeMeta: metav1.TypeMeta{Kind: "ReplicaSet", APIVersion: "apps/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: testNamespace, + UID: uid, + Labels: map[string]string{"app": "demo"}, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "apps/v1", Kind: "Deployment", Name: testDeploymentName, UID: depUID, Controller: ptrTo(true), + }}, + CreationTimestamp: metav1.Now(), + }, + Spec: appsv1.ReplicaSetSpec{ + Replicas: ptrTo(int32(1)), + Selector: baseSelector(), + Template: basePodTemplate(), + }, + } +} + +func withReplicaFailure(rs *appsv1.ReplicaSet, reason, marker string) *appsv1.ReplicaSet { + rs.Status.Conditions = []appsv1.ReplicaSetCondition{{ + Type: appsv1.ReplicaSetReplicaFailure, + Status: corev1.ConditionTrue, + Reason: reason, + Message: fmt.Sprintf("pods %q is forbidden: exceeded quota: compute-quota [%s]", rs.Name, marker), + LastTransitionTime: metav1.Now(), + }} + return rs +} + +func failedWithMarker(marker string) func(observation) bool { + return func(ev observation) bool { + status, ok := ev.Data.(deployment.DeploymentStatus) + if !ok { + return false + } + return status.IsFailed && strings.Contains(status.FailedReason, marker) + } +} diff --git a/pkg/tracker/deployment/tracker.go b/pkg/tracker/deployment/tracker.go index bb575c11..97e15ee5 100644 --- a/pkg/tracker/deployment/tracker.go +++ b/pkg/tracker/deployment/tracker.go @@ -12,6 +12,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" @@ -25,6 +26,11 @@ import ( "github.com/werf/kubedog/pkg/utils" ) +// replicaSetFailedCreateReason is the reason the ReplicaSet controller sets on +// the ReplicaSetReplicaFailure condition when it cannot create pods, e.g. +// because a namespace quota is exceeded. +const replicaSetFailedCreateReason = "FailedCreate" + type ReplicaSetAddedReport struct { ReplicaSet replicaset.ReplicaSet DeploymentStatus DeploymentStatus @@ -47,11 +53,12 @@ type Tracker struct { Conditions []string NewReplicaSetName string - knownReplicaSets map[string]*appsv1.ReplicaSet - lastObject *appsv1.Deployment - failedReason string - podStatuses map[string]pod.PodStatus - rsNameByPod map[string]string + knownReplicaSets map[string]*appsv1.ReplicaSet + lastObject *appsv1.Deployment + failedReason string + reportedReplicaSetFailures map[types.UID]string + podStatuses map[string]pod.PodStatus + rsNameByPod map[string]string ignoreLogs bool ignoreReadinessProbeFailsByContainerName map[string]time.Duration @@ -109,9 +116,10 @@ func NewTracker(name, namespace string, kube kubernetes.Interface, informerFacto PodLogChunk: make(chan *replicaset.ReplicaSetPodLogChunk, 1000), PodError: make(chan PodErrorReport), - knownReplicaSets: make(map[string]*appsv1.ReplicaSet), - podStatuses: make(map[string]pod.PodStatus), - rsNameByPod: make(map[string]string), + knownReplicaSets: make(map[string]*appsv1.ReplicaSet), + reportedReplicaSetFailures: make(map[types.UID]string), + podStatuses: make(map[string]pod.PodStatus), + rsNameByPod: make(map[string]string), ignoreLogs: opts.IgnoreLogs, ignoreReadinessProbeFailsByContainerName: opts.IgnoreReadinessProbeFailsByContainerName, @@ -155,16 +163,23 @@ func (d *Tracker) Track(ctx context.Context) (err error) { return err } defer cleanupFn() + if err := d.handleNewReplicaSetFailure(); err != nil { + return err + } case object := <-d.resourceModified: cleanupFn, err := d.handleDeploymentState(ctx, object) if err != nil { return err } defer cleanupFn() + if err := d.handleNewReplicaSetFailure(); err != nil { + return err + } case <-d.resourceDeleted: d.State = tracker.ResourceDeleted d.lastObject = nil d.knownReplicaSets = make(map[string]*appsv1.ReplicaSet) + d.reportedReplicaSetFailures = make(map[types.UID]string) d.podStatuses = make(map[string]pod.PodStatus) d.rsNameByPod = make(map[string]string) d.TrackedPodsNames = nil @@ -173,21 +188,17 @@ func (d *Tracker) Track(ctx context.Context) (err error) { case failure := <-d.resourceFailed: switch failure := failure.(type) { case string: - d.State = tracker.ResourceFailed - d.failedReason = failure + // Failures are reported by the events informer, which is + // stopped only when Track returns. With no live object there is + // no status to report the failure on, and the failure itself + // belongs to an already deleted object. + if d.lastObject == nil { + break + } - var status DeploymentStatus - if d.lastObject != nil { - d.StatusGeneration++ - newPodsNames, err := d.getNewPodsNames() - if err != nil { - return err - } - status = NewDeploymentStatus(d.lastObject, d.StatusGeneration, d.State == tracker.ResourceFailed, d.failedReason, d.podStatuses, newPodsNames) - } else { - status = DeploymentStatus{IsFailed: true, FailedReason: failure} + if err := d.handleResourceFailure(failure); err != nil { + return err } - d.Failed <- status default: panic(fmt.Errorf("unexpected type %T", failure)) } @@ -200,9 +211,6 @@ func (d *Tracker) Track(ctx context.Context) (err error) { if err != nil { return err } - if len(d.knownReplicaSets) == 0 { - rsNew = true - } d.StatusGeneration++ newPodsNames, err := d.getNewPodsNames() @@ -218,13 +226,34 @@ func (d *Tracker) Track(ctx context.Context) (err error) { }, DeploymentStatus: status, } + + if err := d.handleNewReplicaSetFailure(); err != nil { + return err + } } case rs := <-d.replicaSetModified: - d.knownReplicaSets[rs.Name] = rs + // Additions, modifications and deletions are delivered over + // separate channels, so a modification of an already replaced + // ReplicaSet may be observed after addition of its same-named + // successor: applying it would resurrect the dead incarnation. + if known, found := d.knownReplicaSets[rs.Name]; !found || known.UID == rs.UID { + d.knownReplicaSets[rs.Name] = rs + + if d.lastObject != nil { + if err := d.handleNewReplicaSetFailure(); err != nil { + return err + } + } + } case rs := <-d.replicaSetDeleted: - delete(d.knownReplicaSets, rs.Name) + // Same reordering hazard as for modifications: dropping the + // ReplicaSet by name alone would lose the live successor. + if known, found := d.knownReplicaSets[rs.Name]; found && known.UID == rs.UID { + delete(d.knownReplicaSets, rs.Name) + } + delete(d.reportedReplicaSetFailures, rs.UID) case pod := <-d.podAddedRelay: rsName := utils.GetPodReplicaSetName(pod) @@ -661,3 +690,74 @@ func (d *Tracker) runEventsInformer(ctx context.Context, resource interface{}) ( eventInformer.WithChannels(d.EventMsg, d.resourceFailed, d.errors) return eventInformer.Run(ctx) } + +func (d *Tracker) handleResourceFailure(reason string) error { + d.State = tracker.ResourceFailed + d.failedReason = reason + + d.StatusGeneration++ + newPodsNames, err := d.getNewPodsNames() + if err != nil { + return err + } + d.Failed <- NewDeploymentStatus(d.lastObject, d.StatusGeneration, d.State == tracker.ResourceFailed, d.failedReason, d.podStatuses, newPodsNames) + + return nil +} + +// handleNewReplicaSetFailure fails the tracking as soon as the ReplicaSet the +// Deployment is currently rolling out cannot create its pods. +// +// ReplicaSets of previous rollouts are skipped: their failures must not fail the +// current rollout. A failure is reported once per ReplicaSet incarnation and +// reason and is rearmed once the ReplicaSet controller clears the condition, so +// a ReplicaSet that recovered and then failed again is reported anew. +func (d *Tracker) handleNewReplicaSetFailure() error { + if d.lastObject == nil { + return nil + } + + newReplicaSet, err := utils.FindNewReplicaSet(d.lastObject, lo.Values(d.knownReplicaSets)) + if err != nil { + return err + } + if newReplicaSet == nil { + return nil + } + + failure, failed := replicaSetFailure(newReplicaSet) + if !failed { + delete(d.reportedReplicaSetFailures, newReplicaSet.UID) + return nil + } + + // The message carries volatile details, e.g. the quota usage at the moment the + // pod was rejected, so only the reason takes part in the deduplication. + if d.reportedReplicaSetFailures[newReplicaSet.UID] == failure.Reason { + return nil + } + d.reportedReplicaSetFailures[newReplicaSet.UID] = failure.Reason + + return d.handleResourceFailure(fmt.Sprintf("%s: %s", failure.Reason, failure.Message)) +} + +// replicaSetFailure returns the condition set by the ReplicaSet controller when it +// failed to create pods for the ReplicaSet. +// +// Unlike a Warning event, the ReplicaSetReplicaFailure condition is durable state: +// the controller sets it while pod creation keeps failing and removes it as soon as +// creation succeeds, so no event baseline is needed. FailedDelete is not fatal for a +// rollout and is ignored here, just like the events informer ignores it. +func replicaSetFailure(rs *appsv1.ReplicaSet) (appsv1.ReplicaSetCondition, bool) { + for _, condition := range rs.Status.Conditions { + if condition.Type != appsv1.ReplicaSetReplicaFailure || + condition.Status != corev1.ConditionTrue || + condition.Reason != replicaSetFailedCreateReason { + continue + } + + return condition, true + } + + return appsv1.ReplicaSetCondition{}, false +} diff --git a/pkg/tracker/deployment/tracker_replicaset_test.go b/pkg/tracker/deployment/tracker_replicaset_test.go new file mode 100644 index 00000000..56b08642 --- /dev/null +++ b/pkg/tracker/deployment/tracker_replicaset_test.go @@ -0,0 +1,189 @@ +package deployment_test + +import ( + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + + "github.com/werf/kubedog/pkg/tracker/deployment" +) + +func TestTrack_ForeignReplicaSetCannotMaskOwnedReplicaFailure(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + createdAt := time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC) + foreign := withReplicaSetCreationTimestamp( + withReplicaSetControllerUID(newReplicaSet("foreign-111", "foreign-rs-uid"), "foreign-deployment-uid"), + createdAt, + ) + h.createObject(t, gvrReplicaSets, foreign) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedReplicaSetNamed(foreign.Name)); err != nil { + t.Fatalf("wait for foreign AddedReplicaSet: %v", err) + } + + const marker = "marker-owned-not-masked" + owned := withReplicaFailure( + withReplicaSetCreationTimestamp(newReplicaSet("demo-111", "owned-rs-uid"), createdAt.Add(time.Minute)), + "FailedCreate", + marker, + ) + h.createObject(t, gvrReplicaSets, owned) + + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(marker)); err != nil { + t.Fatalf("wait for owned ReplicaSet failure: %v", err) + } +} + +func TestTrack_IgnoresForeignAndOrphanReplicaSetFailures(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + createdAt := time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC) + const foreignMarker = "marker-foreign-failure" + foreign := withReplicaFailure( + withReplicaSetCreationTimestamp( + withReplicaSetControllerUID(newReplicaSet("foreign-111", "foreign-rs-uid"), "foreign-deployment-uid"), + createdAt, + ), + "FailedCreate", + foreignMarker, + ) + h.createObject(t, gvrReplicaSets, foreign) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedReplicaSetNamed(foreign.Name)); err != nil { + t.Fatalf("wait for foreign AddedReplicaSet: %v", err) + } + + owned := withReplicaSetCreationTimestamp(newReplicaSet("demo-111", "owned-rs-uid"), createdAt.Add(2*time.Minute)) + h.createObject(t, gvrReplicaSets, owned) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedReplicaSetNamed(owned.Name)); err != nil { + t.Fatalf("wait for owned AddedReplicaSet: %v", err) + } + + if err := h.waitForNone("Failed", 2500*time.Millisecond, failedWithMarker(foreignMarker)); err != nil { + t.Fatalf("foreign ReplicaSet must not fail the rollout: %v", err) + } + + h.deleteObject(t, gvrReplicaSets, foreign.Name) + + const orphanMarker = "marker-orphan-failure" + orphan := withReplicaFailure( + withReplicaSetCreationTimestamp(withoutReplicaSetOwner(newReplicaSet("orphan-111", "orphan-rs-uid")), createdAt.Add(time.Minute)), + "FailedCreate", + orphanMarker, + ) + h.createObject(t, gvrReplicaSets, orphan) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedReplicaSetNamed(orphan.Name)); err != nil { + t.Fatalf("wait for orphan AddedReplicaSet: %v", err) + } + + if err := h.waitForNone("Failed", 2500*time.Millisecond, failedWithMarker(orphanMarker)); err != nil { + t.Fatalf("orphan ReplicaSet must not fail the rollout: %v", err) + } +} + +func TestTrack_DoesNotReportMessageOnlyReplicaFailureChanges(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + rs := newReplicaSet("demo-111", "rs-uid-1") + h.createObject(t, gvrReplicaSets, rs) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedNewReplicaSet(rs.Name)); err != nil { + t.Fatalf("wait for AddedReplicaSet: %v", err) + } + + const firstMarker = "marker-message-first" + failed := withReplicaFailure(rs.DeepCopy(), "FailedCreate", firstMarker) + h.updateReplicaSet(t, failed) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(firstMarker)); err != nil { + t.Fatalf("wait for first Failed: %v", err) + } + + const changedMarker = "marker-message-changed" + messageChanged := failed.DeepCopy() + messageChanged.Status.Conditions[0].Message = "updated quota details [" + changedMarker + "]" + h.updateReplicaSet(t, messageChanged) + + if err := h.waitForNone("Failed", 2500*time.Millisecond, failedWithMarker(changedMarker)); err != nil { + t.Fatalf("message-only failure change must not be reported again: %v", err) + } +} + +func TestTrack_IgnoresStaleEventsForRecreatedReplicaSet(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + rsOld := newReplicaSet("demo-111", "rs-uid-old") + h.createObject(t, gvrReplicaSets, rsOld) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedNewReplicaSet(rsOld.Name)); err != nil { + t.Fatalf("wait for old AddedReplicaSet: %v", err) + } + h.deleteObject(t, gvrReplicaSets, rsOld.Name) + + const currentMarker = "marker-current-incarnation" + rsCurrent := newReplicaSet("demo-111", "rs-uid-current") + h.createObject(t, gvrReplicaSets, withReplicaFailure(rsCurrent.DeepCopy(), "FailedCreate", currentMarker)) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(currentMarker)); err != nil { + t.Fatalf("wait for current ReplicaSet failure: %v", err) + } + + const staleMarker = "marker-stale-incarnation" + h.tracker.TestOnlyInjectReplicaSetModified(withReplicaFailure(rsOld.DeepCopy(), "FailedCreate", staleMarker)) + h.tracker.TestOnlyInjectReplicaSetDeleted(rsOld.DeepCopy()) + if err := h.waitForNone("Failed", 2500*time.Millisecond, failedWithMarker(staleMarker)); err != nil { + t.Fatalf("stale ReplicaSet events must not replace the current incarnation: %v", err) + } + + h.updateReplicaSet(t, rsCurrent.DeepCopy()) + const recurringMarker = "marker-current-recurring" + h.updateReplicaSet(t, withReplicaFailure(rsCurrent.DeepCopy(), "FailedCreate", recurringMarker)) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(recurringMarker)); err != nil { + t.Fatalf("current ReplicaSet must remain tracked after stale events: %v", err) + } +} + +func addedReplicaSetNamed(name string) func(observation) bool { + return func(ev observation) bool { + report, ok := ev.Data.(deployment.ReplicaSetAddedReport) + return ok && report.ReplicaSet.Name == name + } +} + +func withReplicaSetControllerUID(rs *appsv1.ReplicaSet, uid types.UID) *appsv1.ReplicaSet { + rs.OwnerReferences[0].UID = uid + return rs +} + +func withoutReplicaSetOwner(rs *appsv1.ReplicaSet) *appsv1.ReplicaSet { + rs.OwnerReferences = nil + return rs +} + +func withReplicaSetCreationTimestamp(rs *appsv1.ReplicaSet, createdAt time.Time) *appsv1.ReplicaSet { + rs.CreationTimestamp = metav1.NewTime(createdAt) + return rs +} diff --git a/pkg/tracker/deployment/tracker_test.go b/pkg/tracker/deployment/tracker_test.go new file mode 100644 index 00000000..612e3d17 --- /dev/null +++ b/pkg/tracker/deployment/tracker_test.go @@ -0,0 +1,273 @@ +package deployment_test + +import ( + "testing" + "time" + + "k8s.io/apimachinery/pkg/runtime" + + "github.com/werf/kubedog/pkg/tracker/deployment" +) + +func TestTrack_FailsOnNewReplicaSetReplicaFailure(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + const marker = "marker-happy-path" + h.createObject(t, gvrReplicaSets, withReplicaFailure(newReplicaSet("demo-111", "rs-uid-1"), "FailedCreate", marker)) + + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(marker)); err != nil { + t.Fatalf("wait for Failed: %v", err) + } +} + +func TestTrack_FailsOnReplicaFailureAppearingAfterReplicaSetAdded(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + rs := newReplicaSet("demo-111", "rs-uid-1") + h.createObject(t, gvrReplicaSets, rs) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedNewReplicaSet(rs.Name)); err != nil { + t.Fatalf("wait for AddedReplicaSet: %v", err) + } + + const marker = "marker-condition-after-add" + h.updateReplicaSet(t, withReplicaFailure(rs.DeepCopy(), "FailedCreate", marker)) + + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(marker)); err != nil { + t.Fatalf("wait for Failed: %v", err) + } +} + +// Models the apply-then-track gap: the ReplicaSet already carries the failure +// condition when tracking starts, so unlike a Warning event there is no history +// to replay and no start boundary to reason about. +func TestTrack_FailsOnPreexistingReplicaFailure(t *testing.T) { + const marker = "marker-preexisting" + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{ + newNotReadyDeployment(), + withReplicaFailure(newReplicaSet("demo-111", "rs-uid-1"), "FailedCreate", marker), + }, + }) + + if _, err := h.waitFor("Failed", 15*time.Second, failedWithMarker(marker)); err != nil { + t.Fatalf("wait for Failed: %v", err) + } +} + +func TestTrack_IgnoresOldReplicaSetReplicaFailure(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + rsOld := newReplicaSet("demo-old", "rs-old") + h.createObject(t, gvrReplicaSets, rsOld) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedNewReplicaSet(rsOld.Name)); err != nil { + t.Fatalf("wait for AddedReplicaSet(old): %v", err) + } + + updatedDep := newNotReadyDeployment() + updatedDep.Generation = 2 + updatedDep.Status.ObservedGeneration = 2 + updatedDep.Spec.Template.Spec.Containers[0].Image = "busybox:2" + h.updateDeployment(t, updatedDep) + if _, err := h.waitFor("Status", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Status after deployment update: %v", err) + } + + rsNew := newReplicaSet("demo-new", "rs-new") + rsNew.Spec.Template.Spec.Containers[0].Image = "busybox:2" + h.createObject(t, gvrReplicaSets, rsNew) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedNewReplicaSet(rsNew.Name)); err != nil { + t.Fatalf("wait for AddedReplicaSet(new): %v", err) + } + + const markerOld = "marker-old-rollout" + h.updateReplicaSet(t, withReplicaFailure(rsOld.DeepCopy(), "FailedCreate", markerOld)) + if err := h.waitForNone("Failed", 2500*time.Millisecond, failedWithMarker(markerOld)); err != nil { + t.Fatalf("previous rollout replicaset must not fail the current one: %v", err) + } + + const markerNew = "marker-new-rollout" + h.updateReplicaSet(t, withReplicaFailure(rsNew.DeepCopy(), "FailedCreate", markerNew)) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(markerNew)); err != nil { + t.Fatalf("wait for Failed: %v", err) + } +} + +func TestTrack_ReportsReplicaFailureOncePerIncarnation(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + rs := newReplicaSet("demo-111", "rs-uid-1") + h.createObject(t, gvrReplicaSets, rs) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedNewReplicaSet(rs.Name)); err != nil { + t.Fatalf("wait for AddedReplicaSet: %v", err) + } + + const marker = "marker-dedupe" + failed := withReplicaFailure(rs.DeepCopy(), "FailedCreate", marker) + h.updateReplicaSet(t, failed) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(marker)); err != nil { + t.Fatalf("wait for Failed: %v", err) + } + + stillFailing := failed.DeepCopy() + stillFailing.Status.ObservedGeneration = 2 + h.updateReplicaSet(t, stillFailing) + if err := h.waitForNone("Failed", 2500*time.Millisecond, failedWithMarker(marker)); err != nil { + t.Fatalf("unchanged failure condition must not be reported twice: %v", err) + } +} + +func TestTrack_ReportsReplicaFailureAgainAfterConditionCleared(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + rs := newReplicaSet("demo-111", "rs-uid-1") + h.createObject(t, gvrReplicaSets, rs) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedNewReplicaSet(rs.Name)); err != nil { + t.Fatalf("wait for AddedReplicaSet: %v", err) + } + + const marker = "marker-rearm" + h.updateReplicaSet(t, withReplicaFailure(rs.DeepCopy(), "FailedCreate", marker)) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(marker)); err != nil { + t.Fatalf("wait for first Failed: %v", err) + } + + h.updateReplicaSet(t, rs.DeepCopy()) + h.updateReplicaSet(t, withReplicaFailure(rs.DeepCopy(), "FailedCreate", marker)) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(marker)); err != nil { + t.Fatalf("failure recurring after recovery must be reported again: %v", err) + } +} + +func TestTrack_IgnoresFailedDeleteReplicaFailure(t *testing.T) { + const marker = "marker-failed-delete" + rs := withReplicaFailure(newReplicaSet("demo-111", "rs-uid-1"), "FailedDelete", marker) + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment(), rs}, + }) + + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedNewReplicaSet(rs.Name)); err != nil { + t.Fatalf("wait for AddedReplicaSet: %v", err) + } + + if err := h.waitForNone("Failed", 2500*time.Millisecond, failedWithMarker(marker)); err != nil { + t.Fatalf("FailedDelete must not fail the rollout: %v", err) + } +} + +func TestTrack_FailsOnReplicaFailureWithEventsDisabled(t *testing.T) { + t.Setenv("KUBEDOG_DISABLE_EVENTS", "1") + + const marker = "marker-events-disabled" + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{ + newNotReadyDeployment(), + withReplicaFailure(newReplicaSet("demo-111", "rs-uid-1"), "FailedCreate", marker), + }, + }) + + if _, err := h.waitFor("Failed", 15*time.Second, failedWithMarker(marker)); err != nil { + t.Fatalf("replica failure detection must not depend on events: %v", err) + } +} + +func TestTrack_FailsOnRecreatedReplicaSetWithSameNameNewUID(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + rsOld := newReplicaSet("demo-111", "rs-uid-old") + h.createObject(t, gvrReplicaSets, rsOld) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedNewReplicaSet(rsOld.Name)); err != nil { + t.Fatalf("wait for AddedReplicaSet(old): %v", err) + } + + h.deleteObject(t, gvrReplicaSets, rsOld.Name) + + const marker = "marker-recreated-rs" + h.createObject(t, gvrReplicaSets, withReplicaFailure(newReplicaSet("demo-111", "rs-uid-new"), "FailedCreate", marker)) + + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(marker)); err != nil { + t.Fatalf("wait for Failed: %v", err) + } +} + +func TestTrack_StopsReportingReplicaFailuresAfterDeploymentDeletion(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + rs := newReplicaSet("demo-111", "rs-uid-1") + h.createObject(t, gvrReplicaSets, rs) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedNewReplicaSet(rs.Name)); err != nil { + t.Fatalf("wait for AddedReplicaSet: %v", err) + } + + h.deleteObject(t, gvrDeployments, testDeploymentName) + if _, err := h.waitFor("Status", 10*time.Second, deletionResetStatus); err != nil { + t.Fatalf("wait for deletion status: %v", err) + } + + const marker = "marker-after-deletion" + h.updateReplicaSet(t, withReplicaFailure(rs.DeepCopy(), "FailedCreate", marker)) + if err := h.waitForNone("Failed", 2500*time.Millisecond, failedWithMarker(marker)); err != nil { + t.Fatalf("replica failures must not be reported for a deleted deployment: %v", err) + } +} + +func addedNewReplicaSet(name string) func(observation) bool { + return func(ev observation) bool { + report, ok := ev.Data.(deployment.ReplicaSetAddedReport) + if !ok { + return false + } + return report.ReplicaSet.Name == name && report.ReplicaSet.IsNew + } +} + +// The zero-value status emitted on resource deletion is the only Status with +// StatusGeneration == 0, since NewDeploymentStatus always increments it first. +func deletionResetStatus(ev observation) bool { + status, ok := ev.Data.(deployment.DeploymentStatus) + if !ok { + return false + } + return status.StatusGeneration == 0 && !status.IsFailed +} diff --git a/pkg/utils/deployment_utils.go b/pkg/utils/deployment_utils.go index 836c00c9..01fac59e 100644 --- a/pkg/utils/deployment_utils.go +++ b/pkg/utils/deployment_utils.go @@ -7,23 +7,46 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// FindNewReplicaSet returns the ReplicaSet the Deployment is currently rolling out. +// +// Only ReplicaSets controlled by the Deployment are considered: label selectors are +// not unique, so a ReplicaSet of another Deployment or of a previous incarnation of +// this one may match both the labels and the pod template. func FindNewReplicaSet(deployment *appsv1.Deployment, rsList []*appsv1.ReplicaSet) (*appsv1.ReplicaSet, error) { newRSTemplate := GetNewReplicaSetTemplate(deployment) - sort.Sort(ReplicaSetsByCreationTimestamp(rsList)) - for i := range rsList { - if EqualIgnoreHash(rsList[i].Spec.Template, newRSTemplate) { + ownedRSList := controlledReplicaSets(deployment, rsList) + sort.Sort(ReplicaSetsByCreationTimestamp(ownedRSList)) + for i := range ownedRSList { + if EqualIgnoreHash(ownedRSList[i].Spec.Template, newRSTemplate) { // In rare cases, such as after cluster upgrades, Deployment may end up with // having more than one new ReplicaSets that have the same template as its template, // see https://github.com/kubernetes/kubernetes/issues/40415 // We deterministically choose the oldest new ReplicaSet. - return rsList[i], nil + return ownedRSList[i], nil } } return nil, nil } +// controlledReplicaSets keeps only the ReplicaSets whose controller reference points +// at the given Deployment. A ReplicaSet without a controller reference is not +// adopted, just like the Deployment controller itself does not adopt one. +func controlledReplicaSets(deployment *appsv1.Deployment, rsList []*appsv1.ReplicaSet) []*appsv1.ReplicaSet { + ownedRSList := make([]*appsv1.ReplicaSet, 0, len(rsList)) + for _, rs := range rsList { + if rs == nil || !metav1.IsControlledBy(rs, deployment) { + continue + } + + ownedRSList = append(ownedRSList, rs) + } + + return ownedRSList +} + func IsReplicaSetNew(deployment *appsv1.Deployment, rsMap map[string]*appsv1.ReplicaSet, rsName string) (bool, error) { rsList := []*appsv1.ReplicaSet{} for _, rs := range rsMap { From 38643e953de5acddd3d5d06af8c68971ac1dda8c Mon Sep 17 00:00:00 2001 From: Alexey Gorovenko Date: Sat, 25 Jul 2026 17:20:31 +0300 Subject: [PATCH 2/5] fix: handle ReplicaSet failure transitions consistently Force Failed payloads to set IsFailed and clear IsReady, while ignoring lagging FailedCreate conditions after the Deployment becomes ready. Wait for the complete initial ReplicaSet snapshot before classifying failures. Preserve informer ordering with unbuffered, cancellation-aware sends, and handle same-name UID replacement and selector leave/reentry without losing a failure transition. Classify invalid specs and quota failures that cannot recover through the rollout as fatal. Keep rollout-recoverable quota failures and event-sourced failures counted against the allowed-failures budget. Latch the first classification until the condition clears so a stale FailedCreate condition cannot escalate after old pods terminate, and account for TerminatingReplicas when checking whether the rollout can release quota. Deduplicate failures per ReplicaSet incarnation, rearm reporting after condition recovery or Deployment readiness, ignore stale events after deletion, and re-evaluate a template-equivalent successor when deletion promotes it to the new ReplicaSet. Fail the readiness task on a failed resource state only for fail modes that are supposed to fail the deploy process, keeping IgnoreAndContinueDeployProcess unaffected. Signed-off-by: Alexey Gorovenko (cherry picked from commit 78ec367071254723c77507e6076af47850bfe0b1) --- pkg/dyntracker/dynamic_readiness_tracker.go | 3 + ...mic_readiness_tracker_failure_mode_test.go | 87 +++++ ...c_readiness_tracker_replicafailure_test.go | 198 +++++++++++ .../statestore/readiness_task_state.go | 42 +-- pkg/tracker/deployment/export_test.go | 16 + pkg/tracker/deployment/failure.go | 37 ++ pkg/tracker/deployment/harness_test.go | 36 +- pkg/tracker/deployment/replicaset_failure.go | 164 +++++++++ .../deployment/replicaset_failure_test.go | 52 +++ pkg/tracker/deployment/status.go | 29 +- pkg/tracker/deployment/tracker.go | 223 ++++++------ .../deployment/tracker_failure_test.go | 49 +++ .../deployment/tracker_quota_recovery_test.go | 144 ++++++++ .../tracker_replicaset_recovery_test.go | 85 +++++ .../deployment/tracker_replicaset_test.go | 320 ++++++++++++++++++ pkg/tracker/deployment/tracker_test.go | 17 + pkg/tracker/replicaset/informer.go | 133 +++++--- .../replicaset/informer_lifecycle_test.go | 217 ++++++++++++ pkg/tracker/replicaset/informer_test.go | 115 +++++++ pkg/utils/deployment_utils.go | 12 +- 20 files changed, 1792 insertions(+), 187 deletions(-) create mode 100644 pkg/dyntracker/dynamic_readiness_tracker_failure_mode_test.go create mode 100644 pkg/dyntracker/dynamic_readiness_tracker_replicafailure_test.go create mode 100644 pkg/tracker/deployment/failure.go create mode 100644 pkg/tracker/deployment/replicaset_failure.go create mode 100644 pkg/tracker/deployment/replicaset_failure_test.go create mode 100644 pkg/tracker/deployment/tracker_failure_test.go create mode 100644 pkg/tracker/deployment/tracker_quota_recovery_test.go create mode 100644 pkg/tracker/deployment/tracker_replicaset_recovery_test.go create mode 100644 pkg/tracker/replicaset/informer_lifecycle_test.go create mode 100644 pkg/tracker/replicaset/informer_test.go diff --git a/pkg/dyntracker/dynamic_readiness_tracker.go b/pkg/dyntracker/dynamic_readiness_tracker.go index 9b4bdde1..6d807426 100644 --- a/pkg/dyntracker/dynamic_readiness_tracker.go +++ b/pkg/dyntracker/dynamic_readiness_tracker.go @@ -1185,6 +1185,9 @@ func (t *DynamicReadinessTracker) handleDeploymentStatus(status *deployment.Depl if status.IsFailed { taskState.ResourceState(taskState.Name(), taskState.Namespace(), taskState.GroupVersionKind()).RWTransaction(func(rs *statestore.ResourceState) { rs.AddError(errors.New(status.FailedReason), "", time.Now()) + if status.FailureMode == deployment.FailureModeFatal { + rs.SetStatus(statestore.ResourceStatusFailed) + } }) } } diff --git a/pkg/dyntracker/dynamic_readiness_tracker_failure_mode_test.go b/pkg/dyntracker/dynamic_readiness_tracker_failure_mode_test.go new file mode 100644 index 00000000..3979c457 --- /dev/null +++ b/pkg/dyntracker/dynamic_readiness_tracker_failure_mode_test.go @@ -0,0 +1,87 @@ +package dyntracker + +import ( + "testing" + + "k8s.io/apimachinery/pkg/runtime/schema" + + "github.com/werf/kubedog/pkg/dyntracker/statestore" + "github.com/werf/kubedog/pkg/tracker/deployment" +) + +func TestHandleDeploymentStatus_KeepsCountedFailureWithinAllowance(t *testing.T) { + taskState := statestore.NewReadinessTaskState( + "demo", + "default", + schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, + statestore.ReadinessTaskStateOptions{TotalAllowFailuresCount: 1}, + ) + status := deployment.DeploymentStatus{ + IsFailed: true, + FailedReason: "counted event failure", + FailureMode: deployment.FailureModeCounted, + } + + (&DynamicReadinessTracker{}).handleDeploymentStatus(&status, taskState) + + if got := taskState.Status(); got != statestore.ReadinessTaskStatusProgressing { + t.Fatalf("counted failure within allowance must keep tracking progressing, got %s", got) + } +} + +func TestHandleDeploymentStatus_FailsOnFatalFailureInLegacyFailMode(t *testing.T) { + taskState := statestore.NewReadinessTaskState( + "demo", + "default", + schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, + statestore.ReadinessTaskStateOptions{ + FailMode: statestore.LegacyHopeUntilEndOfDeployProcess, + TotalAllowFailuresCount: 1, + }, + ) + status := deployment.DeploymentStatus{ + IsFailed: true, + FailedReason: `pods "demo-111" is forbidden: exceeded quota: compute-quota`, + FailureMode: deployment.FailureModeFatal, + } + + (&DynamicReadinessTracker{}).handleDeploymentStatus(&status, taskState) + + if got := taskState.Status(); got != statestore.ReadinessTaskStatusFailed { + t.Fatalf("fatal failure must fail tracking regardless of the allowance, got %s", got) + } +} + +func TestHandleDeploymentStatus_KeepsFatalFailureIgnoredInIgnoreFailMode(t *testing.T) { + taskState := statestore.NewReadinessTaskState( + "demo", + "default", + schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}, + statestore.ReadinessTaskStateOptions{ + FailMode: statestore.IgnoreAndContinueDeployProcess, + TotalAllowFailuresCount: 0, + }, + ) + status := deployment.DeploymentStatus{ + IsFailed: true, + FailedReason: `pods "demo-111" is forbidden: exceeded quota: compute-quota`, + FailureMode: deployment.FailureModeFatal, + } + + (&DynamicReadinessTracker{}).handleDeploymentStatus(&status, taskState) + + resourceState := taskState.ResourceState(taskState.Name(), taskState.Namespace(), taskState.GroupVersionKind()) + + var resourceStatus statestore.ResourceStatus + resourceState.RTransaction(func(rs *statestore.ResourceState) { + resourceStatus = rs.Status() + }) + + if resourceStatus != statestore.ResourceStatusFailed { + t.Fatalf("fatal failure must be recorded on the resource state, got %s", resourceStatus) + } + + if got := taskState.Status(); got != statestore.ReadinessTaskStatusProgressing { + t.Fatalf("ignore fail mode must keep tracking progressing on a fatal failure, got %s", got) + } +} diff --git a/pkg/dyntracker/dynamic_readiness_tracker_replicafailure_test.go b/pkg/dyntracker/dynamic_readiness_tracker_replicafailure_test.go new file mode 100644 index 00000000..c393e8dc --- /dev/null +++ b/pkg/dyntracker/dynamic_readiness_tracker_replicafailure_test.go @@ -0,0 +1,198 @@ +package dyntracker_test + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + k8sfake "k8s.io/client-go/kubernetes/fake" + k8sscheme "k8s.io/client-go/kubernetes/scheme" + + "github.com/werf/kubedog/pkg/dyntracker" + "github.com/werf/kubedog/pkg/dyntracker/logstore" + "github.com/werf/kubedog/pkg/dyntracker/statestore" + "github.com/werf/kubedog/pkg/dyntracker/util" + "github.com/werf/kubedog/pkg/informer" +) + +var replicaFailureDeploymentGVK = schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"} + +func TestDynamicReadinessTracker_SingleDurableReplicaFailure_TerminatesPromptly(t *testing.T) { + const boundedTimeout = 8 * time.Second + + tests := []struct { + name string + allowFailures int + }{ + {name: "allowance below the single failure", allowFailures: 0}, + {name: "allowance equal to the single failure", allowFailures: 1}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dynClient := dynamicfake.NewSimpleDynamicClient(k8sscheme.Scheme, + replicaFailureOneReplicaDeployment(), replicaFailureFailedCreateReplicaSet()) + kubeClient := k8sfake.NewSimpleClientset() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + watchErrCh := make(chan error, 100) + factory := informer.NewConcurrentInformerFactory(ctx.Done(), watchErrCh, dynClient, informer.ConcurrentInformerFactoryOptions{}) + + taskState := util.NewConcurrent(statestore.NewReadinessTaskState( + "demo", "default", replicaFailureDeploymentGVK, + statestore.ReadinessTaskStateOptions{TotalAllowFailuresCount: tt.allowFailures}, + )) + logStore := util.NewConcurrent(logstore.NewLogStore()) + + rt, err := dyntracker.NewDynamicReadinessTracker( + ctx, taskState, logStore, factory, kubeClient, dynClient, nil, replicaFailureStubRESTMapper{}, + dyntracker.DynamicReadinessTrackerOptions{ + Timeout: boundedTimeout, + CaseInsensitiveGVKMatching: true, + IgnoreLogs: true, + }, + ) + if err != nil { + t.Fatalf("NewDynamicReadinessTracker: %v", err) + } + + start := time.Now() + trackErr := rt.Track(ctx) + elapsed := time.Since(start) + + if trackErr == nil { + t.Fatalf("expected Track() to fail once the durable ReplicaFailure is observed, got nil error after %s", elapsed) + } + if !strings.Contains(trackErr.Error(), "readiness failed") { + t.Fatalf("expected a readiness-failed error, got: %v (after %s)", trackErr, elapsed) + } + if elapsed >= boundedTimeout/2 { + t.Fatalf("expected prompt termination well under the %s bailout bound, took %s: %v", boundedTimeout, elapsed, trackErr) + } + + const expectedDiagnostic = `pods "demo-111" is forbidden: exceeded quota: compute-quota` + var diagnosticFound bool + taskState.RTransaction(func(state *statestore.ReadinessTaskState) { + resourceState := state.ResourceState(state.Name(), state.Namespace(), state.GroupVersionKind()) + resourceState.RTransaction(func(resource *statestore.ResourceState) { + for _, errs := range resource.Errors() { + for _, resourceErr := range errs { + if strings.Contains(resourceErr.Err.Error(), expectedDiagnostic) { + diagnosticFound = true + } + } + } + }) + }) + if !diagnosticFound { + t.Fatalf("task state does not retain ReplicaSet diagnostic %q", expectedDiagnostic) + } + }) + } +} + +func replicaFailureOneReplicaDeployment() *appsv1.Deployment { + one := int32(1) + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "demo", Namespace: "default", UID: "dep-uid-1", Generation: 1}, + Spec: appsv1.DeploymentSpec{ + Replicas: &one, + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "demo"}}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "demo"}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "c", Image: "busybox:1"}}}, + }, + }, + Status: appsv1.DeploymentStatus{ObservedGeneration: 1}, + } +} + +func replicaFailureFailedCreateReplicaSet() *appsv1.ReplicaSet { + one := int32(1) + isController := true + return &appsv1.ReplicaSet{ + TypeMeta: metav1.TypeMeta{Kind: "ReplicaSet", APIVersion: "apps/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "demo-111", + Namespace: "default", + UID: "rs-uid-1", + Labels: map[string]string{"app": "demo"}, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "apps/v1", Kind: "Deployment", Name: "demo", UID: "dep-uid-1", Controller: &isController, + }}, + CreationTimestamp: metav1.Now(), + }, + Spec: appsv1.ReplicaSetSpec{ + Replicas: &one, + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "demo"}}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "demo"}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "c", Image: "busybox:1"}}}, + }, + }, + Status: appsv1.ReplicaSetStatus{ + Conditions: []appsv1.ReplicaSetCondition{{ + Type: appsv1.ReplicaSetReplicaFailure, + Status: corev1.ConditionTrue, + Reason: "FailedCreate", + Message: `pods "demo-111" is forbidden: exceeded quota: compute-quota`, + LastTransitionTime: metav1.Now(), + }}, + }, + } +} + +type replicaFailureStubRESTMapper struct{} + +func (replicaFailureStubRESTMapper) KindFor(schema.GroupVersionResource) (schema.GroupVersionKind, error) { + return schema.GroupVersionKind{}, errors.New("not implemented") +} + +func (replicaFailureStubRESTMapper) KindsFor(schema.GroupVersionResource) ([]schema.GroupVersionKind, error) { + return nil, errors.New("not implemented") +} + +func (replicaFailureStubRESTMapper) ResourceFor(schema.GroupVersionResource) (schema.GroupVersionResource, error) { + return schema.GroupVersionResource{}, errors.New("not implemented") +} + +func (replicaFailureStubRESTMapper) ResourcesFor(schema.GroupVersionResource) ([]schema.GroupVersionResource, error) { + return nil, errors.New("not implemented") +} + +func (replicaFailureStubRESTMapper) RESTMapping(gk schema.GroupKind, _ ...string) (*meta.RESTMapping, error) { + if strings.EqualFold(gk.Group, "apps") && strings.EqualFold(gk.Kind, "deployment") { + return &meta.RESTMapping{ + Resource: schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}, + GroupVersionKind: replicaFailureDeploymentGVK, + Scope: meta.RESTScopeNamespace, + }, nil + } + return nil, fmt.Errorf("replicaFailureStubRESTMapper: no mapping for %s", gk) +} + +func (m replicaFailureStubRESTMapper) RESTMappings(gk schema.GroupKind, versions ...string) ([]*meta.RESTMapping, error) { + mapping, err := m.RESTMapping(gk, versions...) + if err != nil { + return nil, err + } + return []*meta.RESTMapping{mapping}, nil +} + +func (replicaFailureStubRESTMapper) ResourceSingularizer(resource string) (string, error) { + return strings.TrimSuffix(resource, "s"), nil +} + +func (replicaFailureStubRESTMapper) Reset() {} diff --git a/pkg/dyntracker/statestore/readiness_task_state.go b/pkg/dyntracker/statestore/readiness_task_state.go index e783ab45..cd9a0d13 100644 --- a/pkg/dyntracker/statestore/readiness_task_state.go +++ b/pkg/dyntracker/statestore/readiness_task_state.go @@ -190,29 +190,33 @@ func initReadinessTaskStateReadyConditions() []ReadinessTaskConditionFn { func initReadinessTaskStateFailureConditions(failMode FailMode, totalAllowFailuresCount int) []ReadinessTaskConditionFn { var failureConditions []ReadinessTaskConditionFn - failureConditions = append(failureConditions, func(taskState *ReadinessTaskState) bool { - var failed bool - lo.Must0(domigraph.BFS(taskState.resourceStatesTree, util.ResourceID(taskState.name, taskState.namespace, taskState.groupVersionKind), func(id string) bool { - state := lo.Must(taskState.resourceStatesTree.Vertex(id)) - state.RTransaction(func(s *ResourceState) { - if s.Status() == ResourceStatusFailed { - failed = true - } - }) + switch failMode { + case IgnoreAndContinueDeployProcess: + // This mode must do nothing when tracking of the resource fails, so neither a + // resource marked as failed nor an exceeded error budget may fail the task. + case FailWholeDeployProcessImmediately, LegacyHopeUntilEndOfDeployProcess: + // A resource marked as failed reports a durable failure it cannot recover from on + // its own, so it fails the task regardless of the allowed failures count. + failureConditions = append(failureConditions, func(taskState *ReadinessTaskState) bool { + var failed bool + lo.Must0(domigraph.BFS(taskState.resourceStatesTree, util.ResourceID(taskState.name, taskState.namespace, taskState.groupVersionKind), func(id string) bool { + state := lo.Must(taskState.resourceStatesTree.Vertex(id)) + state.RTransaction(func(s *ResourceState) { + if s.Status() == ResourceStatusFailed { + failed = true + } + }) - if failed { - return true - } + if failed { + return true + } - return false - })) + return false + })) - return failed - }) + return failed + }) - switch failMode { - case IgnoreAndContinueDeployProcess: - case FailWholeDeployProcessImmediately, LegacyHopeUntilEndOfDeployProcess: failureConditions = append(failureConditions, func(taskState *ReadinessTaskState) bool { var totalErrsCount int lo.Must0(domigraph.BFS(taskState.resourceStatesTree, util.ResourceID(taskState.name, taskState.namespace, taskState.groupVersionKind), func(id string) bool { diff --git a/pkg/tracker/deployment/export_test.go b/pkg/tracker/deployment/export_test.go index 188cdb36..e3bcbb4a 100644 --- a/pkg/tracker/deployment/export_test.go +++ b/pkg/tracker/deployment/export_test.go @@ -2,6 +2,10 @@ package deployment import appsv1 "k8s.io/api/apps/v1" +func (d *Tracker) TestOnlyInjectReplicaSetAdded(rs *appsv1.ReplicaSet) { + d.replicaSetAdded <- rs +} + func (d *Tracker) TestOnlyInjectReplicaSetModified(rs *appsv1.ReplicaSet) { d.replicaSetModified <- rs } @@ -9,3 +13,15 @@ func (d *Tracker) TestOnlyInjectReplicaSetModified(rs *appsv1.ReplicaSet) { func (d *Tracker) TestOnlyInjectReplicaSetDeleted(rs *appsv1.ReplicaSet) { d.replicaSetDeleted <- rs } + +func (d *Tracker) TestOnlyInjectReplicaSetUnselected(rs *appsv1.ReplicaSet) { + d.replicaSetUnselected <- rs +} + +func (d *Tracker) TestOnlyReplicaSetDeletedChanCap() int { + return cap(d.replicaSetDeleted) +} + +func (d *Tracker) TestOnlyInjectResourceFailure(reason string) { + d.resourceFailed <- reason +} diff --git a/pkg/tracker/deployment/failure.go b/pkg/tracker/deployment/failure.go new file mode 100644 index 00000000..ab07d6e7 --- /dev/null +++ b/pkg/tracker/deployment/failure.go @@ -0,0 +1,37 @@ +package deployment + +import ( + "context" + + "k8s.io/apimachinery/pkg/types" + + "github.com/werf/kubedog/pkg/tracker" +) + +type resourceFailure struct { + reason string + fromReplicaSetUID types.UID + mode FailureMode +} + +func (d *Tracker) handleResourceFailure(ctx context.Context, failure resourceFailure) error { + d.State = tracker.ResourceFailed + d.failedFromReplicaSetUID = failure.fromReplicaSetUID + + status, err := d.newStatus(d.lastObject) + if err != nil { + return err + } + + status.IsReady = false + status.IsFailed = true + status.FailedReason = failure.reason + status.FailureMode = failure.mode + + select { + case d.Failed <- status: + return nil + case <-ctx.Done(): + return context.Cause(ctx) + } +} diff --git a/pkg/tracker/deployment/harness_test.go b/pkg/tracker/deployment/harness_test.go index e79a723e..9f27c20b 100644 --- a/pkg/tracker/deployment/harness_test.go +++ b/pkg/tracker/deployment/harness_test.go @@ -49,7 +49,8 @@ type observation struct { } type harnessConfig struct { - dynamicSeed []runtime.Object + dynamicSeed []runtime.Object + prepareDynamicClient func(*dynamicfake.FakeDynamicClient) } type harness struct { @@ -72,6 +73,9 @@ func newHarness(t *testing.T, cfg harnessConfig) *harness { ctx, cancel := context.WithCancel(context.Background()) dynClient := dynamicfake.NewSimpleDynamicClient(k8sscheme.Scheme, cfg.dynamicSeed...) + if cfg.prepareDynamicClient != nil { + cfg.prepareDynamicClient(dynClient) + } kubeClient := k8sfake.NewSimpleClientset() watchErrCh := make(chan error, 100) @@ -257,6 +261,19 @@ func (h *harness) deleteObject(t *testing.T, gvr schema.GroupVersionResource, na } } +// injectReplicaSetDeletedAndWait delivers a ReplicaSet deletion straight to the +// tracker and returns only after the tracker has handled it. Deletions are +// idempotent, so the barrier is built by overfilling the channel: for a capacity +// of N, send N+1 returns once the tracker received the first deletion, and send +// N+2 returns once it received the second one, which can only happen after the +// first one was handled to completion. +func (h *harness) injectReplicaSetDeletedAndWait(t *testing.T, rs *appsv1.ReplicaSet) { + t.Helper() + for i := 0; i < h.tracker.TestOnlyReplicaSetDeletedChanCap()+2; i++ { + h.tracker.TestOnlyInjectReplicaSetDeleted(rs.DeepCopy()) + } +} + func baseSelector() *metav1.LabelSelector { return &metav1.LabelSelector{MatchLabels: map[string]string{"app": "demo"}} } @@ -283,6 +300,17 @@ func newNotReadyDeployment() *appsv1.Deployment { } } +func newReadyDeployment() *appsv1.Deployment { + dep := newNotReadyDeployment() + dep.Status = appsv1.DeploymentStatus{ + ObservedGeneration: 1, + Replicas: 1, + UpdatedReplicas: 1, + AvailableReplicas: 1, + } + return dep +} + func newReplicaSet(name string, uid types.UID) *appsv1.ReplicaSet { return &appsv1.ReplicaSet{ TypeMeta: metav1.TypeMeta{Kind: "ReplicaSet", APIVersion: "apps/v1"}, @@ -316,11 +344,15 @@ func withReplicaFailure(rs *appsv1.ReplicaSet, reason, marker string) *appsv1.Re } func failedWithMarker(marker string) func(observation) bool { + return failedWithMode(marker, deployment.FailureModeFatal) +} + +func failedWithMode(marker string, mode deployment.FailureMode) func(observation) bool { return func(ev observation) bool { status, ok := ev.Data.(deployment.DeploymentStatus) if !ok { return false } - return status.IsFailed && strings.Contains(status.FailedReason, marker) + return status.IsFailed && status.FailureMode == mode && strings.Contains(status.FailedReason, marker) } } diff --git a/pkg/tracker/deployment/replicaset_failure.go b/pkg/tracker/deployment/replicaset_failure.go new file mode 100644 index 00000000..3f8df401 --- /dev/null +++ b/pkg/tracker/deployment/replicaset_failure.go @@ -0,0 +1,164 @@ +package deployment + +import ( + "context" + "fmt" + "strings" + + "github.com/samber/lo" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + + "github.com/werf/kubedog/pkg/tracker" + "github.com/werf/kubedog/pkg/utils" +) + +func (d *Tracker) clearReplicaSetFailure(replicaSetUID types.UID) { + if replicaSetUID == "" || d.failedFromReplicaSetUID != replicaSetUID { + return + } + + d.failedFromReplicaSetUID = "" + + if d.State == tracker.ResourceFailed { + d.State = tracker.ResourceAdded + } +} + +func (d *Tracker) rearmReplicaSetFailure(rs *appsv1.ReplicaSet) { + if _, failed := replicaSetFailure(rs); failed { + return + } + + delete(d.reportedReplicaSetFailures, rs.UID) + d.clearReplicaSetFailure(rs.UID) +} + +func (d *Tracker) isStaleReplicaSet(rs *appsv1.ReplicaSet) bool { + _, deleted := d.deletedReplicaSetUIDs[rs.UID] + return deleted +} + +func (d *Tracker) handleNewReplicaSetFailure(ctx context.Context) error { + if d.lastObject == nil || !d.replicaSetsSynced { + return nil + } + + newReplicaSet, err := utils.FindNewReplicaSet(d.lastObject, lo.Values(d.knownReplicaSets)) + if err != nil { + return err + } + if newReplicaSet == nil { + return nil + } + + failure, failed := replicaSetFailure(newReplicaSet) + if !failed { + return nil + } + + if deploymentStatusReady(d.lastObject) { + delete(d.reportedReplicaSetFailures, newReplicaSet.UID) + d.clearReplicaSetFailure(newReplicaSet.UID) + return nil + } + + mode := replicaSetFailureMode(failure.Message, d.deploymentCanReleaseQuota(newReplicaSet)) + // ReplicaSet controllers retain FailedCreate until a successful retry. Keep + // the first classification until that condition clears so an old condition + // cannot become fatal merely because an older ReplicaSet finished scaling down. + if _, reported := d.reportedReplicaSetFailures[newReplicaSet.UID]; reported { + return nil + } + d.reportedReplicaSetFailures[newReplicaSet.UID] = struct{}{} + + return d.handleResourceFailure(ctx, resourceFailure{ + reason: fmt.Sprintf("%s: %s", failure.Reason, failure.Message), + fromReplicaSetUID: newReplicaSet.UID, + mode: mode, + }) +} + +func replicaSetFailureMode(message string, deploymentCanRecover bool) FailureMode { + message = strings.ToLower(message) + if strings.Contains(message, " is invalid:") || + (strings.Contains(message, "exceeded quota:") && !deploymentCanRecover) { + return FailureModeFatal + } + + return FailureModeCounted +} + +func (d *Tracker) deploymentCanReleaseQuota(newReplicaSet *appsv1.ReplicaSet) bool { + oldReplicaSetsHaveDesiredReplicas := false + allDesiredReplicas := *newReplicaSet.Spec.Replicas + for _, rs := range d.knownReplicaSets { + if rs.UID == newReplicaSet.UID || !metav1.IsControlledBy(rs, d.lastObject) { + continue + } + + desiredReplicas := *rs.Spec.Replicas + if rs.Status.TerminatingReplicas != nil && *rs.Status.TerminatingReplicas > 0 { + return true + } + if rs.Status.Replicas > desiredReplicas { + return true + } + + allDesiredReplicas += desiredReplicas + oldReplicaSetsHaveDesiredReplicas = oldReplicaSetsHaveDesiredReplicas || desiredReplicas > 0 + } + if !oldReplicaSetsHaveDesiredReplicas { + return false + } + + if d.lastObject.Spec.Strategy.Type == appsv1.RecreateDeploymentStrategyType { + return true + } + if d.lastObject.Spec.Strategy.Type != appsv1.RollingUpdateDeploymentStrategyType || + d.lastObject.Spec.Strategy.RollingUpdate == nil { + return false + } + + desiredReplicas := *d.lastObject.Spec.Replicas + rollingUpdate := d.lastObject.Spec.Strategy.RollingUpdate + maxSurge, err := intstr.GetScaledValueFromIntOrPercent( + intstr.ValueOrDefault(rollingUpdate.MaxSurge, intstr.FromInt32(0)), int(desiredReplicas), true, + ) + if err != nil { + return true + } + maxUnavailable, err := intstr.GetScaledValueFromIntOrPercent( + intstr.ValueOrDefault(rollingUpdate.MaxUnavailable, intstr.FromInt32(0)), int(desiredReplicas), false, + ) + if err != nil { + return true + } + if maxSurge == 0 && maxUnavailable == 0 { + maxUnavailable = 1 + } + if int32(maxUnavailable) > desiredReplicas { + maxUnavailable = int(desiredReplicas) + } + + minimumAvailable := desiredReplicas - int32(maxUnavailable) + newUnavailable := *newReplicaSet.Spec.Replicas - newReplicaSet.Status.AvailableReplicas + return allDesiredReplicas-minimumAvailable-newUnavailable > 0 +} + +func replicaSetFailure(rs *appsv1.ReplicaSet) (appsv1.ReplicaSetCondition, bool) { + for _, condition := range rs.Status.Conditions { + if condition.Type != appsv1.ReplicaSetReplicaFailure || + condition.Status != corev1.ConditionTrue || + condition.Reason != replicaSetFailedCreateReason { + continue + } + + return condition, true + } + + return appsv1.ReplicaSetCondition{}, false +} diff --git a/pkg/tracker/deployment/replicaset_failure_test.go b/pkg/tracker/deployment/replicaset_failure_test.go new file mode 100644 index 00000000..0b0cd054 --- /dev/null +++ b/pkg/tracker/deployment/replicaset_failure_test.go @@ -0,0 +1,52 @@ +package deployment + +import "testing" + +func TestReplicaSetFailureMode_whenFailedCreateMessageIsClassified(t *testing.T) { + tests := []struct { + name string + message string + deploymentCanRecover bool + want FailureMode + }{ + { + name: "quota exhaustion is fatal", + message: `pods "demo-111" is forbidden: exceeded quota: compute-quota`, + want: FailureModeFatal, + }, + { + name: "quota exhaustion is counted when rollout can release quota", + message: `pods "demo-111" is forbidden: exceeded quota: compute-quota`, + deploymentCanRecover: true, + want: FailureModeCounted, + }, + { + name: "invalid pod specification is fatal", + message: `Pod "demo-111" is invalid: spec.containers[0].resources.requests: Invalid value`, + want: FailureModeFatal, + }, + { + name: "temporary API failure is counted", + message: "failed calling create: temporary apiserver connection refused", + want: FailureModeCounted, + }, + { + name: "admission webhook rejection is counted", + message: `admission webhook "policy.example" denied the request`, + want: FailureModeCounted, + }, + { + name: "RBAC rejection is counted", + message: `pods is forbidden: User "system:serviceaccount:default:controller" cannot create resource "pods"`, + want: FailureModeCounted, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := replicaSetFailureMode(tt.message, tt.deploymentCanRecover); got != tt.want { + t.Fatalf("replicaSetFailureMode(%q, %t) = %v, want %v", tt.message, tt.deploymentCanRecover, got, tt.want) + } + }) + } +} diff --git a/pkg/tracker/deployment/status.go b/pkg/tracker/deployment/status.go index e3fa21b4..d91a60c8 100644 --- a/pkg/tracker/deployment/status.go +++ b/pkg/tracker/deployment/status.go @@ -9,6 +9,17 @@ import ( "github.com/werf/kubedog/pkg/tracker/pod" ) +// FailureMode qualifies a reported failure: a counted one may be transient and spends the +// consumer's error budget, while a fatal one is durable and bypasses it. +type FailureMode uint8 + +const ( + // FailureModeCounted is a recoverable failure charged against the consumer's error budget. + FailureModeCounted FailureMode = iota + // FailureModeFatal is a rollout-wide failure that cannot recover without an external change. + FailureModeFatal +) + type DeploymentStatus struct { appsv1.DeploymentStatus @@ -23,6 +34,7 @@ type DeploymentStatus struct { IsReady bool IsFailed bool FailedReason string + FailureMode FailureMode Pods map[string]pod.PodStatus // New Pod belongs to the new ReplicaSet of the Deployment, @@ -78,17 +90,14 @@ processingPodsStatuses: TargetValue: *object.Spec.Replicas, } - res.IsReady = true + res.IsReady = deploymentStatusReady(object) if object.Status.UpdatedReplicas != *object.Spec.Replicas { - res.IsReady = false res.WaitingForMessages = append(res.WaitingForMessages, fmt.Sprintf("up-to-date %d->%d", object.Status.UpdatedReplicas, *object.Spec.Replicas)) } if object.Status.Replicas != *object.Spec.Replicas { - res.IsReady = false res.WaitingForMessages = append(res.WaitingForMessages, fmt.Sprintf("replicas %d->%d", object.Status.Replicas, *object.Spec.Replicas)) } if object.Status.AvailableReplicas != *object.Spec.Replicas { - res.IsReady = false res.WaitingForMessages = append(res.WaitingForMessages, fmt.Sprintf("available %d->%d", object.Status.AvailableReplicas, *object.Spec.Replicas)) } } else { @@ -102,3 +111,15 @@ processingPodsStatuses: return res } + +// deploymentStatusReady reports whether the Deployment controller has observed the +// current spec and reports every replica up-to-date, present and available. +func deploymentStatusReady(object *appsv1.Deployment) bool { + if object.Status.ObservedGeneration < object.Generation || object.Spec.Replicas == nil { + return false + } + + return object.Status.UpdatedReplicas == *object.Spec.Replicas && + object.Status.Replicas == *object.Spec.Replicas && + object.Status.AvailableReplicas == *object.Spec.Replicas +} diff --git a/pkg/tracker/deployment/tracker.go b/pkg/tracker/deployment/tracker.go index 97e15ee5..ec7ddd46 100644 --- a/pkg/tracker/deployment/tracker.go +++ b/pkg/tracker/deployment/tracker.go @@ -26,9 +26,8 @@ import ( "github.com/werf/kubedog/pkg/utils" ) -// replicaSetFailedCreateReason is the reason the ReplicaSet controller sets on -// the ReplicaSetReplicaFailure condition when it cannot create pods, e.g. -// because a namespace quota is exceeded. +// replicaSetFailedCreateReason is set by the ReplicaSet controller on the +// ReplicaSetReplicaFailure condition when it cannot create pods. const replicaSetFailedCreateReason = "FailedCreate" type ReplicaSetAddedReport struct { @@ -54,9 +53,11 @@ type Tracker struct { NewReplicaSetName string knownReplicaSets map[string]*appsv1.ReplicaSet + deletedReplicaSetUIDs map[types.UID]struct{} lastObject *appsv1.Deployment - failedReason string - reportedReplicaSetFailures map[types.UID]string + failedFromReplicaSetUID types.UID + reportedReplicaSetFailures map[types.UID]struct{} + replicaSetsSynced bool podStatuses map[string]pod.PodStatus rsNameByPod map[string]string @@ -77,14 +78,16 @@ type Tracker struct { PodLogChunk chan *replicaset.ReplicaSetPodLogChunk PodError chan PodErrorReport - resourceAdded chan *appsv1.Deployment - resourceModified chan *appsv1.Deployment - resourceDeleted chan *appsv1.Deployment - resourceFailed chan interface{} - replicaSetAdded chan *appsv1.ReplicaSet - replicaSetModified chan *appsv1.ReplicaSet - replicaSetDeleted chan *appsv1.ReplicaSet - errors chan error + resourceAdded chan *appsv1.Deployment + resourceModified chan *appsv1.Deployment + resourceDeleted chan *appsv1.Deployment + resourceFailed chan interface{} + replicaSetAdded chan *appsv1.ReplicaSet + replicaSetModified chan *appsv1.ReplicaSet + replicaSetDeleted chan *appsv1.ReplicaSet + replicaSetUnselected chan *appsv1.ReplicaSet + replicaSetSynced chan struct{} + errors chan error podAddedRelay chan *corev1.Pod podStatusesRelay chan map[string]pod.PodStatus @@ -117,21 +120,27 @@ func NewTracker(name, namespace string, kube kubernetes.Interface, informerFacto PodError: make(chan PodErrorReport), knownReplicaSets: make(map[string]*appsv1.ReplicaSet), - reportedReplicaSetFailures: make(map[types.UID]string), + deletedReplicaSetUIDs: make(map[types.UID]struct{}), + reportedReplicaSetFailures: make(map[types.UID]struct{}), podStatuses: make(map[string]pod.PodStatus), rsNameByPod: make(map[string]string), ignoreLogs: opts.IgnoreLogs, ignoreReadinessProbeFailsByContainerName: opts.IgnoreReadinessProbeFailsByContainerName, - errors: make(chan error, 1), - resourceAdded: make(chan *appsv1.Deployment, 1), - resourceModified: make(chan *appsv1.Deployment, 1), - resourceDeleted: make(chan *appsv1.Deployment, 1), - resourceFailed: make(chan interface{}, 1), - replicaSetAdded: make(chan *appsv1.ReplicaSet, 1), - replicaSetModified: make(chan *appsv1.ReplicaSet, 1), - replicaSetDeleted: make(chan *appsv1.ReplicaSet, 1), + errors: make(chan error, 1), + resourceAdded: make(chan *appsv1.Deployment, 1), + resourceModified: make(chan *appsv1.Deployment, 1), + resourceDeleted: make(chan *appsv1.Deployment, 1), + resourceFailed: make(chan interface{}, 1), + // ReplicaSet events come from a single informer goroutine and fan out into three + // channels. Unbuffered sends keep them in order: a buffered Added could otherwise + // be consumed after a newer Modified and overwrite a fresh snapshot with a stale one. + replicaSetAdded: make(chan *appsv1.ReplicaSet), + replicaSetModified: make(chan *appsv1.ReplicaSet), + replicaSetDeleted: make(chan *appsv1.ReplicaSet), + replicaSetUnselected: make(chan *appsv1.ReplicaSet), + replicaSetSynced: make(chan struct{}), podAddedRelay: make(chan *corev1.Pod, 1), podStatusesRelay: make(chan map[string]pod.PodStatus, 10), @@ -163,7 +172,7 @@ func (d *Tracker) Track(ctx context.Context) (err error) { return err } defer cleanupFn() - if err := d.handleNewReplicaSetFailure(); err != nil { + if err := d.handleNewReplicaSetFailure(ctx); err != nil { return err } case object := <-d.resourceModified: @@ -172,14 +181,16 @@ func (d *Tracker) Track(ctx context.Context) (err error) { return err } defer cleanupFn() - if err := d.handleNewReplicaSetFailure(); err != nil { + if err := d.handleNewReplicaSetFailure(ctx); err != nil { return err } case <-d.resourceDeleted: d.State = tracker.ResourceDeleted d.lastObject = nil d.knownReplicaSets = make(map[string]*appsv1.ReplicaSet) - d.reportedReplicaSetFailures = make(map[types.UID]string) + d.deletedReplicaSetUIDs = make(map[types.UID]struct{}) + d.reportedReplicaSetFailures = make(map[types.UID]struct{}) + d.failedFromReplicaSetUID = "" d.podStatuses = make(map[string]pod.PodStatus) d.rsNameByPod = make(map[string]string) d.TrackedPodsNames = nil @@ -188,15 +199,13 @@ func (d *Tracker) Track(ctx context.Context) (err error) { case failure := <-d.resourceFailed: switch failure := failure.(type) { case string: - // Failures are reported by the events informer, which is - // stopped only when Track returns. With no live object there is - // no status to report the failure on, and the failure itself - // belongs to an already deleted object. + // The events informer outlives the object: with no live object + // there is no status to report the failure on. if d.lastObject == nil { break } - if err := d.handleResourceFailure(failure); err != nil { + if err := d.handleResourceFailure(ctx, resourceFailure{reason: failure, mode: FailureModeCounted}); err != nil { return err } default: @@ -204,7 +213,12 @@ func (d *Tracker) Track(ctx context.Context) (err error) { } case rs := <-d.replicaSetAdded: + if d.isStaleReplicaSet(rs) { + break + } + d.knownReplicaSets[rs.Name] = rs + d.rearmReplicaSetFailure(rs) if d.lastObject != nil { rsNew, err := utils.IsReplicaSetNew(d.lastObject, d.knownReplicaSets, rs.Name) @@ -212,12 +226,10 @@ func (d *Tracker) Track(ctx context.Context) (err error) { return err } - d.StatusGeneration++ - newPodsNames, err := d.getNewPodsNames() + status, err := d.newStatus(d.lastObject) if err != nil { return err } - status := NewDeploymentStatus(d.lastObject, d.StatusGeneration, d.State == tracker.ResourceFailed, d.failedReason, d.podStatuses, newPodsNames) d.AddedReplicaSet <- ReplicaSetAddedReport{ ReplicaSet: replicaset.ReplicaSet{ @@ -227,44 +239,72 @@ func (d *Tracker) Track(ctx context.Context) (err error) { DeploymentStatus: status, } - if err := d.handleNewReplicaSetFailure(); err != nil { + if err := d.handleNewReplicaSetFailure(ctx); err != nil { return err } } case rs := <-d.replicaSetModified: - // Additions, modifications and deletions are delivered over - // separate channels, so a modification of an already replaced - // ReplicaSet may be observed after addition of its same-named - // successor: applying it would resurrect the dead incarnation. + if d.isStaleReplicaSet(rs) { + break + } + + // A same-named successor may already be known: applying an older + // incarnation would resurrect the dead one. if known, found := d.knownReplicaSets[rs.Name]; !found || known.UID == rs.UID { d.knownReplicaSets[rs.Name] = rs + d.rearmReplicaSetFailure(rs) if d.lastObject != nil { - if err := d.handleNewReplicaSetFailure(); err != nil { + if err := d.handleNewReplicaSetFailure(ctx); err != nil { return err } } } case rs := <-d.replicaSetDeleted: - // Same reordering hazard as for modifications: dropping the - // ReplicaSet by name alone would lose the live successor. + d.deletedReplicaSetUIDs[rs.UID] = struct{}{} + + // Dropping the ReplicaSet by name alone would lose a live successor. + if known, found := d.knownReplicaSets[rs.Name]; found && known.UID == rs.UID { + delete(d.knownReplicaSets, rs.Name) + } + delete(d.reportedReplicaSetFailures, rs.UID) + + d.clearReplicaSetFailure(rs.UID) + + // Deletion may promote an already failing ReplicaSet to the new one: + // its failure was suppressed before and nothing else would report it. + if d.lastObject != nil { + if err := d.handleNewReplicaSetFailure(ctx); err != nil { + return err + } + } + + case rs := <-d.replicaSetUnselected: if known, found := d.knownReplicaSets[rs.Name]; found && known.UID == rs.UID { delete(d.knownReplicaSets, rs.Name) } delete(d.reportedReplicaSetFailures, rs.UID) + d.clearReplicaSetFailure(rs.UID) + + if d.lastObject != nil { + if err := d.handleNewReplicaSetFailure(ctx); err != nil { + return err + } + } + + case <-d.replicaSetSynced: + d.replicaSetsSynced = true + if err := d.handleNewReplicaSetFailure(ctx); err != nil { + return err + } case pod := <-d.podAddedRelay: rsName := utils.GetPodReplicaSetName(pod) d.rsNameByPod[pod.Name] = rsName if d.lastObject != nil { - d.StatusGeneration++ - newPodsNames, err := d.getNewPodsNames() - if err != nil { - return err - } rsNew, err := utils.IsReplicaSetNew(d.lastObject, d.knownReplicaSets, rsName) if err != nil { return err @@ -272,7 +312,11 @@ func (d *Tracker) Track(ctx context.Context) (err error) { if len(d.knownReplicaSets) == 0 { rsNew = true } - status := NewDeploymentStatus(d.lastObject, d.StatusGeneration, d.State == tracker.ResourceFailed, d.failedReason, d.podStatuses, newPodsNames) + + status, err := d.newStatus(d.lastObject) + if err != nil { + return err + } d.AddedPod <- PodAddedReport{ ReplicaSetPod: replicaset.ReplicaSetPod{ @@ -366,12 +410,10 @@ func (d *Tracker) Track(ctx context.Context) (err error) { d.podStatuses[podName] = containerError.PodStatus } if d.lastObject != nil { - d.StatusGeneration++ - newPodsNames, err := d.getNewPodsNames() + status, err := d.newStatus(d.lastObject) if err != nil { return err } - status := NewDeploymentStatus(d.lastObject, d.StatusGeneration, d.State == tracker.ResourceFailed, d.failedReason, d.podStatuses, newPodsNames) for podName, containerError := range podContainerErrors { rsName, hasKey := d.rsNameByPod[podName] @@ -522,6 +564,8 @@ func (d *Tracker) runDeploymentInformer(ctx context.Context) (cleanupFn func(), func (d *Tracker) runReplicaSetsInformer(ctx context.Context, object *appsv1.Deployment) (cleanupFn func(), err error) { rsInformer := replicaset.NewReplicaSetInformer(&d.Tracker, utils.ControllerAccessor(object)) rsInformer.WithChannels(d.replicaSetAdded, d.replicaSetModified, d.replicaSetDeleted, d.errors) + rsInformer.WithUnselectedChannel(d.replicaSetUnselected) + rsInformer.WithSyncedChannel(d.replicaSetSynced) return rsInformer.Run(ctx) } @@ -604,13 +648,11 @@ func (d *Tracker) runPodTracker(_ctx context.Context, podName, rsName string) er func (d *Tracker) handleDeploymentState(ctx context.Context, object *appsv1.Deployment) (cleanupFn func(), err error) { d.lastObject = object - d.StatusGeneration++ - newPodsNames, err := d.getNewPodsNames() + status, err := d.newStatus(object) if err != nil { return nil, err } - status := NewDeploymentStatus(object, d.StatusGeneration, d.State == tracker.ResourceFailed, d.failedReason, d.podStatuses, newPodsNames) cleanupFn = func() {} @@ -626,6 +668,9 @@ func (d *Tracker) handleDeploymentState(ctx context.Context, object *appsv1.Depl podsInformerCleanupFn, err := d.runPodsInformer(ctx, object) if err != nil { + // An informer left running has no consumer: its callbacks park forever + // on the unbuffered channels of a tracker that never starts. + replicasetsInformerCleanupFn() return nil, fmt.Errorf("run pods informer: %w", err) } @@ -633,6 +678,8 @@ func (d *Tracker) handleDeploymentState(ctx context.Context, object *appsv1.Depl if os.Getenv("KUBEDOG_DISABLE_EVENTS") != "1" { eventsInformerCleanupFn, err = d.runEventsInformer(ctx, object) if err != nil { + replicasetsInformerCleanupFn() + podsInformerCleanupFn() return nil, fmt.Errorf("run events informer: %w", err) } } @@ -691,73 +738,15 @@ func (d *Tracker) runEventsInformer(ctx context.Context, resource interface{}) ( return eventInformer.Run(ctx) } -func (d *Tracker) handleResourceFailure(reason string) error { - d.State = tracker.ResourceFailed - d.failedReason = reason - +// newStatus deliberately leaves the sticky tracker failure out: a failure is reported +// once by handleResourceFailure, and restamping it into every update reports it again. +func (d *Tracker) newStatus(object *appsv1.Deployment) (DeploymentStatus, error) { d.StatusGeneration++ - newPodsNames, err := d.getNewPodsNames() - if err != nil { - return err - } - d.Failed <- NewDeploymentStatus(d.lastObject, d.StatusGeneration, d.State == tracker.ResourceFailed, d.failedReason, d.podStatuses, newPodsNames) - return nil -} - -// handleNewReplicaSetFailure fails the tracking as soon as the ReplicaSet the -// Deployment is currently rolling out cannot create its pods. -// -// ReplicaSets of previous rollouts are skipped: their failures must not fail the -// current rollout. A failure is reported once per ReplicaSet incarnation and -// reason and is rearmed once the ReplicaSet controller clears the condition, so -// a ReplicaSet that recovered and then failed again is reported anew. -func (d *Tracker) handleNewReplicaSetFailure() error { - if d.lastObject == nil { - return nil - } - - newReplicaSet, err := utils.FindNewReplicaSet(d.lastObject, lo.Values(d.knownReplicaSets)) + newPodsNames, err := d.getNewPodsNames() if err != nil { - return err - } - if newReplicaSet == nil { - return nil - } - - failure, failed := replicaSetFailure(newReplicaSet) - if !failed { - delete(d.reportedReplicaSetFailures, newReplicaSet.UID) - return nil - } - - // The message carries volatile details, e.g. the quota usage at the moment the - // pod was rejected, so only the reason takes part in the deduplication. - if d.reportedReplicaSetFailures[newReplicaSet.UID] == failure.Reason { - return nil - } - d.reportedReplicaSetFailures[newReplicaSet.UID] = failure.Reason - - return d.handleResourceFailure(fmt.Sprintf("%s: %s", failure.Reason, failure.Message)) -} - -// replicaSetFailure returns the condition set by the ReplicaSet controller when it -// failed to create pods for the ReplicaSet. -// -// Unlike a Warning event, the ReplicaSetReplicaFailure condition is durable state: -// the controller sets it while pod creation keeps failing and removes it as soon as -// creation succeeds, so no event baseline is needed. FailedDelete is not fatal for a -// rollout and is ignored here, just like the events informer ignores it. -func replicaSetFailure(rs *appsv1.ReplicaSet) (appsv1.ReplicaSetCondition, bool) { - for _, condition := range rs.Status.Conditions { - if condition.Type != appsv1.ReplicaSetReplicaFailure || - condition.Status != corev1.ConditionTrue || - condition.Reason != replicaSetFailedCreateReason { - continue - } - - return condition, true + return DeploymentStatus{}, err } - return appsv1.ReplicaSetCondition{}, false + return NewDeploymentStatus(object, d.StatusGeneration, false, "", d.podStatuses, newPodsNames), nil } diff --git a/pkg/tracker/deployment/tracker_failure_test.go b/pkg/tracker/deployment/tracker_failure_test.go new file mode 100644 index 00000000..2f8d13f1 --- /dev/null +++ b/pkg/tracker/deployment/tracker_failure_test.go @@ -0,0 +1,49 @@ +package deployment + +import ( + "context" + "errors" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/werf/kubedog/pkg/tracker/pod" +) + +func TestHandleResourceFailure_whenContextCanceledWithoutReceiver(t *testing.T) { + ctx, cancel := context.WithCancelCause(context.Background()) + expectedErr := errors.New("test cancellation") + + trk := &Tracker{ + lastObject: &appsv1.Deployment{}, + knownReplicaSets: make(map[string]*appsv1.ReplicaSet), + podStatuses: make(map[string]pod.PodStatus), + rsNameByPod: make(map[string]string), + Failed: make(chan DeploymentStatus), + } + + started := make(chan struct{}) + done := make(chan error, 1) + go func() { + close(started) + done <- trk.handleResourceFailure(ctx, resourceFailure{ + reason: "failure", + fromReplicaSetUID: types.UID("rs-uid-1"), + mode: FailureModeCounted, + }) + }() + + <-started + cancel(expectedErr) + + select { + case err := <-done: + if !errors.Is(err, expectedErr) { + t.Fatalf("handleResourceFailure() error = %v, want %v", err, expectedErr) + } + case <-time.After(time.Second): + t.Fatal("handleResourceFailure() remained blocked after context cancellation") + } +} diff --git a/pkg/tracker/deployment/tracker_quota_recovery_test.go b/pkg/tracker/deployment/tracker_quota_recovery_test.go new file mode 100644 index 00000000..f2f6db71 --- /dev/null +++ b/pkg/tracker/deployment/tracker_quota_recovery_test.go @@ -0,0 +1,144 @@ +package deployment_test + +import ( + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/intstr" + dynamicfake "k8s.io/client-go/dynamic/fake" + k8stesting "k8s.io/client-go/testing" + + "github.com/werf/kubedog/pkg/tracker/deployment" +) + +func TestTrack_RecoversFromQuotaFailureWhenOldReplicaSetCanScaleDown(t *testing.T) { + deploymentObject := rollingDeployment(1, 1) + h := newHarness(t, harnessConfig{dynamicSeed: []runtime.Object{deploymentObject}}) + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + oldReplicaSet := scalableOldReplicaSet() + h.createObject(t, gvrReplicaSets, oldReplicaSet) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, nil); err != nil { + t.Fatalf("wait for old AddedReplicaSet: %v", err) + } + + const marker = "marker-quota-recoverable-by-scale-down" + newReplicaSet := withReplicaFailure(newReplicaSet("demo-new", "rs-uid-new"), "FailedCreate", marker) + h.createObject(t, gvrReplicaSets, newReplicaSet) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMode(marker, deployment.FailureModeCounted)); err != nil { + t.Fatalf("quota failure must remain recoverable while an old ReplicaSet can scale down: %v", err) + } +} + +func TestTrack_RecoversFromQuotaFailureWhileOldReplicaSetPodsAreTerminating(t *testing.T) { + deploymentObject := rollingDeployment(0, 1) + h := newHarness(t, harnessConfig{dynamicSeed: []runtime.Object{deploymentObject}}) + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + oldReplicaSet := scalableOldReplicaSet() + *oldReplicaSet.Spec.Replicas = 0 + oldReplicaSet.Status.Replicas = 0 + oldReplicaSet.Status.AvailableReplicas = 0 + oldReplicaSet.Status.TerminatingReplicas = ptrTo(int32(1)) + h.createObject(t, gvrReplicaSets, oldReplicaSet) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, nil); err != nil { + t.Fatalf("wait for terminating old AddedReplicaSet: %v", err) + } + + const marker = "marker-quota-recoverable-by-termination" + newReplicaSet := withReplicaFailure(newReplicaSet("demo-new", "rs-uid-new"), "FailedCreate", marker) + h.createObject(t, gvrReplicaSets, newReplicaSet) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMode(marker, deployment.FailureModeCounted)); err != nil { + t.Fatalf("quota failure must remain recoverable while old Pods terminate: %v", err) + } +} + +func TestTrack_ClassifiesInitialQuotaFailureAfterAllReplicaSetsAreObserved(t *testing.T) { + deploymentObject := rollingDeployment(1, 1) + oldReplicaSet := scalableOldReplicaSet() + const marker = "marker-quota-initial-snapshot" + newReplicaSet := withReplicaFailure(newReplicaSet("demo-new", "rs-uid-new"), "FailedCreate", marker) + + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{deploymentObject}, + prepareDynamicClient: initialReplicaSetListReactor(t, newReplicaSet, oldReplicaSet), + }) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMode(marker, deployment.FailureModeCounted)); err != nil { + t.Fatalf("initial failure must use the complete ReplicaSet snapshot: %v", err) + } +} + +func TestTrack_DoesNotEscalateStaleQuotaFailureAfterOldReplicaSetScalesDown(t *testing.T) { + deploymentObject := rollingDeployment(1, 1) + h := newHarness(t, harnessConfig{dynamicSeed: []runtime.Object{deploymentObject}}) + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + oldReplicaSet := scalableOldReplicaSet() + h.createObject(t, gvrReplicaSets, oldReplicaSet) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, nil); err != nil { + t.Fatalf("wait for old AddedReplicaSet: %v", err) + } + + const marker = "marker-quota-stale-after-scale-down" + newReplicaSet := withReplicaFailure(newReplicaSet("demo-new", "rs-uid-new"), "FailedCreate", marker) + h.createObject(t, gvrReplicaSets, newReplicaSet) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMode(marker, deployment.FailureModeCounted)); err != nil { + t.Fatalf("wait for initial counted failure: %v", err) + } + + scaledDown := oldReplicaSet.DeepCopy() + *scaledDown.Spec.Replicas = 0 + scaledDown.Status.Replicas = 0 + scaledDown.Status.AvailableReplicas = 0 + h.updateReplicaSet(t, scaledDown) + if err := h.waitForNone("Failed", 2500*time.Millisecond, failedWithMode(marker, deployment.FailureModeFatal)); err != nil { + t.Fatalf("stale quota condition must not become fatal while the ReplicaSet controller retries: %v", err) + } +} + +func rollingDeployment(maxUnavailable, maxSurge int32) *appsv1.Deployment { + deploymentObject := newNotReadyDeployment() + unavailable := intstr.FromInt32(maxUnavailable) + surge := intstr.FromInt32(maxSurge) + deploymentObject.Spec.Strategy = appsv1.DeploymentStrategy{ + Type: appsv1.RollingUpdateDeploymentStrategyType, + RollingUpdate: &appsv1.RollingUpdateDeployment{ + MaxUnavailable: &unavailable, + MaxSurge: &surge, + }, + } + return deploymentObject +} + +func scalableOldReplicaSet() *appsv1.ReplicaSet { + oldReplicaSet := newReplicaSet("demo-old", "rs-uid-old") + oldReplicaSet.Spec.Template.Spec.Containers[0].Image = "busybox:old" + oldReplicaSet.Status.Replicas = 1 + oldReplicaSet.Status.AvailableReplicas = 1 + return oldReplicaSet +} + +func initialReplicaSetListReactor(t *testing.T, replicaSets ...*appsv1.ReplicaSet) func(*dynamicfake.FakeDynamicClient) { + t.Helper() + + return func(client *dynamicfake.FakeDynamicClient) { + client.PrependReactor("list", "replicasets", func(k8stesting.Action) (bool, runtime.Object, error) { + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "ReplicaSetList"}) + for _, replicaSet := range replicaSets { + list.Items = append(list.Items, *mustToUnstructured(t, replicaSet)) + } + return true, list, nil + }) + } +} diff --git a/pkg/tracker/deployment/tracker_replicaset_recovery_test.go b/pkg/tracker/deployment/tracker_replicaset_recovery_test.go new file mode 100644 index 00000000..aea06928 --- /dev/null +++ b/pkg/tracker/deployment/tracker_replicaset_recovery_test.go @@ -0,0 +1,85 @@ +package deployment_test + +import ( + "testing" + "time" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + + "github.com/werf/kubedog/pkg/tracker/deployment" +) + +func TestTrack_RecoversFromTransientReplicaFailure(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + const marker = "marker-transient-apiserver-failure" + rs := newReplicaSet("demo-111", "rs-uid-1") + failed := withReplicaFailure(rs.DeepCopy(), "FailedCreate", marker) + failed.Status.Conditions[0].Message = "temporary apiserver connection refused [" + marker + "]" + h.createObject(t, gvrReplicaSets, failed) + + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMode(marker, deployment.FailureModeCounted)); err != nil { + t.Fatalf("wait for counted failure: %v", err) + } + + h.updateReplicaSet(t, rs.DeepCopy()) + h.updateDeployment(t, newReadyDeployment()) + if _, err := h.waitFor("Ready", 10*time.Second, nil); err != nil { + t.Fatalf("transient failure must allow later readiness: %v", err) + } +} + +func TestTrack_DetectsFailureAfterSameNameReplicaSetReplacementUpdate(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + oldReplicaSet := newReplicaSet("demo-111", "rs-uid-old") + h.createObject(t, gvrReplicaSets, oldReplicaSet) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedNewReplicaSet(oldReplicaSet.Name)); err != nil { + t.Fatalf("wait for old AddedReplicaSet: %v", err) + } + + const marker = "marker-replacement-update" + replacement := withReplicaFailure(oldReplicaSet.DeepCopy(), "FailedCreate", marker) + replacement.UID = types.UID("rs-uid-new") + h.updateReplicaSet(t, replacement) + + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(marker)); err != nil { + t.Fatalf("same-name replacement delivered as Update must be tracked: %v", err) + } +} + +func TestTrack_ReportsReplicaFailureAfterSelectorReentry(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + const marker = "marker-selector-reentry" + rs := withReplicaFailure(newReplicaSet("demo-111", "rs-uid-1"), "FailedCreate", marker) + h.createObject(t, gvrReplicaSets, rs) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(marker)); err != nil { + t.Fatalf("wait for initial failure: %v", err) + } + + h.tracker.TestOnlyInjectReplicaSetUnselected(rs.DeepCopy()) + h.tracker.TestOnlyInjectReplicaSetAdded(rs.DeepCopy()) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(marker)); err != nil { + t.Fatalf("same UID must become reportable after selector reentry: %v", err) + } +} diff --git a/pkg/tracker/deployment/tracker_replicaset_test.go b/pkg/tracker/deployment/tracker_replicaset_test.go index 56b08642..01b7d2e3 100644 --- a/pkg/tracker/deployment/tracker_replicaset_test.go +++ b/pkg/tracker/deployment/tracker_replicaset_test.go @@ -166,6 +166,319 @@ func TestTrack_IgnoresStaleEventsForRecreatedReplicaSet(t *testing.T) { } } +func TestTrack_IgnoresReplicaFailureWhenDeploymentStatusIsReady(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newReadyDeployment()}, + }) + + if _, err := h.waitFor("Ready", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Ready: %v", err) + } + + const marker = "marker-ready-deployment-wins" + h.createObject(t, gvrReplicaSets, withReplicaFailure(newReplicaSet("demo-111", "rs-uid-1"), "FailedCreate", marker)) + + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedReplicaSetNamed("demo-111")); err != nil { + t.Fatalf("wait for AddedReplicaSet: %v", err) + } + + if err := h.waitForNone("Failed", 2500*time.Millisecond, failedWithMarker(marker)); err != nil { + t.Fatal(err) + } +} + +func TestTrack_FailsOnReplicaFailureWhileDeploymentIsNotReady(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + const marker = "marker-not-ready-deployment-fails" + h.createObject(t, gvrReplicaSets, withReplicaFailure(newReplicaSet("demo-111", "rs-uid-1"), "FailedCreate", marker)) + + ev, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(marker)) + if err != nil { + t.Fatalf("wait for Failed: %v", err) + } + + status, ok := ev.Data.(deployment.DeploymentStatus) + if !ok { + t.Fatalf("unexpected Failed payload: %T", ev.Data) + } + if status.IsReady { + t.Fatalf("reported failure must not be ready: %+v", status) + } + + if err := h.waitForNone("Ready", 2500*time.Millisecond, nil); err != nil { + t.Fatal(err) + } +} + +func TestTrack_ReportsReplicaFailureAgainAfterDeploymentStoppedBeingReady(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + const marker = "marker-readiness-rearms-reporting" + h.createObject(t, gvrReplicaSets, withReplicaFailure(newReplicaSet("demo-111", "rs-uid-1"), "FailedCreate", marker)) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(marker)); err != nil { + t.Fatalf("wait for first Failed: %v", err) + } + + h.updateDeployment(t, newReadyDeployment()) + if _, err := h.waitFor("Ready", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Ready: %v", err) + } + + replicaLost := newReadyDeployment() + replicaLost.Status.AvailableReplicas = 0 + replicaLost.Status.UnavailableReplicas = 1 + h.updateDeployment(t, replicaLost) + + ev, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(marker)) + if err != nil { + t.Fatalf("a Deployment that stopped being ready must be failed anew by the condition it recovered from: %v", err) + } + + status, ok := ev.Data.(deployment.DeploymentStatus) + if !ok { + t.Fatalf("unexpected Failed payload: %T", ev.Data) + } + if status.IsReady { + t.Fatalf("reported failure must not be ready: %+v", status) + } + if status.AvailableReplicas != 0 || status.UnavailableReplicas != 1 { + t.Fatalf("reported failure must describe the lost replica: %+v", status) + } +} + +func TestTrack_ReportsReplicaFailureAgainAfterRollbackToRecoveredReplicaSet(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + const firstMarker = "marker-before-rollout" + h.createObject(t, gvrReplicaSets, withReplicaFailure(newReplicaSet("demo-111", "rs-uid-1"), "FailedCreate", firstMarker)) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(firstMarker)); err != nil { + t.Fatalf("wait for first Failed: %v", err) + } + + rolledOut := newNotReadyDeployment() + rolledOut.Spec.Template.Spec.Containers[0].Image = "busybox:2" + rolledOut.Status.Replicas = 1 + h.updateDeployment(t, rolledOut) + if _, err := h.waitFor("Status", 10*time.Second, statusWithReplicas(1)); err != nil { + t.Fatalf("wait for rollout Status: %v", err) + } + + successor := newReplicaSet("demo-222", "rs-uid-2") + successor.Spec.Template.Spec.Containers[0].Image = "busybox:2" + h.createObject(t, gvrReplicaSets, successor) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedNewReplicaSet(successor.Name)); err != nil { + t.Fatalf("wait for successor AddedReplicaSet: %v", err) + } + + h.updateReplicaSet(t, newReplicaSet("demo-111", "rs-uid-1")) + + const secondMarker = "marker-after-rollback" + h.updateReplicaSet(t, withReplicaFailure(newReplicaSet("demo-111", "rs-uid-1"), "FailedCreate", secondMarker)) + if err := h.waitForNone("Failed", 2500*time.Millisecond, failedWithMarker(secondMarker)); err != nil { + t.Fatalf("previous rollout ReplicaSet must not fail the current rollout: %v", err) + } + + h.updateDeployment(t, newNotReadyDeployment()) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(secondMarker)); err != nil { + t.Fatalf("failure of the ReplicaSet rolled back onto must be reported anew: %v", err) + } +} + +func TestTrack_ReportsReplicaFailureOnceAcrossStatusUpdates(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + const marker = "marker-reported-once" + h.createObject(t, gvrReplicaSets, withReplicaFailure(newReplicaSet("demo-111", "rs-uid-1"), "FailedCreate", marker)) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(marker)); err != nil { + t.Fatalf("wait for Failed: %v", err) + } + + dep := newNotReadyDeployment() + dep.Status.Replicas = 1 + h.updateDeployment(t, dep) + + ev, err := h.waitFor("Status", 10*time.Second, statusWithReplicas(1)) + if err != nil { + t.Fatalf("wait for Status: %v", err) + } + status, ok := ev.Data.(deployment.DeploymentStatus) + if !ok { + t.Fatalf("unexpected Status payload: %T", ev.Data) + } + if status.IsFailed || status.FailedReason != "" { + t.Fatalf("routine status must not carry the already reported failure: %+v", status) + } + + if err := h.waitForNone("Failed", 2500*time.Millisecond, failedWithMarker(marker)); err != nil { + t.Fatal(err) + } +} + +func TestTrack_StopsReportingWithdrawnReplicaFailure(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + rs := newReplicaSet("demo-111", "rs-uid-1") + h.createObject(t, gvrReplicaSets, rs) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedReplicaSetNamed(rs.Name)); err != nil { + t.Fatalf("wait for AddedReplicaSet: %v", err) + } + + const withdrawnMarker = "marker-withdrawn" + h.updateReplicaSet(t, withReplicaFailure(rs.DeepCopy(), "FailedCreate", withdrawnMarker)) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(withdrawnMarker)); err != nil { + t.Fatalf("wait for Failed: %v", err) + } + + h.updateReplicaSet(t, rs.DeepCopy()) + + dep := newNotReadyDeployment() + dep.Status.Replicas = 1 + h.updateDeployment(t, dep) + if err := h.waitForNone("Failed", 2500*time.Millisecond, failedWithMarker(withdrawnMarker)); err != nil { + t.Fatal(err) + } + + const rearmedMarker = "marker-rearmed" + h.updateReplicaSet(t, withReplicaFailure(rs.DeepCopy(), "FailedCreate", rearmedMarker)) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(rearmedMarker)); err != nil { + t.Fatalf("failure must be reported again after the condition reappeared: %v", err) + } +} + +func TestTrack_IgnoresStaleModifiedDeliveredAfterReplicaSetDeletion(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + createdAt := time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC) + rsOld := withReplicaSetCreationTimestamp(newReplicaSet("demo-111", "rs-uid-old"), createdAt) + h.createObject(t, gvrReplicaSets, rsOld) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedReplicaSetNamed(rsOld.Name)); err != nil { + t.Fatalf("wait for AddedReplicaSet: %v", err) + } + + h.injectReplicaSetDeletedAndWait(t, rsOld.DeepCopy()) + + rsCurrent := withReplicaSetCreationTimestamp(newReplicaSet("demo-222", "rs-uid-current"), createdAt.Add(time.Minute)) + h.createObject(t, gvrReplicaSets, rsCurrent) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedReplicaSetNamed(rsCurrent.Name)); err != nil { + t.Fatalf("wait for AddedReplicaSet: %v", err) + } + + const staleMarker = "marker-stale" + h.tracker.TestOnlyInjectReplicaSetModified(withReplicaFailure(rsOld.DeepCopy(), "FailedCreate", staleMarker)) + if err := h.waitForNone("Failed", 2500*time.Millisecond, failedWithMarker(staleMarker)); err != nil { + t.Fatal(err) + } + + const currentMarker = "marker-current" + h.updateReplicaSet(t, withReplicaFailure(rsCurrent.DeepCopy(), "FailedCreate", currentMarker)) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(currentMarker)); err != nil { + t.Fatalf("current ReplicaSet must remain tracked after the stale event: %v", err) + } +} + +func TestTrack_ReportsFailureOfReplicaSetPromotedByDeletion(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + createdAt := time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC) + current := withReplicaSetCreationTimestamp(newReplicaSet("demo-111", "rs-uid-current"), createdAt) + h.createObject(t, gvrReplicaSets, current) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedNewReplicaSet(current.Name)); err != nil { + t.Fatalf("wait for current AddedReplicaSet: %v", err) + } + + const marker = "marker-promoted-by-deletion" + duplicate := withReplicaFailure( + withReplicaSetCreationTimestamp(newReplicaSet("demo-222", "rs-uid-duplicate"), createdAt.Add(time.Minute)), + "FailedCreate", + marker, + ) + h.createObject(t, gvrReplicaSets, duplicate) + if _, err := h.waitFor("AddedReplicaSet", 10*time.Second, addedReplicaSetNamed(duplicate.Name)); err != nil { + t.Fatalf("wait for duplicate AddedReplicaSet: %v", err) + } + + if err := h.waitForNone("Failed", 2500*time.Millisecond, failedWithMarker(marker)); err != nil { + t.Fatalf("a ReplicaSet that is not the new one must not fail the rollout: %v", err) + } + + h.deleteObject(t, gvrReplicaSets, current.Name) + + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(marker)); err != nil { + t.Fatalf("failure of the ReplicaSet promoted by the deletion must be reported: %v", err) + } +} + +func TestTrack_PreservesReplicaSetEventOrderAcrossChannels(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + rs := newReplicaSet("demo-111", "rs-uid-1") + const initialMarker = "marker-initial-failure" + const recurringMarker = "marker-recurring-failure" + + // Occupies the tracker so the events below arrive while it is busy: only then can + // buffered fan-out channels reorder them and make this test a discriminator. + h.tracker.TestOnlyInjectResourceFailure("keeps the tracker busy") + h.tracker.TestOnlyInjectReplicaSetAdded(withReplicaFailure(rs.DeepCopy(), "FailedCreate", initialMarker)) + h.tracker.TestOnlyInjectReplicaSetModified(rs.DeepCopy()) + h.tracker.TestOnlyInjectReplicaSetModified(withReplicaFailure(rs.DeepCopy(), "FailedCreate", recurringMarker)) + + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(initialMarker)); err != nil { + t.Fatalf("initial failure must be reported despite fan-out across channels: %v", err) + } + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(recurringMarker)); err != nil { + t.Fatalf("failure recurring after recovery must be reported anew: %v", err) + } +} + func addedReplicaSetNamed(name string) func(observation) bool { return func(ev observation) bool { report, ok := ev.Data.(deployment.ReplicaSetAddedReport) @@ -173,6 +486,13 @@ func addedReplicaSetNamed(name string) func(observation) bool { } } +func statusWithReplicas(replicas int32) func(observation) bool { + return func(ev observation) bool { + status, ok := ev.Data.(deployment.DeploymentStatus) + return ok && status.Replicas == replicas + } +} + func withReplicaSetControllerUID(rs *appsv1.ReplicaSet, uid types.UID) *appsv1.ReplicaSet { rs.OwnerReferences[0].UID = uid return rs diff --git a/pkg/tracker/deployment/tracker_test.go b/pkg/tracker/deployment/tracker_test.go index 612e3d17..7fa279bd 100644 --- a/pkg/tracker/deployment/tracker_test.go +++ b/pkg/tracker/deployment/tracker_test.go @@ -200,6 +200,23 @@ func TestTrack_FailsOnReplicaFailureWithEventsDisabled(t *testing.T) { } } +func TestTrack_ReportsEventFailureAsCounted(t *testing.T) { + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{newNotReadyDeployment()}, + }) + + if _, err := h.waitFor("Added", 10*time.Second, nil); err != nil { + t.Fatalf("wait for Added: %v", err) + } + + const marker = "marker-counted-event-failure" + h.tracker.TestOnlyInjectResourceFailure(marker) + + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMode(marker, deployment.FailureModeCounted)); err != nil { + t.Fatalf("event failure must remain subject to the error budget: %v", err) + } +} + func TestTrack_FailsOnRecreatedReplicaSetWithSameNameNewUID(t *testing.T) { h := newHarness(t, harnessConfig{ dynamicSeed: []runtime.Object{newNotReadyDeployment()}, diff --git a/pkg/tracker/replicaset/informer.go b/pkg/tracker/replicaset/informer.go index f271162c..196731be 100644 --- a/pkg/tracker/replicaset/informer.go +++ b/pkg/tracker/replicaset/informer.go @@ -44,11 +44,13 @@ type ReplicaSetPodError struct { // ReplicaSetInformer monitor ReplicaSet events to use with controllers (Deployment, StatefulSet, DaemonSet) type ReplicaSetInformer struct { tracker.Tracker - Controller utils.ControllerMetadata - ReplicaSetAdded chan *appsv1.ReplicaSet - ReplicaSetModified chan *appsv1.ReplicaSet - ReplicaSetDeleted chan *appsv1.ReplicaSet - Errors chan error + Controller utils.ControllerMetadata + ReplicaSetAdded chan *appsv1.ReplicaSet + ReplicaSetModified chan *appsv1.ReplicaSet + ReplicaSetDeleted chan *appsv1.ReplicaSet + replicaSetUnselected chan *appsv1.ReplicaSet + replicaSetSynced chan struct{} + Errors chan error } func NewReplicaSetInformer(trk *tracker.Tracker, controller utils.ControllerMetadata) *ReplicaSetInformer { @@ -81,7 +83,29 @@ func (r *ReplicaSetInformer) WithChannels(added chan *appsv1.ReplicaSet, return r } +// WithUnselectedChannel reports ReplicaSets that stop matching the controller selector +// without being deleted. Consumers can remove them without permanently tombstoning the UID. +func (r *ReplicaSetInformer) WithUnselectedChannel(unselected chan *appsv1.ReplicaSet) *ReplicaSetInformer { + r.replicaSetUnselected = unselected + return r +} + +// WithSyncedChannel reports when this handler has consumed its initial ReplicaSet list. +func (r *ReplicaSetInformer) WithSyncedChannel(synced chan struct{}) *ReplicaSetInformer { + r.replicaSetSynced = synced + return r +} + func (r *ReplicaSetInformer) Run(ctx context.Context) (cleanupFn func(), err error) { + // The event channels may be unbuffered, so a send with no receiver left blocks the + // shared informer goroutine forever: stop sending once the consumer is gone. + ctx, stopSending := context.WithCancel(ctx) + defer func() { + if err != nil { + stopSending() + } + }() + var inform *util.Concurrent[*informer.Informer] if err := r.InformerFactory.RWTransactionErr(func(factory *informer.InformerFactory) error { inform, err = factory.ForNamespace(schema.GroupVersionResource{ @@ -104,45 +128,58 @@ func (r *ReplicaSetInformer) Run(ctx context.Context) (cleanupFn func(), err err } if err := inform.RWTransactionErr(func(inf *informer.Informer) error { + toReplicaSet := func(obj interface{}) *appsv1.ReplicaSet { + if d, ok := obj.(cache.DeletedFinalStateUnknown); ok { + obj = d.Obj + } + + rsObj := &appsv1.ReplicaSet{} + lo.Must0(runtime.DefaultUnstructuredConverter.FromUnstructured(obj.(*unstructured.Unstructured).Object, rsObj)) + return rsObj + } + matches := func(rs *appsv1.ReplicaSet) bool { + return labelSelector.Matches(apilabels.Set(rs.GetLabels())) + } + handler, err := inf.AddEventHandler( - cache.FilteringResourceEventHandler{ - FilterFunc: func(obj interface{}) bool { - if d, ok := obj.(cache.DeletedFinalStateUnknown); ok { - obj = d.Obj + cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + rsObj := toReplicaSet(obj) + if matches(rsObj) { + sendReplicaSet(ctx, r.ReplicaSetAdded, rsObj) } - - rsObj := &appsv1.ReplicaSet{} - lo.Must0(runtime.DefaultUnstructuredConverter.FromUnstructured(obj.(*unstructured.Unstructured).Object, rsObj)) - return labelSelector.Matches(apilabels.Set(rsObj.GetLabels())) }, - Handler: cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - if d, ok := obj.(cache.DeletedFinalStateUnknown); ok { - obj = d.Obj + UpdateFunc: func(oldObj, newObj interface{}) { + oldReplicaSet := toReplicaSet(oldObj) + newReplicaSet := toReplicaSet(newObj) + oldMatches := matches(oldReplicaSet) + newMatches := matches(newReplicaSet) + + switch { + case oldReplicaSet.UID != newReplicaSet.UID: + if oldMatches { + sendReplicaSet(ctx, r.ReplicaSetDeleted, oldReplicaSet) } - - rsObj := &appsv1.ReplicaSet{} - lo.Must0(runtime.DefaultUnstructuredConverter.FromUnstructured(obj.(*unstructured.Unstructured).Object, rsObj)) - r.ReplicaSetAdded <- rsObj - }, - UpdateFunc: func(oldObj, newObj interface{}) { - if d, ok := newObj.(cache.DeletedFinalStateUnknown); ok { - newObj = d.Obj + if newMatches { + sendReplicaSet(ctx, r.ReplicaSetAdded, newReplicaSet) } - - rsObj := &appsv1.ReplicaSet{} - lo.Must0(runtime.DefaultUnstructuredConverter.FromUnstructured(newObj.(*unstructured.Unstructured).Object, rsObj)) - r.ReplicaSetModified <- rsObj - }, - DeleteFunc: func(obj interface{}) { - if d, ok := obj.(cache.DeletedFinalStateUnknown); ok { - obj = d.Obj + case oldMatches && newMatches: + sendReplicaSet(ctx, r.ReplicaSetModified, newReplicaSet) + case oldMatches: + if r.replicaSetUnselected != nil { + sendReplicaSet(ctx, r.replicaSetUnselected, newReplicaSet) + } else { + sendReplicaSet(ctx, r.ReplicaSetDeleted, oldReplicaSet) } - - rsObj := &appsv1.ReplicaSet{} - lo.Must0(runtime.DefaultUnstructuredConverter.FromUnstructured(obj.(*unstructured.Unstructured).Object, rsObj)) - r.ReplicaSetDeleted <- rsObj - }, + case newMatches: + sendReplicaSet(ctx, r.ReplicaSetAdded, newReplicaSet) + } + }, + DeleteFunc: func(obj interface{}) { + rsObj := toReplicaSet(obj) + if matches(rsObj) { + sendReplicaSet(ctx, r.ReplicaSetDeleted, rsObj) + } }, }, ) @@ -151,10 +188,23 @@ func (r *ReplicaSetInformer) Run(ctx context.Context) (cleanupFn func(), err err } cleanupFn = func() { + stopSending() inf.RemoveEventHandler(handler) } inf.Run() + if r.replicaSetSynced != nil { + go func() { + if !cache.WaitForCacheSync(ctx.Done(), handler.HasSynced) { + return + } + + select { + case r.replicaSetSynced <- struct{}{}: + case <-ctx.Done(): + } + }() + } return nil }); err != nil { @@ -163,3 +213,10 @@ func (r *ReplicaSetInformer) Run(ctx context.Context) (cleanupFn func(), err err return cleanupFn, nil } + +func sendReplicaSet(ctx context.Context, ch chan<- *appsv1.ReplicaSet, rsObj *appsv1.ReplicaSet) { + select { + case ch <- rsObj: + case <-ctx.Done(): + } +} diff --git a/pkg/tracker/replicaset/informer_lifecycle_test.go b/pkg/tracker/replicaset/informer_lifecycle_test.go new file mode 100644 index 00000000..34c17bf8 --- /dev/null +++ b/pkg/tracker/replicaset/informer_lifecycle_test.go @@ -0,0 +1,217 @@ +package replicaset + +import ( + "context" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + dynamicfake "k8s.io/client-go/dynamic/fake" + k8sscheme "k8s.io/client-go/kubernetes/scheme" + + "github.com/werf/kubedog/pkg/informer" + "github.com/werf/kubedog/pkg/tracker" + "github.com/werf/kubedog/pkg/utils" +) + +var lifecycleReplicaSetGVR = schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "replicasets"} + +type lifecycleInformerHarness struct { + ctx context.Context + dynamic dynamic.Interface + added chan *appsv1.ReplicaSet + modified chan *appsv1.ReplicaSet + deleted chan *appsv1.ReplicaSet + unselected chan *appsv1.ReplicaSet + synced chan struct{} +} + +func TestRun_whenInitialReplicaSetsAreConsumed(t *testing.T) { + replicaSet := lifecycleReplicaSet("demo-111", "rs-uid-1") + h := newLifecycleInformerHarness(t, replicaSet) + + select { + case <-h.synced: + t.Fatal("handler reported synced before its initial Add was consumed") + default: + } + + receiveReplicaSet(t, h.added) + select { + case <-h.synced: + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for handler sync") + } +} + +func TestRun_whenUpdateReplacesReplicaSetUID(t *testing.T) { + oldReplicaSet := lifecycleReplicaSet("demo-111", "rs-uid-old") + h := newLifecycleInformerHarness(t, oldReplicaSet) + + if got := receiveReplicaSet(t, h.added); got.UID != oldReplicaSet.UID { + t.Fatalf("initial Added UID = %q, want %q", got.UID, oldReplicaSet.UID) + } + + newReplicaSet := oldReplicaSet.DeepCopy() + newReplicaSet.UID = types.UID("rs-uid-new") + updateLifecycleReplicaSet(t, h, newReplicaSet) + + if got := receiveReplicaSet(t, h.deleted); got.UID != oldReplicaSet.UID { + t.Fatalf("replacement Deleted UID = %q, want %q", got.UID, oldReplicaSet.UID) + } + if got := receiveReplicaSet(t, h.added); got.UID != newReplicaSet.UID { + t.Fatalf("replacement Added UID = %q, want %q", got.UID, newReplicaSet.UID) + } +} + +func TestRun_whenReplacementStopsMatchingSelector(t *testing.T) { + oldReplicaSet := lifecycleReplicaSet("demo-111", "rs-uid-old") + h := newLifecycleInformerHarness(t, oldReplicaSet) + + if got := receiveReplicaSet(t, h.added); got.UID != oldReplicaSet.UID { + t.Fatalf("initial Added UID = %q, want %q", got.UID, oldReplicaSet.UID) + } + + replacement := oldReplicaSet.DeepCopy() + replacement.UID = types.UID("rs-uid-new") + replacement.Labels = map[string]string{"app": "other"} + updateLifecycleReplicaSet(t, h, replacement) + + if got := receiveReplicaSet(t, h.deleted); got.UID != oldReplicaSet.UID { + t.Fatalf("replacement Deleted UID = %q, want %q", got.UID, oldReplicaSet.UID) + } +} + +func TestRun_whenReplicaSetLeavesAndReentersSelector(t *testing.T) { + replicaSet := lifecycleReplicaSet("demo-111", "rs-uid-1") + h := newLifecycleInformerHarness(t, replicaSet) + + if got := receiveReplicaSet(t, h.added); got.UID != replicaSet.UID { + t.Fatalf("initial Added UID = %q, want %q", got.UID, replicaSet.UID) + } + + unselected := replicaSet.DeepCopy() + unselected.Labels = map[string]string{"app": "other"} + updateLifecycleReplicaSet(t, h, unselected) + if got := receiveReplicaSet(t, h.unselected); got.UID != replicaSet.UID { + t.Fatalf("Unselected UID = %q, want %q", got.UID, replicaSet.UID) + } + + reentered := replicaSet.DeepCopy() + updateLifecycleReplicaSet(t, h, reentered) + if got := receiveReplicaSet(t, h.added); got.UID != replicaSet.UID { + t.Fatalf("reentry Added UID = %q, want %q", got.UID, replicaSet.UID) + } +} + +func newLifecycleInformerHarness(t *testing.T, seed *appsv1.ReplicaSet) *lifecycleInformerHarness { + t.Helper() + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + dynClient := dynamicfake.NewSimpleDynamicClient(k8sscheme.Scheme, seed) + watchErrCh := make(chan error, 10) + factory := informer.NewConcurrentInformerFactory(ctx.Done(), watchErrCh, dynClient, informer.ConcurrentInformerFactoryOptions{}) + dep := lifecycleDeployment() + + h := &lifecycleInformerHarness{ + ctx: ctx, + dynamic: dynClient, + added: make(chan *appsv1.ReplicaSet), + modified: make(chan *appsv1.ReplicaSet), + deleted: make(chan *appsv1.ReplicaSet), + unselected: make(chan *appsv1.ReplicaSet), + synced: make(chan struct{}), + } + + rsInformer := NewReplicaSetInformer(&tracker.Tracker{ + Namespace: testNamespace, + ResourceName: dep.Name, + InformerFactory: factory, + }, utils.ControllerAccessor(dep)).WithChannels(h.added, h.modified, h.deleted, watchErrCh). + WithUnselectedChannel(h.unselected). + WithSyncedChannel(h.synced) + + cleanup, err := rsInformer.Run(ctx) + if err != nil { + t.Fatalf("run ReplicaSet informer: %v", err) + } + t.Cleanup(cleanup) + + return h +} + +func updateLifecycleReplicaSet(t *testing.T, h *lifecycleInformerHarness, rs *appsv1.ReplicaSet) { + t.Helper() + + current, err := h.dynamic.Resource(lifecycleReplicaSetGVR).Namespace(testNamespace).Get(h.ctx, rs.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("get ReplicaSet: %v", err) + } + rs.ResourceVersion = current.GetResourceVersion() + + object, err := runtime.DefaultUnstructuredConverter.ToUnstructured(rs) + if err != nil { + t.Fatalf("convert ReplicaSet: %v", err) + } + if _, err := h.dynamic.Resource(lifecycleReplicaSetGVR).Namespace(testNamespace).Update( + h.ctx, + &unstructured.Unstructured{Object: object}, + metav1.UpdateOptions{}, + ); err != nil { + t.Fatalf("update ReplicaSet: %v", err) + } +} + +func receiveReplicaSet(t *testing.T, ch <-chan *appsv1.ReplicaSet) *appsv1.ReplicaSet { + t.Helper() + + select { + case rs := <-ch: + return rs + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for ReplicaSet event") + return nil + } +} + +func lifecycleDeployment() *appsv1.Deployment { + return &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "demo", Namespace: testNamespace, UID: types.UID("dep-uid-1")}, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "demo"}}, + }, + } +} + +func lifecycleReplicaSet(name string, uid types.UID) *appsv1.ReplicaSet { + dep := lifecycleDeployment() + return &appsv1.ReplicaSet{ + TypeMeta: metav1.TypeMeta{Kind: "ReplicaSet", APIVersion: "apps/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: testNamespace, + UID: uid, + Labels: map[string]string{"app": "demo"}, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "apps/v1", Kind: "Deployment", Name: dep.Name, UID: dep.UID, Controller: ptrTo(true), + }}, + }, + Spec: appsv1.ReplicaSetSpec{ + Selector: dep.Spec.Selector, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "demo"}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "c", Image: "busybox:1"}}}, + }, + }, + } +} diff --git a/pkg/tracker/replicaset/informer_test.go b/pkg/tracker/replicaset/informer_test.go new file mode 100644 index 00000000..97a4bc05 --- /dev/null +++ b/pkg/tracker/replicaset/informer_test.go @@ -0,0 +1,115 @@ +package replicaset + +import ( + "context" + "runtime" + "strings" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + dynamicfake "k8s.io/client-go/dynamic/fake" + k8sscheme "k8s.io/client-go/kubernetes/scheme" + + "github.com/werf/kubedog/pkg/informer" + "github.com/werf/kubedog/pkg/tracker" + "github.com/werf/kubedog/pkg/utils" +) + +const testNamespace = "default" + +// callbackFrame matches a goroutine parked inside an event callback. Matching the file +// instead of a function name keeps the check honest whether the send sits in the +// callback itself or in a helper. +const callbackFrame = "pkg/tracker/replicaset/informer.go:" + +func TestRun_UnblocksEventCallbackOnCleanup(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + dep := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "demo", Namespace: testNamespace, UID: types.UID("dep-uid-1")}, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "demo"}}, + }, + } + rs := &appsv1.ReplicaSet{ + TypeMeta: metav1.TypeMeta{Kind: "ReplicaSet", APIVersion: "apps/v1"}, + ObjectMeta: metav1.ObjectMeta{ + Name: "demo-111", + Namespace: testNamespace, + UID: types.UID("rs-uid-1"), + Labels: map[string]string{"app": "demo"}, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "apps/v1", Kind: "Deployment", Name: dep.Name, UID: dep.UID, Controller: ptrTo(true), + }}, + }, + Spec: appsv1.ReplicaSetSpec{ + Selector: dep.Spec.Selector, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "demo"}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "c", Image: "busybox:1"}}}, + }, + }, + } + + dynClient := dynamicfake.NewSimpleDynamicClient(k8sscheme.Scheme, rs) + watchErrCh := make(chan error, 10) + factory := informer.NewConcurrentInformerFactory(ctx.Done(), watchErrCh, dynClient, informer.ConcurrentInformerFactoryOptions{}) + + rsInformer := NewReplicaSetInformer(&tracker.Tracker{ + Namespace: testNamespace, + ResourceName: dep.Name, + InformerFactory: factory, + }, utils.ControllerAccessor(dep)).WithChannels( + make(chan *appsv1.ReplicaSet), + make(chan *appsv1.ReplicaSet), + make(chan *appsv1.ReplicaSet), + make(chan error), + ) + + cleanupFn, err := rsInformer.Run(ctx) + if err != nil { + t.Fatalf("run replicaset informer: %v", err) + } + + if !waitForCallbackParked(true, 10*time.Second) { + t.Fatal("no event callback parked on send: the test cannot tell whether cleanup releases one") + } + + cleanupFn() + + if !waitForCallbackParked(false, 10*time.Second) { + t.Fatal("event callback still parked 10s after cleanup: the shared informer goroutine is wedged") + } +} + +func waitForCallbackParked(want bool, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for { + if callbackParked() == want { + return true + } + if time.Now().After(deadline) { + return false + } + time.Sleep(10 * time.Millisecond) + } +} + +func callbackParked() bool { + buf := make([]byte, 64*1024) + for { + n := runtime.Stack(buf, true) + if n < len(buf) { + return strings.Contains(string(buf[:n]), callbackFrame) + } + buf = make([]byte, 2*len(buf)) + } +} + +func ptrTo[T any](v T) *T { return &v } diff --git a/pkg/utils/deployment_utils.go b/pkg/utils/deployment_utils.go index 01fac59e..68be3afe 100644 --- a/pkg/utils/deployment_utils.go +++ b/pkg/utils/deployment_utils.go @@ -11,10 +11,8 @@ import ( ) // FindNewReplicaSet returns the ReplicaSet the Deployment is currently rolling out. -// -// Only ReplicaSets controlled by the Deployment are considered: label selectors are -// not unique, so a ReplicaSet of another Deployment or of a previous incarnation of -// this one may match both the labels and the pod template. +// Only controlled ReplicaSets are considered: label selectors are not unique, so a +// ReplicaSet of another Deployment may match both the labels and the pod template. func FindNewReplicaSet(deployment *appsv1.Deployment, rsList []*appsv1.ReplicaSet) (*appsv1.ReplicaSet, error) { newRSTemplate := GetNewReplicaSetTemplate(deployment) ownedRSList := controlledReplicaSets(deployment, rsList) @@ -31,9 +29,9 @@ func FindNewReplicaSet(deployment *appsv1.Deployment, rsList []*appsv1.ReplicaSe return nil, nil } -// controlledReplicaSets keeps only the ReplicaSets whose controller reference points -// at the given Deployment. A ReplicaSet without a controller reference is not -// adopted, just like the Deployment controller itself does not adopt one. +// controlledReplicaSets keeps only the ReplicaSets owned by the Deployment. Adoption of an +// orphan is left to the Deployment controller: tracking one before it happens would report +// a ReplicaSet that may never become ours. func controlledReplicaSets(deployment *appsv1.Deployment, rsList []*appsv1.ReplicaSet) []*appsv1.ReplicaSet { ownedRSList := make([]*appsv1.ReplicaSet, 0, len(rsList)) for _, rs := range rsList { From 8ceef4c3c442798b122a1dd5412ef98baf626243 Mon Sep 17 00:00:00 2001 From: Ilya Lesikov Date: Tue, 27 Oct 2026 14:55:18 +0300 Subject: [PATCH 3/5] fix: keep ReplicaSet quota failures counted while the rollout can release quota The classifier subtracted every unavailable new pod, so a rollout whose pods were created but not ready yet, or were terminating, was failed as stuck. It also trusted the FailedCreate condition, which outlives the shortage while the ReplicaSet controller backs off. Assume existing new pods become available, confirm a deadlock with a fresh pod list and the named ResourceQuota, and report it only once it has held for ten seconds. Signed-off-by: Ilya Lesikov --- ...c_readiness_tracker_replicafailure_test.go | 17 +- pkg/tracker/deployment/export_test.go | 10 +- pkg/tracker/deployment/harness_test.go | 33 ++- pkg/tracker/deployment/replicaset_failure.go | 198 ++++++++++++- .../deployment/replicaset_failure_test.go | 63 ++++- pkg/tracker/deployment/tracker.go | 18 +- .../deployment/tracker_quota_recovery_test.go | 266 ++++++++++++++++++ .../deployment/tracker_replicaset_test.go | 13 +- 8 files changed, 596 insertions(+), 22 deletions(-) diff --git a/pkg/dyntracker/dynamic_readiness_tracker_replicafailure_test.go b/pkg/dyntracker/dynamic_readiness_tracker_replicafailure_test.go index c393e8dc..bbadf7cd 100644 --- a/pkg/dyntracker/dynamic_readiness_tracker_replicafailure_test.go +++ b/pkg/dyntracker/dynamic_readiness_tracker_replicafailure_test.go @@ -11,6 +11,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" dynamicfake "k8s.io/client-go/dynamic/fake" @@ -27,7 +28,7 @@ import ( var replicaFailureDeploymentGVK = schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"} func TestDynamicReadinessTracker_SingleDurableReplicaFailure_TerminatesPromptly(t *testing.T) { - const boundedTimeout = 8 * time.Second + const boundedTimeout = 60 * time.Second tests := []struct { name string @@ -41,7 +42,7 @@ func TestDynamicReadinessTracker_SingleDurableReplicaFailure_TerminatesPromptly( t.Run(tt.name, func(t *testing.T) { dynClient := dynamicfake.NewSimpleDynamicClient(k8sscheme.Scheme, replicaFailureOneReplicaDeployment(), replicaFailureFailedCreateReplicaSet()) - kubeClient := k8sfake.NewSimpleClientset() + kubeClient := k8sfake.NewSimpleClientset(replicaFailureExhaustedQuota()) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -147,13 +148,23 @@ func replicaFailureFailedCreateReplicaSet() *appsv1.ReplicaSet { Type: appsv1.ReplicaSetReplicaFailure, Status: corev1.ConditionTrue, Reason: "FailedCreate", - Message: `pods "demo-111" is forbidden: exceeded quota: compute-quota`, + Message: `pods "demo-111" is forbidden: exceeded quota: compute-quota, requested: pods=1, used: pods=1, limited: pods=1`, LastTransitionTime: metav1.Now(), }}, }, } } +func replicaFailureExhaustedQuota() *corev1.ResourceQuota { + return &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{Name: "compute-quota", Namespace: "default"}, + Status: corev1.ResourceQuotaStatus{ + Hard: corev1.ResourceList{corev1.ResourcePods: resource.MustParse("1")}, + Used: corev1.ResourceList{corev1.ResourcePods: resource.MustParse("1")}, + }, + } +} + type replicaFailureStubRESTMapper struct{} func (replicaFailureStubRESTMapper) KindFor(schema.GroupVersionResource) (schema.GroupVersionKind, error) { diff --git a/pkg/tracker/deployment/export_test.go b/pkg/tracker/deployment/export_test.go index e3bcbb4a..1f0f04c4 100644 --- a/pkg/tracker/deployment/export_test.go +++ b/pkg/tracker/deployment/export_test.go @@ -1,6 +1,10 @@ package deployment -import appsv1 "k8s.io/api/apps/v1" +import ( + "time" + + appsv1 "k8s.io/api/apps/v1" +) func (d *Tracker) TestOnlyInjectReplicaSetAdded(rs *appsv1.ReplicaSet) { d.replicaSetAdded <- rs @@ -25,3 +29,7 @@ func (d *Tracker) TestOnlyReplicaSetDeletedChanCap() int { func (d *Tracker) TestOnlyInjectResourceFailure(reason string) { d.resourceFailed <- reason } + +func (d *Tracker) TestOnlySetQuotaDeadlockConfirmation(confirmation time.Duration) { + d.quotaDeadlockConfirmation = confirmation +} diff --git a/pkg/tracker/deployment/harness_test.go b/pkg/tracker/deployment/harness_test.go index 9f27c20b..c98a7de3 100644 --- a/pkg/tracker/deployment/harness_test.go +++ b/pkg/tracker/deployment/harness_test.go @@ -11,6 +11,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -51,11 +52,16 @@ type observation struct { type harnessConfig struct { dynamicSeed []runtime.Object prepareDynamicClient func(*dynamicfake.FakeDynamicClient) + kubeSeed []runtime.Object + prepareKubeClient func(*k8sfake.Clientset) + // quotaDeadlockConfirmation defaults to a short period to keep fatal quota tests fast. + quotaDeadlockConfirmation time.Duration } type harness struct { ctx context.Context dynamic dynamic.Interface + kube *k8sfake.Clientset tracker *deployment.Tracker events chan observation @@ -76,16 +82,29 @@ func newHarness(t *testing.T, cfg harnessConfig) *harness { if cfg.prepareDynamicClient != nil { cfg.prepareDynamicClient(dynClient) } - kubeClient := k8sfake.NewSimpleClientset() + kubeSeed := cfg.kubeSeed + if kubeSeed == nil { + kubeSeed = []runtime.Object{computeQuota(1, 1)} + } + kubeClient := k8sfake.NewSimpleClientset(kubeSeed...) + if cfg.prepareKubeClient != nil { + cfg.prepareKubeClient(kubeClient) + } watchErrCh := make(chan error, 100) factory := informer.NewConcurrentInformerFactory(ctx.Done(), watchErrCh, dynClient, informer.ConcurrentInformerFactoryOptions{}) trk := deployment.NewTracker(testDeploymentName, testNamespace, kubeClient, factory, tracker.Options{IgnoreLogs: true}) + quotaDeadlockConfirmation := cfg.quotaDeadlockConfirmation + if quotaDeadlockConfirmation == 0 { + quotaDeadlockConfirmation = 50 * time.Millisecond + } + trk.TestOnlySetQuotaDeadlockConfirmation(quotaDeadlockConfirmation) h := &harness{ ctx: ctx, dynamic: dynClient, + kube: kubeClient, tracker: trk, events: make(chan observation, 4096), done: make(chan struct{}), @@ -337,12 +356,22 @@ func withReplicaFailure(rs *appsv1.ReplicaSet, reason, marker string) *appsv1.Re Type: appsv1.ReplicaSetReplicaFailure, Status: corev1.ConditionTrue, Reason: reason, - Message: fmt.Sprintf("pods %q is forbidden: exceeded quota: compute-quota [%s]", rs.Name, marker), + Message: fmt.Sprintf("pods %q is forbidden: exceeded quota: compute-quota, requested: pods=1, used: pods=1, limited: pods=1 [%s]", rs.Name, marker), LastTransitionTime: metav1.Now(), }} return rs } +func computeQuota(hardPods, usedPods int64) *corev1.ResourceQuota { + return &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{Name: "compute-quota", Namespace: testNamespace}, + Status: corev1.ResourceQuotaStatus{ + Hard: corev1.ResourceList{corev1.ResourcePods: *resource.NewQuantity(hardPods, resource.DecimalSI)}, + Used: corev1.ResourceList{corev1.ResourcePods: *resource.NewQuantity(usedPods, resource.DecimalSI)}, + }, + } +} + func failedWithMarker(marker string) func(observation) bool { return failedWithMode(marker, deployment.FailureModeFatal) } diff --git a/pkg/tracker/deployment/replicaset_failure.go b/pkg/tracker/deployment/replicaset_failure.go index 3f8df401..a14e4a6b 100644 --- a/pkg/tracker/deployment/replicaset_failure.go +++ b/pkg/tracker/deployment/replicaset_failure.go @@ -3,11 +3,14 @@ package deployment import ( "context" "fmt" + "regexp" "strings" + "time" "github.com/samber/lo" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" @@ -16,6 +19,13 @@ import ( "github.com/werf/kubedog/pkg/utils" ) +const ( + rolloutAPIRequestTimeout = 10 * time.Second + defaultQuotaDeadlockConfirmation = 10 * time.Second +) + +var exceededQuotaPattern = regexp.MustCompile(`exceeded quota: ([^,\s]+), requested: (\S+), used: \S+, limited: \S+`) + func (d *Tracker) clearReplicaSetFailure(replicaSetUID types.UID) { if replicaSetUID == "" || d.failedFromReplicaSetUID != replicaSetUID { return @@ -33,10 +43,15 @@ func (d *Tracker) rearmReplicaSetFailure(rs *appsv1.ReplicaSet) { return } - delete(d.reportedReplicaSetFailures, rs.UID) + d.forgetReplicaSetFailure(rs.UID) d.clearReplicaSetFailure(rs.UID) } +func (d *Tracker) forgetReplicaSetFailure(replicaSetUID types.UID) { + delete(d.reportedReplicaSetFailures, replicaSetUID) + delete(d.quotaDeadlockSince, replicaSetUID) +} + func (d *Tracker) isStaleReplicaSet(rs *appsv1.ReplicaSet) bool { _, deleted := d.deletedReplicaSetUIDs[rs.UID] return deleted @@ -61,19 +76,29 @@ func (d *Tracker) handleNewReplicaSetFailure(ctx context.Context) error { } if deploymentStatusReady(d.lastObject) { - delete(d.reportedReplicaSetFailures, newReplicaSet.UID) + d.forgetReplicaSetFailure(newReplicaSet.UID) d.clearReplicaSetFailure(newReplicaSet.UID) return nil } - mode := replicaSetFailureMode(failure.Message, d.deploymentCanReleaseQuota(newReplicaSet)) // ReplicaSet controllers retain FailedCreate until a successful retry. Keep // the first classification until that condition clears so an old condition // cannot become fatal merely because an older ReplicaSet finished scaling down. if _, reported := d.reportedReplicaSetFailures[newReplicaSet.UID]; reported { return nil } + + deploymentCanRecover := true + if isQuotaFailure(failure.Message) { + var decided bool + deploymentCanRecover, decided = d.confirmQuotaDeadlock(ctx, newReplicaSet, failure) + if !decided { + return nil + } + } + d.reportedReplicaSetFailures[newReplicaSet.UID] = struct{}{} + mode := replicaSetFailureMode(failure.Message, deploymentCanRecover) return d.handleResourceFailure(ctx, resourceFailure{ reason: fmt.Sprintf("%s: %s", failure.Reason, failure.Message), @@ -83,18 +108,67 @@ func (d *Tracker) handleNewReplicaSetFailure(ctx context.Context) error { } func replicaSetFailureMode(message string, deploymentCanRecover bool) FailureMode { - message = strings.ToLower(message) - if strings.Contains(message, " is invalid:") || - (strings.Contains(message, "exceeded quota:") && !deploymentCanRecover) { + if strings.Contains(strings.ToLower(message), " is invalid:") || + (isQuotaFailure(message) && !deploymentCanRecover) { return FailureModeFatal } return FailureModeCounted } -func (d *Tracker) deploymentCanReleaseQuota(newReplicaSet *appsv1.ReplicaSet) bool { +func isQuotaFailure(message string) bool { + return strings.Contains(strings.ToLower(message), "exceeded quota:") +} + +// confirmQuotaDeadlock reports a deadlock only once it has held for a while: the checks read +// different objects at different moments, and quota usage drops only when the quota controller +// catches up with deleted pods. Until then the failure stays undecided. +func (d *Tracker) confirmQuotaDeadlock(ctx context.Context, newReplicaSet *appsv1.ReplicaSet, failure appsv1.ReplicaSetCondition) (deploymentCanRecover, decided bool) { + if d.deploymentCanReleaseQuota(ctx, newReplicaSet, failure) { + return true, true + } + + since, pending := d.quotaDeadlockSince[newReplicaSet.UID] + if !pending { + d.quotaDeadlockSince[newReplicaSet.UID] = time.Now() + time.AfterFunc(d.quotaDeadlockConfirmation, func() { + select { + case d.quotaDeadlockRecheck <- struct{}{}: + default: + } + }) + + return false, false + } + + return false, time.Since(since) >= d.quotaDeadlockConfirmation +} + +// deploymentCanReleaseQuota errs towards true: false fails the deploy immediately. +func (d *Tracker) deploymentCanReleaseQuota(ctx context.Context, newReplicaSet *appsv1.ReplicaSet, failure appsv1.ReplicaSetCondition) bool { + if d.rolloutCanReleaseQuota(newReplicaSet, newReplicaSet.Status.Replicas) { + return true + } + + // The status that brings the condition counts only the pods listed before the failed + // sync created any, and cached objects do not show terminating pods. + activeNewPods, releasing, err := d.inspectRolloutPods(ctx, newReplicaSet) + if err != nil || releasing || activeNewPods >= *newReplicaSet.Spec.Replicas || + d.rolloutCanReleaseQuota(newReplicaSet, activeNewPods) { + return true + } + + // The ReplicaSet controller keeps the condition across its backed-off retries, so the + // shortage it reports may be over already. + exceeded, err := d.quotaStillExceeded(ctx, failure.Message) + return err != nil || !exceeded +} + +// rolloutCanReleaseQuota assumes every existing pod of the new ReplicaSet becomes available +// and checks whether the Deployment controller would then scale an old ReplicaSet down. +func (d *Tracker) rolloutCanReleaseQuota(newReplicaSet *appsv1.ReplicaSet, activeNewPods int32) bool { oldReplicaSetsHaveDesiredReplicas := false - allDesiredReplicas := *newReplicaSet.Spec.Replicas + var oldDesiredReplicas int32 for _, rs := range d.knownReplicaSets { if rs.UID == newReplicaSet.UID || !metav1.IsControlledBy(rs, d.lastObject) { continue @@ -108,7 +182,7 @@ func (d *Tracker) deploymentCanReleaseQuota(newReplicaSet *appsv1.ReplicaSet) bo return true } - allDesiredReplicas += desiredReplicas + oldDesiredReplicas += desiredReplicas oldReplicaSetsHaveDesiredReplicas = oldReplicaSetsHaveDesiredReplicas || desiredReplicas > 0 } if !oldReplicaSetsHaveDesiredReplicas { @@ -144,9 +218,111 @@ func (d *Tracker) deploymentCanReleaseQuota(newReplicaSet *appsv1.ReplicaSet) bo maxUnavailable = int(desiredReplicas) } + // The Deployment controller scales old ReplicaSets down by at most + // sum(old spec) + new available - minimum available. minimumAvailable := desiredReplicas - int32(maxUnavailable) - newUnavailable := *newReplicaSet.Spec.Replicas - newReplicaSet.Status.AvailableReplicas - return allDesiredReplicas-minimumAvailable-newUnavailable > 0 + return oldDesiredReplicas+activeNewPods > minimumAvailable +} + +// inspectRolloutPods counts the active pods of the new ReplicaSet and reports whether any pod +// of the rollout is terminating or about to be scaled down, both of which release quota. +func (d *Tracker) inspectRolloutPods(ctx context.Context, newReplicaSet *appsv1.ReplicaSet) (activeNewPods int32, releasing bool, err error) { + selector, err := metav1.LabelSelectorAsSelector(d.lastObject.Spec.Selector) + if err != nil { + return 0, false, err + } + + listCtx, cancel := context.WithTimeout(ctx, rolloutAPIRequestTimeout) + defer cancel() + + // An unset resource version makes a quorum read, which sees the pods created + // before the ReplicaSet status carrying the condition was written. + pods, err := d.Kube.CoreV1().Pods(d.Namespace).List(listCtx, metav1.ListOptions{LabelSelector: selector.String()}) + if err != nil { + return 0, false, err + } + + desiredByReplicaSet := make(map[types.UID]int32) + for _, rs := range d.knownReplicaSets { + if metav1.IsControlledBy(rs, d.lastObject) { + desiredByReplicaSet[rs.UID] = *rs.Spec.Replicas + } + } + + activeByReplicaSet := make(map[types.UID]int32) + for i := range pods.Items { + pod := &pods.Items[i] + owner := metav1.GetControllerOf(pod) + if owner == nil { + continue + } + if _, owned := desiredByReplicaSet[owner.UID]; !owned { + continue + } + if pod.DeletionTimestamp != nil { + return 0, true, nil + } + if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed { + continue + } + + activeByReplicaSet[owner.UID]++ + } + + for replicaSetUID, active := range activeByReplicaSet { + if replicaSetUID != newReplicaSet.UID && active > desiredByReplicaSet[replicaSetUID] { + return 0, true, nil + } + } + + return activeByReplicaSet[newReplicaSet.UID], false, nil +} + +// quotaStillExceeded parses the message of the ResourceQuota admission plugin and reports +// whether the named quota still has no room for the request. +func (d *Tracker) quotaStillExceeded(ctx context.Context, message string) (bool, error) { + match := exceededQuotaPattern.FindStringSubmatch(message) + if match == nil { + return false, fmt.Errorf("parse quota failure %q", message) + } + + requested := make(corev1.ResourceList) + for _, item := range strings.Split(match[2], ",") { + name, value, found := strings.Cut(item, "=") + if !found { + return false, fmt.Errorf("parse requested resource %q", item) + } + + quantity, err := resource.ParseQuantity(value) + if err != nil { + return false, fmt.Errorf("parse requested quantity %q: %w", item, err) + } + + requested[corev1.ResourceName(name)] = quantity + } + + getCtx, cancel := context.WithTimeout(ctx, rolloutAPIRequestTimeout) + defer cancel() + + quota, err := d.Kube.CoreV1().ResourceQuotas(d.Namespace).Get(getCtx, match[1], metav1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("get resource quota %q: %w", match[1], err) + } + + for name, quantity := range requested { + hard, limited := quota.Status.Hard[name] + if !limited { + continue + } + + used := quota.Status.Used[name] + used.Add(quantity) + if used.Cmp(hard) > 0 { + return true, nil + } + } + + return false, nil } func replicaSetFailure(rs *appsv1.ReplicaSet) (appsv1.ReplicaSetCondition, bool) { diff --git a/pkg/tracker/deployment/replicaset_failure_test.go b/pkg/tracker/deployment/replicaset_failure_test.go index 0b0cd054..e73feb9d 100644 --- a/pkg/tracker/deployment/replicaset_failure_test.go +++ b/pkg/tracker/deployment/replicaset_failure_test.go @@ -1,6 +1,16 @@ package deployment -import "testing" +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sfake "k8s.io/client-go/kubernetes/fake" + + "github.com/werf/kubedog/pkg/tracker" +) func TestReplicaSetFailureMode_whenFailedCreateMessageIsClassified(t *testing.T) { tests := []struct { @@ -50,3 +60,54 @@ func TestReplicaSetFailureMode_whenFailedCreateMessageIsClassified(t *testing.T) }) } } + +func TestQuotaStillExceeded_whenQuotaStatusChanged(t *testing.T) { + const message = `pods "demo-111" is forbidden: exceeded quota: compute-quota, requested: limits.cpu=750m,limits.memory=1Gi, used: limits.cpu=3600m,limits.memory=3Gi, limited: limits.cpu=4,limits.memory=4Gi` + + tests := []struct { + name string + message string + used corev1.ResourceList + want bool + wantErr bool + }{ + { + name: "one requested resource still exceeds the quota", + message: message, + used: corev1.ResourceList{corev1.ResourceLimitsCPU: resource.MustParse("3600m"), corev1.ResourceLimitsMemory: resource.MustParse("1Gi")}, + want: true, + }, + { + name: "quota has room for every requested resource", + message: message, + used: corev1.ResourceList{corev1.ResourceLimitsCPU: resource.MustParse("3250m"), corev1.ResourceLimitsMemory: resource.MustParse("3Gi")}, + want: false, + }, + { + name: "message is not a quota admission failure", + message: `pods "demo-111" is forbidden: exceeded quota: compute-quota`, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + quota := &corev1.ResourceQuota{ + ObjectMeta: metav1.ObjectMeta{Name: "compute-quota", Namespace: "default"}, + Status: corev1.ResourceQuotaStatus{ + Hard: corev1.ResourceList{corev1.ResourceLimitsCPU: resource.MustParse("4"), corev1.ResourceLimitsMemory: resource.MustParse("4Gi")}, + Used: tt.used, + }, + } + d := &Tracker{Tracker: tracker.Tracker{Kube: k8sfake.NewSimpleClientset(quota), Namespace: "default"}} + + got, err := d.quotaStillExceeded(context.Background(), tt.message) + if (err != nil) != tt.wantErr { + t.Fatalf("quotaStillExceeded() error = %v, wantErr %t", err, tt.wantErr) + } + if got != tt.want { + t.Fatalf("quotaStillExceeded() = %t, want %t", got, tt.want) + } + }) + } +} diff --git a/pkg/tracker/deployment/tracker.go b/pkg/tracker/deployment/tracker.go index ec7ddd46..017fe9d0 100644 --- a/pkg/tracker/deployment/tracker.go +++ b/pkg/tracker/deployment/tracker.go @@ -57,6 +57,8 @@ type Tracker struct { lastObject *appsv1.Deployment failedFromReplicaSetUID types.UID reportedReplicaSetFailures map[types.UID]struct{} + quotaDeadlockSince map[types.UID]time.Time + quotaDeadlockConfirmation time.Duration replicaSetsSynced bool podStatuses map[string]pod.PodStatus rsNameByPod map[string]string @@ -87,6 +89,7 @@ type Tracker struct { replicaSetDeleted chan *appsv1.ReplicaSet replicaSetUnselected chan *appsv1.ReplicaSet replicaSetSynced chan struct{} + quotaDeadlockRecheck chan struct{} errors chan error podAddedRelay chan *corev1.Pod @@ -122,6 +125,8 @@ func NewTracker(name, namespace string, kube kubernetes.Interface, informerFacto knownReplicaSets: make(map[string]*appsv1.ReplicaSet), deletedReplicaSetUIDs: make(map[types.UID]struct{}), reportedReplicaSetFailures: make(map[types.UID]struct{}), + quotaDeadlockSince: make(map[types.UID]time.Time), + quotaDeadlockConfirmation: defaultQuotaDeadlockConfirmation, podStatuses: make(map[string]pod.PodStatus), rsNameByPod: make(map[string]string), @@ -141,6 +146,7 @@ func NewTracker(name, namespace string, kube kubernetes.Interface, informerFacto replicaSetDeleted: make(chan *appsv1.ReplicaSet), replicaSetUnselected: make(chan *appsv1.ReplicaSet), replicaSetSynced: make(chan struct{}), + quotaDeadlockRecheck: make(chan struct{}, 1), podAddedRelay: make(chan *corev1.Pod, 1), podStatusesRelay: make(chan map[string]pod.PodStatus, 10), @@ -190,6 +196,7 @@ func (d *Tracker) Track(ctx context.Context) (err error) { d.knownReplicaSets = make(map[string]*appsv1.ReplicaSet) d.deletedReplicaSetUIDs = make(map[types.UID]struct{}) d.reportedReplicaSetFailures = make(map[types.UID]struct{}) + d.quotaDeadlockSince = make(map[types.UID]time.Time) d.failedFromReplicaSetUID = "" d.podStatuses = make(map[string]pod.PodStatus) d.rsNameByPod = make(map[string]string) @@ -269,7 +276,7 @@ func (d *Tracker) Track(ctx context.Context) (err error) { if known, found := d.knownReplicaSets[rs.Name]; found && known.UID == rs.UID { delete(d.knownReplicaSets, rs.Name) } - delete(d.reportedReplicaSetFailures, rs.UID) + d.forgetReplicaSetFailure(rs.UID) d.clearReplicaSetFailure(rs.UID) @@ -285,7 +292,7 @@ func (d *Tracker) Track(ctx context.Context) (err error) { if known, found := d.knownReplicaSets[rs.Name]; found && known.UID == rs.UID { delete(d.knownReplicaSets, rs.Name) } - delete(d.reportedReplicaSetFailures, rs.UID) + d.forgetReplicaSetFailure(rs.UID) d.clearReplicaSetFailure(rs.UID) if d.lastObject != nil { @@ -300,6 +307,13 @@ func (d *Tracker) Track(ctx context.Context) (err error) { return err } + case <-d.quotaDeadlockRecheck: + if d.lastObject != nil { + if err := d.handleNewReplicaSetFailure(ctx); err != nil { + return err + } + } + case pod := <-d.podAddedRelay: rsName := utils.GetPodReplicaSetName(pod) d.rsNameByPod[pod.Name] = rsName diff --git a/pkg/tracker/deployment/tracker_quota_recovery_test.go b/pkg/tracker/deployment/tracker_quota_recovery_test.go index f2f6db71..589de978 100644 --- a/pkg/tracker/deployment/tracker_quota_recovery_test.go +++ b/pkg/tracker/deployment/tracker_quota_recovery_test.go @@ -1,15 +1,21 @@ package deployment_test import ( + "errors" "testing" "time" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" dynamicfake "k8s.io/client-go/dynamic/fake" + k8sfake "k8s.io/client-go/kubernetes/fake" k8stesting "k8s.io/client-go/testing" "github.com/werf/kubedog/pkg/tracker/deployment" @@ -106,6 +112,266 @@ func TestTrack_DoesNotEscalateStaleQuotaFailureAfterOldReplicaSetScalesDown(t *t } } +func TestTrack_ClassifiesQuotaFailureByWhetherTheRolloutCanReleaseQuota(t *testing.T) { + type rollout struct { + replicas, maxUnavailable, maxSurge int32 + oldSpec, newSpec, newStatusReplicas int32 + newPods []*corev1.Pod + quota *corev1.ResourceQuota + forbidQuota bool + } + + tests := []struct { + name string + rollout rollout + want deployment.FailureMode + }{ + { + // The pod created in the failed sync is missing from the status: the + // ReplicaSet controller counts the pods it listed before creating any. + name: "pod created before the failure can become available and scale an old pod down", + rollout: rollout{ + replicas: 3, maxUnavailable: 0, maxSurge: 2, + oldSpec: 3, newSpec: 2, newStatusReplicas: 0, + newPods: []*corev1.Pod{newReplicaSetPod("demo-new-1", "rs-uid-new")}, + }, + want: deployment.FailureModeCounted, + }, + { + name: "status already counts the pod created before the failure", + rollout: rollout{ + replicas: 3, maxUnavailable: 0, maxSurge: 2, + oldSpec: 3, newSpec: 2, newStatusReplicas: 1, + }, + want: deployment.FailureModeCounted, + }, + { + name: "terminating pod releases quota", + rollout: rollout{ + replicas: 3, maxUnavailable: 0, maxSurge: 1, + oldSpec: 2, newSpec: 2, newStatusReplicas: 1, + newPods: []*corev1.Pod{ + newReplicaSetPod("demo-new-1", "rs-uid-new"), + terminating(newReplicaSetPod("demo-old-1", "rs-uid-old")), + }, + }, + want: deployment.FailureModeCounted, + }, + { + // The ReplicaSet controller has not retried since an old pod went away. + name: "quota has room again", + rollout: rollout{ + replicas: 3, maxUnavailable: 0, maxSurge: 1, + oldSpec: 2, newSpec: 2, newStatusReplicas: 1, + newPods: []*corev1.Pod{newReplicaSetPod("demo-new-1", "rs-uid-new")}, + quota: computeQuota(4, 3), + }, + want: deployment.FailureModeCounted, + }, + { + name: "quota no longer exists", + rollout: rollout{ + replicas: 3, maxUnavailable: 0, maxSurge: 1, + oldSpec: 2, newSpec: 2, newStatusReplicas: 1, + newPods: []*corev1.Pod{newReplicaSetPod("demo-new-1", "rs-uid-new")}, + }, + want: deployment.FailureModeCounted, + }, + { + name: "quota cannot be read", + rollout: rollout{ + replicas: 3, maxUnavailable: 0, maxSurge: 1, + oldSpec: 2, newSpec: 2, newStatusReplicas: 1, + newPods: []*corev1.Pod{newReplicaSetPod("demo-new-1", "rs-uid-new")}, + quota: computeQuota(3, 3), + forbidQuota: true, + }, + want: deployment.FailureModeCounted, + }, + { + name: "new pod and old pods hold all of the quota", + rollout: rollout{ + replicas: 3, maxUnavailable: 0, maxSurge: 1, + oldSpec: 2, newSpec: 2, newStatusReplicas: 1, + newPods: []*corev1.Pod{newReplicaSetPod("demo-new-1", "rs-uid-new")}, + quota: computeQuota(3, 3), + }, + want: deployment.FailureModeFatal, + }, + { + // A cached ReplicaSet may still carry the condition after a retry created every pod. + name: "every desired new pod exists", + rollout: rollout{ + replicas: 2, maxUnavailable: 0, maxSurge: 1, + oldSpec: 0, newSpec: 2, newStatusReplicas: 1, + newPods: []*corev1.Pod{ + newReplicaSetPod("demo-new-1", "rs-uid-new"), + newReplicaSetPod("demo-new-2", "rs-uid-new"), + }, + quota: computeQuota(2, 2), + }, + want: deployment.FailureModeCounted, + }, + { + name: "no new pod can be created and no old pod may go away", + rollout: rollout{ + replicas: 3, maxUnavailable: 0, maxSurge: 2, + oldSpec: 3, newSpec: 2, newStatusReplicas: 0, + quota: computeQuota(3, 3), + }, + want: deployment.FailureModeFatal, + }, + { + name: "single replica with the default strategy cannot surge", + rollout: rollout{ + replicas: 1, maxUnavailable: 0, maxSurge: 1, + oldSpec: 1, newSpec: 1, newStatusReplicas: 0, + quota: computeQuota(1, 1), + }, + want: deployment.FailureModeFatal, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := tt.rollout + + deploymentObject := rollingDeployment(r.maxUnavailable, r.maxSurge) + deploymentObject.Spec.Replicas = ptrTo(r.replicas) + + oldReplicaSet := scalableOldReplicaSet() + oldReplicaSet.Spec.Replicas = ptrTo(r.oldSpec) + oldReplicaSet.Status.Replicas = r.oldSpec + oldReplicaSet.Status.AvailableReplicas = r.oldSpec + + const marker = "marker-quota-classification" + newReplicaSet := withReplicaFailure(newReplicaSet("demo-new", "rs-uid-new"), "FailedCreate", marker) + newReplicaSet.Spec.Replicas = ptrTo(r.newSpec) + newReplicaSet.Status.Replicas = r.newStatusReplicas + + kubeSeed := []runtime.Object{} + if r.quota != nil { + kubeSeed = append(kubeSeed, r.quota) + } + for _, pod := range r.newPods { + kubeSeed = append(kubeSeed, pod) + } + + var prepareKubeClient func(*k8sfake.Clientset) + if r.forbidQuota { + prepareKubeClient = forbidResourceQuotas + } + + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{deploymentObject}, + prepareDynamicClient: initialReplicaSetListReactor(t, newReplicaSet, oldReplicaSet), + kubeSeed: kubeSeed, + prepareKubeClient: prepareKubeClient, + }) + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMode(marker, tt.want)); err != nil { + t.Fatalf("want quota failure classified as %v: %v", tt.want, err) + } + }) + } +} + +func TestTrack_ConfirmsQuotaDeadlockBeforeReportingItFatal(t *testing.T) { + const confirmation = 2 * time.Second + + tests := []struct { + name string + releaseQuotaMeantime bool + want deployment.FailureMode + }{ + {name: "deadlock persists", want: deployment.FailureModeFatal}, + { + // The quota controller lowers the usage only after it notices a deleted pod. + name: "quota usage drops before the confirmation", + releaseQuotaMeantime: true, + want: deployment.FailureModeCounted, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + deploymentObject := rollingDeployment(0, 1) + deploymentObject.Spec.Replicas = ptrTo(int32(3)) + + oldReplicaSet := scalableOldReplicaSet() + oldReplicaSet.Spec.Replicas = ptrTo(int32(2)) + oldReplicaSet.Status.Replicas = 2 + oldReplicaSet.Status.AvailableReplicas = 2 + + const marker = "marker-quota-confirmation" + newReplicaSet := withReplicaFailure(newReplicaSet("demo-new", "rs-uid-new"), "FailedCreate", marker) + newReplicaSet.Spec.Replicas = ptrTo(int32(2)) + newReplicaSet.Status.Replicas = 1 + + h := newHarness(t, harnessConfig{ + dynamicSeed: []runtime.Object{deploymentObject}, + prepareDynamicClient: initialReplicaSetListReactor(t, newReplicaSet, oldReplicaSet), + kubeSeed: []runtime.Object{computeQuota(3, 3), newReplicaSetPod("demo-new-1", "rs-uid-new")}, + quotaDeadlockConfirmation: confirmation, + }) + + waitForQuotaRead(t, h) + if tt.releaseQuotaMeantime { + if _, err := h.kube.CoreV1().ResourceQuotas(testNamespace).UpdateStatus(h.ctx, computeQuota(3, 2), metav1.UpdateOptions{}); err != nil { + t.Fatalf("update quota status: %v", err) + } + } + + if err := h.waitForNone("Failed", confirmation/2, nil); err != nil { + t.Fatalf("quota failure must stay undecided until the deadlock is confirmed: %v", err) + } + if _, err := h.waitFor("Failed", 10*time.Second, failedWithMode(marker, tt.want)); err != nil { + t.Fatalf("want quota failure classified as %v after the confirmation: %v", tt.want, err) + } + }) + } +} + +func waitForQuotaRead(t *testing.T, h *harness) { + t.Helper() + + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + for _, action := range h.kube.Actions() { + if action.Matches("get", "resourcequotas") { + return + } + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("tracker did not read the resource quota") +} + +func forbidResourceQuotas(client *k8sfake.Clientset) { + client.PrependReactor("get", "resourcequotas", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden(corev1.Resource("resourcequotas"), "compute-quota", errors.New("denied")) + }) +} + +func newReplicaSetPod(name string, replicaSetUID types.UID) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: testNamespace, + Labels: map[string]string{"app": "demo"}, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "apps/v1", Kind: "ReplicaSet", Name: name, UID: replicaSetUID, Controller: ptrTo(true), + }}, + }, + Status: corev1.PodStatus{Phase: corev1.PodPending}, + } +} + +func terminating(pod *corev1.Pod) *corev1.Pod { + pod.DeletionTimestamp = ptrTo(metav1.Now()) + pod.Finalizers = []string{"example.com/keep"} + return pod +} + func rollingDeployment(maxUnavailable, maxSurge int32) *appsv1.Deployment { deploymentObject := newNotReadyDeployment() unavailable := intstr.FromInt32(maxUnavailable) diff --git a/pkg/tracker/deployment/tracker_replicaset_test.go b/pkg/tracker/deployment/tracker_replicaset_test.go index 01b7d2e3..532dba91 100644 --- a/pkg/tracker/deployment/tracker_replicaset_test.go +++ b/pkg/tracker/deployment/tracker_replicaset_test.go @@ -1,6 +1,7 @@ package deployment_test import ( + "fmt" "testing" "time" @@ -467,9 +468,11 @@ func TestTrack_PreservesReplicaSetEventOrderAcrossChannels(t *testing.T) { // Occupies the tracker so the events below arrive while it is busy: only then can // buffered fan-out channels reorder them and make this test a discriminator. h.tracker.TestOnlyInjectResourceFailure("keeps the tracker busy") - h.tracker.TestOnlyInjectReplicaSetAdded(withReplicaFailure(rs.DeepCopy(), "FailedCreate", initialMarker)) + // An invalid pod spec is classified at once, unlike a quota failure, which waits for the + // deadlock to be confirmed and is dropped if it clears in the meantime. + h.tracker.TestOnlyInjectReplicaSetAdded(withInvalidPodFailure(rs.DeepCopy(), initialMarker)) h.tracker.TestOnlyInjectReplicaSetModified(rs.DeepCopy()) - h.tracker.TestOnlyInjectReplicaSetModified(withReplicaFailure(rs.DeepCopy(), "FailedCreate", recurringMarker)) + h.tracker.TestOnlyInjectReplicaSetModified(withInvalidPodFailure(rs.DeepCopy(), recurringMarker)) if _, err := h.waitFor("Failed", 10*time.Second, failedWithMarker(initialMarker)); err != nil { t.Fatalf("initial failure must be reported despite fan-out across channels: %v", err) @@ -479,6 +482,12 @@ func TestTrack_PreservesReplicaSetEventOrderAcrossChannels(t *testing.T) { } } +func withInvalidPodFailure(rs *appsv1.ReplicaSet, marker string) *appsv1.ReplicaSet { + rs = withReplicaFailure(rs, "FailedCreate", marker) + rs.Status.Conditions[0].Message = fmt.Sprintf("Pod %q is invalid: spec.containers[0].image: Required value [%s]", rs.Name, marker) + return rs +} + func addedReplicaSetNamed(name string) func(observation) bool { return func(ev observation) bool { report, ok := ev.Data.(deployment.ReplicaSetAddedReport) From a20d88c9964e8cb064012bef9e3439270edd9722 Mon Sep 17 00:00:00 2001 From: Ilya Lesikov Date: Tue, 27 Oct 2026 14:55:18 +0300 Subject: [PATCH 4/5] chore: fix errorlint in deployment tracker test harness and document FailureMode field Signed-off-by: Ilya Lesikov --- pkg/tracker/deployment/harness_test.go | 2 +- pkg/tracker/deployment/status.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/tracker/deployment/harness_test.go b/pkg/tracker/deployment/harness_test.go index c98a7de3..d41751d1 100644 --- a/pkg/tracker/deployment/harness_test.go +++ b/pkg/tracker/deployment/harness_test.go @@ -215,7 +215,7 @@ func (h *harness) waitFor(kind string, timeout time.Duration, pred func(observat } h.backlog = append(h.backlog, ev) case <-h.done: - return observation{}, fmt.Errorf("%w (err=%v) while waiting for kind=%s", errTrackDone, h.getTrackErr(), kind) + return observation{}, fmt.Errorf("%w (err=%w) while waiting for kind=%s", errTrackDone, h.getTrackErr(), kind) case <-deadline.C: return observation{}, fmt.Errorf("%w after %s waiting for kind=%s", errTimeout, timeout, kind) } diff --git a/pkg/tracker/deployment/status.go b/pkg/tracker/deployment/status.go index d91a60c8..6f93b70b 100644 --- a/pkg/tracker/deployment/status.go +++ b/pkg/tracker/deployment/status.go @@ -34,7 +34,8 @@ type DeploymentStatus struct { IsReady bool IsFailed bool FailedReason string - FailureMode FailureMode + // FailureMode qualifies the failure when IsFailed is set. + FailureMode FailureMode Pods map[string]pod.PodStatus // New Pod belongs to the new ReplicaSet of the Deployment, From 8bcce864f0adbbec3733044e7d519f699eb18f6d Mon Sep 17 00:00:00 2001 From: Ilya Lesikov Date: Thu, 29 Oct 2026 04:04:30 +0300 Subject: [PATCH 5/5] fix: bound quota quantities parsed from a ReplicaSet failure message Anyone allowed to write the ReplicaSet status controls the FailedCreate message, and a quantity with a huge exponent kept the tracker busy for minutes in arbitrary-precision arithmetic. Accept only the bounded forms the quota admission plugin prints and count the failure otherwise. Signed-off-by: Ilya Lesikov --- pkg/tracker/deployment/replicaset_failure.go | 11 ++++++++++- pkg/tracker/deployment/replicaset_failure_test.go | 7 +++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/pkg/tracker/deployment/replicaset_failure.go b/pkg/tracker/deployment/replicaset_failure.go index a14e4a6b..7a3cfbeb 100644 --- a/pkg/tracker/deployment/replicaset_failure.go +++ b/pkg/tracker/deployment/replicaset_failure.go @@ -24,7 +24,12 @@ const ( defaultQuotaDeadlockConfirmation = 10 * time.Second ) -var exceededQuotaPattern = regexp.MustCompile(`exceeded quota: ([^,\s]+), requested: (\S+), used: \S+, limited: \S+`) +var ( + exceededQuotaPattern = regexp.MustCompile(`exceeded quota: ([^,\s]+), requested: (\S+), used: \S+, limited: \S+`) + // boundedQuantityPattern admits the canonical quantities the quota admission plugin prints + // and keeps their exponent small: parsing a huge one takes minutes. + boundedQuantityPattern = regexp.MustCompile(`^[0-9]{1,30}(\.[0-9]{1,30})?([numkMGTPE]|[KMGTPE]i|[eE][+-]?[0-9]{1,2})?$`) +) func (d *Tracker) clearReplicaSetFailure(replicaSetUID types.UID) { if replicaSetUID == "" || d.failedFromReplicaSetUID != replicaSetUID { @@ -293,6 +298,10 @@ func (d *Tracker) quotaStillExceeded(ctx context.Context, message string) (bool, return false, fmt.Errorf("parse requested resource %q", item) } + if !boundedQuantityPattern.MatchString(value) { + return false, fmt.Errorf("parse requested quantity %q", item) + } + quantity, err := resource.ParseQuantity(value) if err != nil { return false, fmt.Errorf("parse requested quantity %q: %w", item, err) diff --git a/pkg/tracker/deployment/replicaset_failure_test.go b/pkg/tracker/deployment/replicaset_failure_test.go index e73feb9d..b7f9799c 100644 --- a/pkg/tracker/deployment/replicaset_failure_test.go +++ b/pkg/tracker/deployment/replicaset_failure_test.go @@ -83,6 +83,13 @@ func TestQuotaStillExceeded_whenQuotaStatusChanged(t *testing.T) { used: corev1.ResourceList{corev1.ResourceLimitsCPU: resource.MustParse("3250m"), corev1.ResourceLimitsMemory: resource.MustParse("3Gi")}, want: false, }, + { + // Anyone allowed to write the ReplicaSet status controls the message, and parsing + // such a quantity takes the arbitrary-precision arithmetic minutes. + name: "requested quantity is out of the range the quota admission plugin prints", + message: `pods "demo-111" is forbidden: exceeded quota: compute-quota, requested: limits.cpu=1e-1000000000, used: limits.cpu=0, limited: limits.cpu=4`, + wantErr: true, + }, { name: "message is not a quota admission failure", message: `pods "demo-111" is forbidden: exceeded quota: compute-quota`,