feat: add node_readiness_rules with enforcement_mode and dry_run labels

docs/TEST_README.md

@@ -241,6 +241,9 @@ After running the test scenario, you should see the following metrics:
    ```bash
    # Number of active rules
    curl -s http://localhost:8080/metrics | grep "node_readiness_rules_total"
+
+   # Number of active rules by enforcement mode and dry-run state
+   curl -s http://localhost:8080/metrics | grep "node_readiness_rules{"
    ```
 
 2. **Taint Operations:**
@@ -296,6 +299,9 @@ After completing Steps 1-9, verify the metrics reflect the test scenario:
 # Should show 1 rule (network-readiness-rule)
 curl -s http://localhost:8080/metrics | grep 'node_readiness_rules_total'
 
+# Should show 1 rule under its enforcement_mode/dry_run combination
+curl -s http://localhost:8080/metrics | grep 'node_readiness_rules{'
+
 # Should show taint removal operations for worker2, worker3, worker4
 curl -s http://localhost:8080/metrics | grep 'node_readiness_taint_operations_total{.*operation="remove"}'
 

docs/book/src/operations/monitoring.md

@@ -10,6 +10,8 @@ The controller serves metrics on `/metrics` only when metrics are explicitly ena
 
 ### `node_readiness_rules_total`
 
+***Deprecated:** use [`node_readiness_rules`](#node_readiness_rules) instead. It provides the same rule count with additional `enforcement_mode` and `dry_run` labels. `node_readiness_rules_total` is still published for compatibility.*
+
 Number of `NodeReadinessRule` objects tracked by the controller.
 
 | Property | Value |
@@ -18,6 +20,23 @@ Number of `NodeReadinessRule` objects tracked by the controller.
 | Labels | none |
 | Recorded when | The controller refreshes or removes a tracked rule |
 
+### `node_readiness_rules`
+
+Number of `NodeReadinessRule` objects tracked by the controller by enforcement mode and dry-run state.
+
+| Property | Value |
+| --- | --- |
+| Type | `gauge` |
+| Labels | `enforcement_mode`, `dry_run` |
+| Recorded when | Computed on each Prometheus scrape from the cached rule list |
+
+#### Labels
+
+| Label | Description | Values |
+| --- | --- | --- |
+| `enforcement_mode` | Enforcement mode of the rule | `bootstrap-only`, `continuous` |
+| `dry_run` | Whether the rule is in dry-run mode | `true`, `false` |
+
 ### `node_readiness_taint_operations_total`
 
 Total number of taint operations performed by the controller.

internal/controller/helper_unit_test.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 
 	readinessv1alpha1 "sigs.k8s.io/node-readiness-controller/api/v1alpha1"
+	"sigs.k8s.io/node-readiness-controller/internal/metrics"
 )
 
 func TestBootstrapAnnotationKey(t *testing.T) {
@@ -303,3 +304,41 @@ func TestApplyNodeStatusDelta(t *testing.T) {
 		g.Expect(rule.Status.FailedNodes).To(BeEmpty())
 	})
 }
+
+func TestListRuleInventory(t *testing.T) {
+	g := NewWithT(t)
+
+	c := &RuleReadinessController{}
+	ctx := t.Context()
+
+	newRule := func(name string, mode readinessv1alpha1.EnforcementMode, dryRun bool, deleting bool) *readinessv1alpha1.NodeReadinessRule {
+		rule := &readinessv1alpha1.NodeReadinessRule{
+			ObjectMeta: metav1.ObjectMeta{Name: name},
+			Spec: readinessv1alpha1.NodeReadinessRuleSpec{
+				EnforcementMode: mode,
+				DryRun:          dryRun,
+			},
+		}
+		if deleting {
+			now := metav1.Now()
+			rule.DeletionTimestamp = &now
+			rule.Finalizers = []string{finalizerName}
+		}
+		return rule
+	}
+
+	rules := []*readinessv1alpha1.NodeReadinessRule{
+		newRule("rule-a", readinessv1alpha1.EnforcementModeBootstrapOnly, false, false),
+		newRule("rule-b", readinessv1alpha1.EnforcementModeBootstrapOnly, false, false),
+		newRule("rule-c", readinessv1alpha1.EnforcementModeContinuous, true, false),
+		newRule("rule-deleting", readinessv1alpha1.EnforcementModeContinuous, false, true),
+	}
+
+	counts, err := c.ListRuleInventory(ctx, rules)
+	g.Expect(err).NotTo(HaveOccurred())
+
+	g.Expect(counts).To(Equal(map[metrics.RuleModeKey]float64{
+		{EnforcementMode: "bootstrap-only", DryRun: false}: 2,
+		{EnforcementMode: "continuous", DryRun: true}:      1,
+	}))
+}

internal/controller/nodereadinessrule_controller.go

@@ -682,6 +682,22 @@ func (r *RuleReadinessController) ListBlockedNodes(ctx context.Context, nodes []
 	return result, nil
 }
 
+// ListRuleInventory counts rules by enforcement mode and dry-run state.
+func (r *RuleReadinessController) ListRuleInventory(_ context.Context, rules []*readinessv1alpha1.NodeReadinessRule) (map[metrics.RuleModeKey]float64, error) {
+	counts := make(map[metrics.RuleModeKey]float64)
+
+	for _, rule := range rules {
+		if !rule.DeletionTimestamp.IsZero() {
+			continue
+		}
+
+		key := metrics.RuleModeKey{EnforcementMode: string(rule.Spec.EnforcementMode), DryRun: rule.Spec.DryRun}
+		counts[key]++
+	}
+
+	return counts, nil
+}
+
 // parseNodeSelector parses a rule's NodeSelector into a labels.Selector.
 func parseNodeSelector(rule *readinessv1alpha1.NodeReadinessRule) (labels.Selector, error) {
 	return metav1.LabelSelectorAsSelector(&rule.Spec.NodeSelector)

internal/metrics/collector.go

@@ -18,6 +18,7 @@ package metrics
 
 import (
 	"context"
+	"strconv"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -59,12 +60,24 @@ type BlockedNodesLister interface {
 	ListBlockedNodes(ctx context.Context, nodes []corev1.Node, rules []*readinessv1alpha1.NodeReadinessRule) (map[string]RuleBlockedConditions, error)
 }
 
+// RuleModeKey identifies a bucket of rules sharing the same enforcement mode and dry-run state.
+type RuleModeKey struct {
+	EnforcementMode string
+	DryRun          bool
+}
+
+// RuleInventoryLister counts NodeReadinessRules by enforcement mode and dry-run state.
+type RuleInventoryLister interface {
+	ListRuleInventory(ctx context.Context, rules []*readinessv1alpha1.NodeReadinessRule) (map[RuleModeKey]float64, error)
+}
+
 // ReadinessLister aggregates the scrape-time lookups the collector needs.
 type ReadinessLister interface {
 	NodeLister
 	RuleLister
 	RuleNodeStateLister
 	BlockedNodesLister
+	RuleInventoryLister
 }
 
 var ruleNodesDesc = prometheus.NewDesc(
@@ -81,6 +94,13 @@ var blockedNodesDesc = prometheus.NewDesc(
 	nil,
 )
 
+var ruleInventoryByModeDesc = prometheus.NewDesc(
+	"node_readiness_rules",
+	"Number of NodeReadinessRules by enforcement mode and dry-run state",
+	[]string{"enforcement_mode", "dry_run"},
+	nil,
+)
+
 // ReadinessCollector is a prometheus.Collector that reads at scrape time.
 type ReadinessCollector struct {
 	lister ReadinessLister
@@ -94,6 +114,7 @@ func NewReadinessCollector(lister ReadinessLister) *ReadinessCollector {
 func (c *ReadinessCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- ruleNodesDesc
 	ch <- blockedNodesDesc
+	ch <- ruleInventoryByModeDesc
 }
 
 // Collect implements prometheus.Collector.
@@ -133,4 +154,13 @@ func (c *ReadinessCollector) Collect(ch chan<- prometheus.Metric) {
 			}
 		}
 	}
+
+	ruleInventory, err := c.lister.ListRuleInventory(ctx, rules)
+	if err != nil {
+		ctrl.Log.V(2).Info("Failed to list rule inventory", "error", err)
+	} else {
+		for key, count := range ruleInventory {
+			ch <- prometheus.MustNewConstMetric(ruleInventoryByModeDesc, prometheus.GaugeValue, count, key.EnforcementMode, strconv.FormatBool(key.DryRun))
+		}
+	}
 }

internal/metrics/collector_test.go

@@ -46,11 +46,15 @@ type stubLister struct {
 	blocked    map[string]RuleBlockedConditions
 	blockedErr error
 
+	inventory    map[RuleModeKey]float64
+	inventoryErr error
+
 	mu                    sync.Mutex
 	gotNodesForRuleStates []corev1.Node
 	gotNodesForBlocked    []corev1.Node
 	gotRulesForRuleStates []*readinessv1alpha1.NodeReadinessRule
 	gotRulesForBlocked    []*readinessv1alpha1.NodeReadinessRule
+	gotRulesForInventory  []*readinessv1alpha1.NodeReadinessRule
 }
 
 func (s *stubLister) ListNodes(_ context.Context) ([]corev1.Node, error) {
@@ -89,6 +93,16 @@ func (s *stubLister) ListBlockedNodes(_ context.Context, nodes []corev1.Node, ru
 	return s.blocked, nil
 }
 
+func (s *stubLister) ListRuleInventory(_ context.Context, rules []*readinessv1alpha1.NodeReadinessRule) (map[RuleModeKey]float64, error) {
+	s.mu.Lock()
+	s.gotRulesForInventory = rules
+	s.mu.Unlock()
+	if s.inventoryErr != nil {
+		return nil, s.inventoryErr
+	}
+	return s.inventory, nil
+}
+
 func TestReadinessCollector_NoRules(t *testing.T) {
 	c := NewReadinessCollector(&stubLister{counts: map[string]RuleNodeCounts{}})
 
@@ -220,8 +234,9 @@ func collectAll(t *testing.T, c *ReadinessCollector) map[string][]*dto.Metric {
 
 func TestReadinessCollector_RuleNodesErrorDoesNotBlockBlockedNodes(t *testing.T) {
 	c := NewReadinessCollector(&stubLister{
-		err:     errors.New("cache not synced"),
-		blocked: map[string]RuleBlockedConditions{"gpu-ready": {"GPUDriverReady": 2}},
+		err:          errors.New("cache not synced"),
+		blocked:      map[string]RuleBlockedConditions{"gpu-ready": {"GPUDriverReady": 2}},
+		inventoryErr: errors.New("rule inventory cache not synced"),
 	})
 
 	got := collectAll(t, c)
@@ -241,8 +256,9 @@ func TestReadinessCollector_RuleNodesErrorDoesNotBlockBlockedNodes(t *testing.T)
 
 func TestReadinessCollector_BlockedNodesErrorDoesNotBlockRuleNodes(t *testing.T) {
 	c := NewReadinessCollector(&stubLister{
-		counts:     map[string]RuleNodeCounts{"gpu-ready": {Held: 3, Released: 1}},
-		blockedErr: errors.New("cache not synced"),
+		counts:       map[string]RuleNodeCounts{"gpu-ready": {Held: 3, Released: 1}},
+		blockedErr:   errors.New("cache not synced"),
+		inventoryErr: errors.New("rule inventory cache not synced"),
 	})
 
 	got := collectAll(t, c)
@@ -349,6 +365,67 @@ func TestReadinessCollector_RulesSharedBetweenBothListers(t *testing.T) {
 	}
 }
 
+func TestReadinessCollector_RuleInventory_ByModeAndDryRun(t *testing.T) {
+	c := NewReadinessCollector(&stubLister{
+		counts:  map[string]RuleNodeCounts{},
+		blocked: map[string]RuleBlockedConditions{},
+		inventory: map[RuleModeKey]float64{
+			{EnforcementMode: "bootstrap-only", DryRun: false}: 2,
+			{EnforcementMode: "continuous", DryRun: true}:      1,
+		},
+	})
+
+	expected := `
+		# HELP node_readiness_rules Number of NodeReadinessRules by enforcement mode and dry-run state
+		# TYPE node_readiness_rules gauge
+		node_readiness_rules{dry_run="false",enforcement_mode="bootstrap-only"} 2
+		node_readiness_rules{dry_run="true",enforcement_mode="continuous"} 1
+	`
+	if err := testutil.CollectAndCompare(c, strings.NewReader(expected), "node_readiness_rules"); err != nil {
+		t.Fatalf("unexpected collect mismatch: %v", err)
+	}
+}
+
+func TestReadinessCollector_RuleInventoryErrorSkipsBothInventoryMetrics(t *testing.T) {
+	stub := &stubLister{
+		counts:       map[string]RuleNodeCounts{},
+		blocked:      map[string]RuleBlockedConditions{},
+		inventoryErr: errors.New("cache not synced"),
+	}
+	c := NewReadinessCollector(stub)
+
+	ch := make(chan prometheus.Metric, 8)
+	c.Collect(ch)
+	close(ch)
+
+	for m := range ch {
+		if m.Desc() == ruleInventoryByModeDesc {
+			t.Fatalf("expected no rule inventory metrics when ListRuleInventory fails, got %v", m.Desc())
+		}
+	}
+}
+
+func TestReadinessCollector_RuleInventoryReceivesSharedRules(t *testing.T) {
+	rules := []*readinessv1alpha1.NodeReadinessRule{{ObjectMeta: metav1.ObjectMeta{Name: "gpu-ready"}}}
+	stub := &stubLister{
+		rules:     rules,
+		counts:    map[string]RuleNodeCounts{},
+		blocked:   map[string]RuleBlockedConditions{},
+		inventory: map[RuleModeKey]float64{},
+	}
+	c := NewReadinessCollector(stub)
+
+	ch := make(chan prometheus.Metric, 8)
+	c.Collect(ch)
+	close(ch)
+	for range ch {
+	}
+
+	if len(stub.gotRulesForInventory) != 1 || stub.gotRulesForInventory[0].Name != "gpu-ready" {
+		t.Fatalf("ListRuleInventory did not receive the shared rule snapshot: %v", stub.gotRulesForInventory)
+	}
+}
+
 func TestReadinessCollector_CollectAndLint(t *testing.T) {
 	c := NewReadinessCollector(&stubLister{
 		nodes: []corev1.Node{{}},
@@ -358,6 +435,9 @@ func TestReadinessCollector_CollectAndLint(t *testing.T) {
 		blocked: map[string]RuleBlockedConditions{
 			"gpu-ready": {"GPUDriverReady": 2, "CNIReady": 0},
 		},
+		inventory: map[RuleModeKey]float64{
+			{EnforcementMode: "bootstrap-only", DryRun: false}: 1,
+		},
 	})
 
 	problems, err := testutil.CollectAndLint(c)