fix(asap-tools): require explicit cluster scrape intervals

Author: milindsrivastava1997

asap-tools/experiments/config/experiment_type/cluster_data_alibaba_msresource_2021.yaml

@@ -28,7 +28,7 @@ exporters:
       data_type: "msresource"
       data_year: 2021
       parts_mode: "all-parts"  # or "part-index" with part_index: 0
-      scrape_interval: "10s"
+      scrape_interval: "1s"
       # Optional configuration
       # log_level: "INFO"  # DEBUG, INFO, WARN, ERROR (default: INFO)
       # memory_limit: "2g"

asap-tools/experiments/config/experiment_type/cluster_data_alibaba_msresource_2022.yaml

@@ -28,7 +28,7 @@ exporters:
       data_type: "msresource"
       data_year: 2022
       parts_mode: "all-parts"  # or "part-index" with part_index: 0
-      scrape_interval: "10s"
+      scrape_interval: "1s"
       # Optional configuration
       # log_level: "INFO"  # DEBUG, INFO, WARN, ERROR (default: INFO)
       # memory_limit: "2g"

asap-tools/experiments/config/experiment_type/cluster_data_alibaba_node_2021.yaml

@@ -35,7 +35,7 @@ exporters:
       parts_mode: "all-parts"  # or "part-index" with part_index: 0
       #part-index: 0
       speedup: 3  # Speedup factor: 1=real-time (default), 10=10x faster, 100=100x faster
-      scrape_interval: "10s"
+      scrape_interval: "1s"
       # Optional configuration
       # log_level: "INFO"  # DEBUG, INFO, WARN, ERROR (default: INFO)
       # memory_limit: "2g"

asap-tools/experiments/config/experiment_type/cluster_data_alibaba_node_2022.yaml

@@ -30,7 +30,7 @@ exporters:
       #parts_mode: "all-parts"  # or "part-index" with part_index: 0
       parts_mode: "part-index"
       part_index: 0
-      scrape_interval: "10s"
+      scrape_interval: "1s"
       # Optional configuration
       # log_level: "INFO"  # DEBUG, INFO, WARN, ERROR (default: INFO)
       # memory_limit: "2g"

asap-tools/experiments/config/experiment_type/cluster_data_google.yaml

@@ -40,8 +40,8 @@ exporters:
       parts_mode: "part-index"  # or "part-index" with part_index: 0
       part_index: 0
       # A Google task_usage scrape contains many active task series.
-      scrape_interval: "60s"
-      scrape_timeout: "30s"
+      scrape_interval: "1s"
+      scrape_timeout: "1s"
       # Optional configuration
       # log_level: "INFO"  # DEBUG, INFO, WARN, ERROR (default: INFO)
       # memory_limit: "2g"

asap-tools/experiments/experiment_utils/config.py

@@ -14,9 +14,6 @@
 from experiment_utils.providers.factory import create_provider
 
 
-DEFAULT_CLUSTER_DATA_SCRAPE_INTERVAL = "10s"
-
-
 def validate_basic_config(
     cfg: DictConfig,
     required_params: List[Tuple[str, str]],
@@ -527,9 +524,9 @@ def get_prometheus_data_ingestion_interval_ms(
         exporter_list = exporters.get("exporter_list", {})
         cluster_data_exporter = exporter_list.get("cluster_data_exporter")
         if cluster_data_exporter is not None:
-            s = cluster_data_exporter.get(
-                "scrape_interval", DEFAULT_CLUSTER_DATA_SCRAPE_INTERVAL
-            )
+            if "scrape_interval" not in cluster_data_exporter:
+                raise ValueError("cluster_data_exporter requires 'scrape_interval'")
+            s = cluster_data_exporter["scrape_interval"]
 
     # ponytail: check ms before s — "100ms".endswith("s") is True and would misroute
     if s.endswith("ms"):

asap-tools/experiments/experiment_utils/core.py

@@ -45,7 +45,7 @@ def read_exporter_config(experiment_params: DictConfig) -> Tuple[Optional[Dict],
         cde_config = exporters_config.exporter_list.cluster_data_exporter
 
         # Check required keys
-        required_keys = ["provider", "port"]
+        required_keys = ["provider", "port", "scrape_interval"]
         missing_keys = [key for key in required_keys if key not in cde_config]
         if missing_keys:
             return (

asap-tools/experiments/generate_prometheus_config.py

@@ -5,7 +5,6 @@
 from omegaconf import DictConfig
 
 import experiment_utils
-from experiment_utils.config import DEFAULT_CLUSTER_DATA_SCRAPE_INTERVAL
 from utils import get_node_range
 
 
@@ -281,11 +280,12 @@ def main(args, experiment_config=None):
         target_node_idx = args.node_offset + 2
         target = f"{args.node_ip_prefix}.{target_node_idx}:{port}"
 
+        if "scrape_interval" not in cde_config:
+            raise ValueError("cluster_data_exporter requires 'scrape_interval'")
+
         scrape_config = {
             "job_name": "cluster_data_exporter",
-            "scrape_interval": cde_config.get(
-                "scrape_interval", DEFAULT_CLUSTER_DATA_SCRAPE_INTERVAL
-            ),
+            "scrape_interval": cde_config["scrape_interval"],
             "static_configs": [
                 {
                     "targets": [target],

asap-tools/experiments/tests/test_cluster_data_config.py

@@ -53,14 +53,14 @@ def test_google_scrape_interval_matches_controller_ingestion_interval(self):
             if job["job_name"] == "cluster_data_exporter"
         )
 
-        self.assertEqual(cde_job["scrape_interval"], "60s")
-        self.assertEqual(cde_job["scrape_timeout"], "30s")
+        self.assertEqual(cde_job["scrape_interval"], "1s")
+        self.assertEqual(cde_job["scrape_timeout"], "1s")
         self.assertEqual(
             get_prometheus_data_ingestion_interval_ms(
                 OmegaConf.create({"scrape_interval": "1s"}),
                 OmegaConf.create(experiment_config),
             ),
-            60000,
+            1000,
         )
 
     def test_alibaba_scrape_interval_matches_controller_ingestion_interval(self):
@@ -77,16 +77,16 @@ def test_alibaba_scrape_interval_matches_controller_ingestion_interval(self):
             if job["job_name"] == "cluster_data_exporter"
         )
 
-        self.assertEqual(cde_job["scrape_interval"], "10s")
+        self.assertEqual(cde_job["scrape_interval"], "1s")
         self.assertEqual(
             get_prometheus_data_ingestion_interval_ms(
                 OmegaConf.create({"scrape_interval": "1s"}),
                 OmegaConf.create(experiment_config),
             ),
-            10000,
+            1000,
         )
 
-    def test_missing_cluster_data_interval_uses_shared_default(self):
+    def test_missing_cluster_data_interval_fails_loudly(self):
         with open(
             EXPERIMENTS_DIR
             / "config/experiment_type/cluster_data_alibaba_node_2022.yaml"
@@ -96,21 +96,13 @@ def test_missing_cluster_data_interval_uses_shared_default(self):
             "scrape_interval"
         ]
 
-        prometheus_config = self._generate_config(experiment_config)
-        cde_job = next(
-            job
-            for job in prometheus_config["scrape_configs"]
-            if job["job_name"] == "cluster_data_exporter"
-        )
-
-        self.assertEqual(cde_job["scrape_interval"], "10s")
-        self.assertEqual(
+        with self.assertRaisesRegex(ValueError, "scrape_interval"):
+            self._generate_config(experiment_config)
+        with self.assertRaisesRegex(ValueError, "scrape_interval"):
             get_prometheus_data_ingestion_interval_ms(
                 OmegaConf.create({"scrape_interval": "1s"}),
                 OmegaConf.create(experiment_config),
-            ),
-            10000,
-        )
+            )
 
 
 if __name__ == "__main__":

docs/03-how-to-guides/operations/run-alibaba-cluster-data-experiment.md

@@ -89,7 +89,7 @@ sketchdb-cluster-data-exporter:latest
 The checked-in experiment configuration is
 `asap-tools/experiments/config/experiment_type/cluster_data_alibaba_node_2021.yaml`.
 It runs four global CPU quantiles, ten repetitions each, at 3x replay speed.
-Its cluster-data scrape interval is explicitly set to 10 seconds, and the
+Its cluster-data scrape interval is explicitly set to 1 second, and the
 controller uses that same interval for ingestion planning.
 
 `controller.punting` is set to `false` in
@@ -159,10 +159,11 @@ python3 experiment_run_e2e.py \
 
 This configuration runs both `sketchdb` and `baseline` modes. Google
 `task_usage` produces a much larger `/metrics` response than the Alibaba
-Node trace, so the Google config sets the cluster-data scrape interval to
-60 seconds and timeout to 30 seconds. The Prometheus config generator honors
-these optional exporter settings. The Google query repetition delay is also
-60 seconds so repetitions do not outrun new scrapes.
+Node trace, but the Google and Alibaba configs currently use a 1-second
+cluster-data scrape interval so Prometheus and controller planning remain
+aligned. The Google config uses a 1-second scrape timeout as well; increase
+both values together only after validating the larger response against the
+available Prometheus capacity.
 
 The default query is: