fix(asap-tools): support Google cluster-data experiments

asap-tools/experiments/HYDRA_CONFIG_USAGE.md

@@ -12,6 +12,7 @@ python experiment_run_e2e.py \
   providers.cloudlab.num_nodes=4 \
   providers.cloudlab.username=myuser \
   providers.cloudlab.hostname_suffix=myexp.cloudlab.us \
+  controller.punting=false \
   prometheus.local_config_dir=/path/to/prometheus/config
 ```
 
@@ -23,6 +24,8 @@ All experiment scripts require these core parameters:
 - `providers.cloudlab.num_nodes`: Number of CloudLab nodes
 - `providers.cloudlab.username`: Your CloudLab username
 - `providers.cloudlab.hostname_suffix`: CloudLab experiment hostname suffix
+- `controller.punting`: Whether to enable controller query punting (`true` or
+  `false`) for runners that use the shared Hydra validation
 
 ### Script-Specific Required Parameters
 - **experiment_run_clickhouse.py**: `experiment_type=clickhouse`, `experiment_params.dataset.name`, `experiment_params.dataset.local_data_file`, `experiment_params.query_groups[0].sql_file`
@@ -110,6 +113,7 @@ python experiment_run_e2e.py \
   providers.cloudlab.num_nodes=2 \
   providers.cloudlab.username=myuser \
   providers.cloudlab.hostname_suffix=dev.cloudlab.us \
+  controller.punting=false \
   prometheus.local_config_dir=/path/to/prometheus/config \
   logging.level=DEBUG \
   flow.steady_state_wait=60
@@ -206,6 +210,7 @@ python experiment_run_clickhouse.py \
   providers.cloudlab.num_nodes=1 \
   providers.cloudlab.username=myuser \
   providers.cloudlab.hostname_suffix=myexp.cloudlab.us \
+  controller.punting=false \
   experiment_params.dataset.name=clickbench \
   experiment_params.dataset.local_data_file=/path/to/hits.json \
   'experiment_params.query_groups[0].sql_file=/path/to/queries.sql'
@@ -234,6 +239,7 @@ python experiment_run_e2e.py \
   providers.cloudlab.num_nodes=4 \
   providers.cloudlab.username=myuser \
   providers.cloudlab.hostname_suffix=myexp.cloudlab.us \
+  controller.punting=false \
   prometheus.local_config_dir=/path/to/prometheus/config
 ```
 
@@ -244,7 +250,8 @@ python experiment_run_exporters_and_prometheus.py \
   experiment.name=monitoring_test \
   providers.cloudlab.num_nodes=4 \
   providers.cloudlab.username=myuser \
-  providers.cloudlab.hostname_suffix=myexp.cloudlab.us
+  providers.cloudlab.hostname_suffix=myexp.cloudlab.us \
+  controller.punting=false
 ```
 
 ### experiment_run_empty_flink.py (Simplified Streaming)
@@ -337,6 +344,7 @@ python experiment_run_e2e.py \
   providers.cloudlab.num_nodes=4 \
   providers.cloudlab.username=user \
   providers.cloudlab.hostname_suffix=exp.cloudlab.us \
+  controller.punting=false \
   prometheus.local_config_dir=/path/to/config
 ```
 

asap-tools/experiments/config/experiment_type/cluster_data_alibaba_msresource_2021.yaml

@@ -42,7 +42,7 @@ query_groups:
   - sum by (microservice_id) (alibaba_microservice_cpu_usage)
   - alibaba_microservice_memory_usage
   - sum by (microservice_id) (alibaba_microservice_memory_usage)
-  repetition_delay_ms: 10000
+  repetition_delay_ms: 1000
   client_options:
     repetitions: 10
     query_time_offset: 10

asap-tools/experiments/config/experiment_type/cluster_data_alibaba_msresource_2022.yaml

@@ -42,7 +42,7 @@ query_groups:
   - sum by (microservice_id) (alibaba_microservice_cpu_usage)
   - alibaba_microservice_memory_usage
   - sum by (microservice_id) (alibaba_microservice_memory_usage)
-  repetition_delay_ms: 10000
+  repetition_delay_ms: 1000
   client_options:
     repetitions: 10
     query_time_offset: 10

asap-tools/experiments/config/experiment_type/cluster_data_alibaba_node_2021.yaml

@@ -53,7 +53,7 @@ query_groups:
   #- sum by (node_id) (alibaba_node_cpu_usage)
   #- alibaba_node_memory_usage
   #- sum by (node_id) (alibaba_node_memory_usage)
-  repetition_delay_ms: 10000
+  repetition_delay_ms: 1000
   client_options:
     repetitions: 10
     query_time_offset: 10

asap-tools/experiments/config/experiment_type/cluster_data_alibaba_node_2022.yaml

@@ -45,7 +45,7 @@ query_groups:
   #- sum by (node_id) (alibaba_node_cpu_usage)
   #- alibaba_node_memory_usage
   #- sum by (node_id) (alibaba_node_memory_usage)
-  repetition_delay_ms: 10000
+  repetition_delay_ms: 1000
   client_options:
     repetitions: 10
     query_time_offset: 10

asap-tools/experiments/config/experiment_type/cluster_data_google.yaml

@@ -9,6 +9,11 @@ experiment:
   - mode: baseline
     server: prometheus
 
+# Monitoring tool configuration
+monitoring:
+  tool: prometheus
+  deployment_mode: bare_metal
+
 servers:
 - name: prometheus
   url: http://localhost:9090
@@ -34,6 +39,8 @@ exporters:
       #parts_mode: "all-parts"  # or "part-index" with part_index: 0
       parts_mode: "part-index"  # or "part-index" with part_index: 0
       part_index: 0
+      # A Google task_usage scrape contains many active task series.
+      scrape_timeout: "1s"
       # Optional configuration
       # log_level: "INFO"  # DEBUG, INFO, WARN, ERROR (default: INFO)
       # memory_limit: "2g"
@@ -52,7 +59,7 @@ query_groups:
   #- sum by (task_index) (google_canonical_memory_usage_0)
   ## Query Google max CPU usage
   #- google_max_cpu_usage_0
-  repetition_delay_ms: 10000
+  repetition_delay_ms: 1000
   client_options:
     repetitions: 10
     query_time_offset: 10

asap-tools/experiments/experiment_run_clickhouse.py

@@ -118,6 +118,11 @@
 DEFAULT_MONITOR_INTERVAL_SECONDS = 1.0
 
 
+def _start_clickhouse_controller(controller_service, cfg, **kwargs):
+    """Start the ClickHouse planner using the configured punting setting."""
+    controller_service.start(punting=cfg.controller.punting, **kwargs)
+
+
 def _inline_sql_queries_in_experiment_config(local_experiment_root_dir: str) -> None:
     """Enrich the saved experiment_params.yaml by inlining SQL from sql_file references.
 
@@ -545,13 +550,14 @@ def main(cfg: DictConfig) -> None:
             controller_service = ControllerService(
                 provider=provider, use_container=False, node_offset=node_offset
             )
-            controller_service.start(
+            _start_clickhouse_controller(
+                controller_service,
+                cfg,
                 controller_input_file=os.path.join(
                     remote_controller_dir, "planner_input.yaml"
                 ),
                 streaming_engine="precompute",
                 controller_remote_output_dir=remote_controller_dir,
-                punting=False,
                 discovery_backend=DiscoveryBackend(
                     type="clickhouse",
                     url=clickhouse_url,

asap-tools/experiments/experiment_run_e2e.py

@@ -332,6 +332,15 @@ def main(cfg: DictConfig):
                     experiment_output_dir=experiment_output_dir,
                     local_experiment_dir=local_experiment_dir,
                 )
+            if cluster_data_service and config.check_exporter_and_queries_exist(
+                "cluster_data_exporter", cfg.experiment_params
+            ):
+                cluster_data_service.start(
+                    config=exporter_config["exporter_list"]["cluster_data_exporter"],
+                    experiment_output_dir=experiment_output_dir,
+                    local_experiment_dir=local_experiment_dir,
+                    num_nodes=num_nodes_in_experiment,
+                )
             prometheus_service.start(
                 experiment_output_dir=experiment_output_dir,
                 local_experiment_dir=local_experiment_dir,
@@ -400,8 +409,12 @@ def main(cfg: DictConfig):
             )
 
         # Handle cluster data exporter for replaying cluster traces
-        if cluster_data_service and config.check_exporter_and_queries_exist(
-            "cluster_data_exporter", cfg.experiment_params
+        if (
+            cluster_data_service
+            and experiment_mode != constants.SKETCHDB_EXPERIMENT_NAME
+            and config.check_exporter_and_queries_exist(
+                "cluster_data_exporter", cfg.experiment_params
+            )
         ):
             cluster_data_service.start(
                 config=exporter_config["exporter_list"]["cluster_data_exporter"],

asap-tools/experiments/experiment_utils/config.py

@@ -510,8 +510,9 @@ def read_workloads_config(experiment_params: DictConfig):
 
 
 def get_prometheus_data_ingestion_interval_ms(prometheus_config):
-    """Extract scrape interval from Prometheus configuration, returned in milliseconds."""
+    """Extract Prometheus' global scrape interval in milliseconds."""
     s = prometheus_config.scrape_interval
+
     # ponytail: check ms before s — "100ms".endswith("s") is True and would misroute
     if s.endswith("ms"):
         return int(s[:-2])

asap-tools/experiments/experiment_utils/core.py

@@ -53,6 +53,13 @@ def read_exporter_config(experiment_params: DictConfig) -> Tuple[Optional[Dict],
                 f"Missing keys in cluster_data_exporter section: {missing_keys}",
             )
 
+        if "scrape_interval" in cde_config:
+            return (
+                None,
+                "cluster_data_exporter no longer accepts 'scrape_interval'; "
+                "set prometheus.scrape_interval instead",
+            )
+
         # Validate provider
         provider = cde_config.provider
         if provider not in ["google", "alibaba"]:

asap-tools/experiments/generate_prometheus_config.py

@@ -282,14 +282,17 @@ def main(args, experiment_config=None):
 
         scrape_config = {
             "job_name": "cluster_data_exporter",
-            "scrape_interval": "10s",  # Fast scrape interval for high-resolution cluster data
+            "scrape_interval": config["global"]["scrape_interval"],
             "static_configs": [
                 {
                     "targets": [target],
                 }
             ],
         }
 
+        if "scrape_timeout" in cde_config:
+            scrape_config["scrape_timeout"] = cde_config["scrape_timeout"]
+
         # Add metric_relabel_configs to only keep required metrics
         add_metric_relabel_configs(
             scrape_config, "cluster_data_exporter", experiment_config

asap-tools/experiments/generate_victoriametrics_config.py

@@ -285,6 +285,9 @@ def main(args, experiment_config=None):
             "static_configs": [{"targets": targets}],
         }
 
+        if "scrape_timeout" in cluster_data_config:
+            scrape_job["scrape_timeout"] = cluster_data_config["scrape_timeout"]
+
         add_metric_relabel_configs(
             scrape_job, "cluster_data_exporter", experiment_config
         )