diff --git a/asap-tools/experiments/HYDRA_CONFIG_USAGE.md b/asap-tools/experiments/HYDRA_CONFIG_USAGE.md index 14a2004..3cc39e7 100644 --- a/asap-tools/experiments/HYDRA_CONFIG_USAGE.md +++ b/asap-tools/experiments/HYDRA_CONFIG_USAGE.md @@ -12,6 +12,7 @@ python experiment_run_e2e.py \ providers.cloudlab.num_nodes=4 \ providers.cloudlab.username=myuser \ providers.cloudlab.hostname_suffix=myexp.cloudlab.us \ + controller.punting=false \ prometheus.local_config_dir=/path/to/prometheus/config ``` @@ -23,6 +24,8 @@ All experiment scripts require these core parameters: - `providers.cloudlab.num_nodes`: Number of CloudLab nodes - `providers.cloudlab.username`: Your CloudLab username - `providers.cloudlab.hostname_suffix`: CloudLab experiment hostname suffix +- `controller.punting`: Whether to enable controller query punting (`true` or + `false`) for runners that use the shared Hydra validation ### Script-Specific Required Parameters - **experiment_run_clickhouse.py**: `experiment_type=clickhouse`, `experiment_params.dataset.name`, `experiment_params.dataset.local_data_file`, `experiment_params.query_groups[0].sql_file` @@ -110,6 +113,7 @@ python experiment_run_e2e.py \ providers.cloudlab.num_nodes=2 \ providers.cloudlab.username=myuser \ providers.cloudlab.hostname_suffix=dev.cloudlab.us \ + controller.punting=false \ prometheus.local_config_dir=/path/to/prometheus/config \ logging.level=DEBUG \ flow.steady_state_wait=60 @@ -206,6 +210,7 @@ python experiment_run_clickhouse.py \ providers.cloudlab.num_nodes=1 \ providers.cloudlab.username=myuser \ providers.cloudlab.hostname_suffix=myexp.cloudlab.us \ + controller.punting=false \ experiment_params.dataset.name=clickbench \ experiment_params.dataset.local_data_file=/path/to/hits.json \ 'experiment_params.query_groups[0].sql_file=/path/to/queries.sql' @@ -234,6 +239,7 @@ python experiment_run_e2e.py \ providers.cloudlab.num_nodes=4 \ providers.cloudlab.username=myuser \ providers.cloudlab.hostname_suffix=myexp.cloudlab.us \ + controller.punting=false \ prometheus.local_config_dir=/path/to/prometheus/config ``` @@ -244,7 +250,8 @@ python experiment_run_exporters_and_prometheus.py \ experiment.name=monitoring_test \ providers.cloudlab.num_nodes=4 \ providers.cloudlab.username=myuser \ - providers.cloudlab.hostname_suffix=myexp.cloudlab.us + providers.cloudlab.hostname_suffix=myexp.cloudlab.us \ + controller.punting=false ``` ### experiment_run_empty_flink.py (Simplified Streaming) @@ -337,6 +344,7 @@ python experiment_run_e2e.py \ providers.cloudlab.num_nodes=4 \ providers.cloudlab.username=user \ providers.cloudlab.hostname_suffix=exp.cloudlab.us \ + controller.punting=false \ prometheus.local_config_dir=/path/to/config ``` diff --git a/asap-tools/experiments/config/experiment_type/cluster_data_alibaba_msresource_2021.yaml b/asap-tools/experiments/config/experiment_type/cluster_data_alibaba_msresource_2021.yaml index 0963b89..ea38e1d 100644 --- a/asap-tools/experiments/config/experiment_type/cluster_data_alibaba_msresource_2021.yaml +++ b/asap-tools/experiments/config/experiment_type/cluster_data_alibaba_msresource_2021.yaml @@ -42,7 +42,7 @@ query_groups: - sum by (microservice_id) (alibaba_microservice_cpu_usage) - alibaba_microservice_memory_usage - sum by (microservice_id) (alibaba_microservice_memory_usage) - repetition_delay_ms: 10000 + repetition_delay_ms: 1000 client_options: repetitions: 10 query_time_offset: 10 diff --git a/asap-tools/experiments/config/experiment_type/cluster_data_alibaba_msresource_2022.yaml b/asap-tools/experiments/config/experiment_type/cluster_data_alibaba_msresource_2022.yaml index 6474fe0..2a1be88 100644 --- a/asap-tools/experiments/config/experiment_type/cluster_data_alibaba_msresource_2022.yaml +++ b/asap-tools/experiments/config/experiment_type/cluster_data_alibaba_msresource_2022.yaml @@ -42,7 +42,7 @@ query_groups: - sum by (microservice_id) (alibaba_microservice_cpu_usage) - alibaba_microservice_memory_usage - sum by (microservice_id) (alibaba_microservice_memory_usage) - repetition_delay_ms: 10000 + repetition_delay_ms: 1000 client_options: repetitions: 10 query_time_offset: 10 diff --git a/asap-tools/experiments/config/experiment_type/cluster_data_alibaba_node_2021.yaml b/asap-tools/experiments/config/experiment_type/cluster_data_alibaba_node_2021.yaml index 582ca1e..e431112 100644 --- a/asap-tools/experiments/config/experiment_type/cluster_data_alibaba_node_2021.yaml +++ b/asap-tools/experiments/config/experiment_type/cluster_data_alibaba_node_2021.yaml @@ -53,7 +53,7 @@ query_groups: #- sum by (node_id) (alibaba_node_cpu_usage) #- alibaba_node_memory_usage #- sum by (node_id) (alibaba_node_memory_usage) - repetition_delay_ms: 10000 + repetition_delay_ms: 1000 client_options: repetitions: 10 query_time_offset: 10 diff --git a/asap-tools/experiments/config/experiment_type/cluster_data_alibaba_node_2022.yaml b/asap-tools/experiments/config/experiment_type/cluster_data_alibaba_node_2022.yaml index c6f69bf..c6f644b 100644 --- a/asap-tools/experiments/config/experiment_type/cluster_data_alibaba_node_2022.yaml +++ b/asap-tools/experiments/config/experiment_type/cluster_data_alibaba_node_2022.yaml @@ -45,7 +45,7 @@ query_groups: #- sum by (node_id) (alibaba_node_cpu_usage) #- alibaba_node_memory_usage #- sum by (node_id) (alibaba_node_memory_usage) - repetition_delay_ms: 10000 + repetition_delay_ms: 1000 client_options: repetitions: 10 query_time_offset: 10 diff --git a/asap-tools/experiments/config/experiment_type/cluster_data_google.yaml b/asap-tools/experiments/config/experiment_type/cluster_data_google.yaml index 5ce4ad2..70436d2 100644 --- a/asap-tools/experiments/config/experiment_type/cluster_data_google.yaml +++ b/asap-tools/experiments/config/experiment_type/cluster_data_google.yaml @@ -9,6 +9,11 @@ experiment: - mode: baseline server: prometheus +# Monitoring tool configuration +monitoring: + tool: prometheus + deployment_mode: bare_metal + servers: - name: prometheus url: http://localhost:9090 @@ -34,6 +39,8 @@ exporters: #parts_mode: "all-parts" # or "part-index" with part_index: 0 parts_mode: "part-index" # or "part-index" with part_index: 0 part_index: 0 + # A Google task_usage scrape contains many active task series. + scrape_timeout: "1s" # Optional configuration # log_level: "INFO" # DEBUG, INFO, WARN, ERROR (default: INFO) # memory_limit: "2g" @@ -52,7 +59,7 @@ query_groups: #- sum by (task_index) (google_canonical_memory_usage_0) ## Query Google max CPU usage #- google_max_cpu_usage_0 - repetition_delay_ms: 10000 + repetition_delay_ms: 1000 client_options: repetitions: 10 query_time_offset: 10 diff --git a/asap-tools/experiments/experiment_run_clickhouse.py b/asap-tools/experiments/experiment_run_clickhouse.py index 4e3e7e2..1f5cb78 100644 --- a/asap-tools/experiments/experiment_run_clickhouse.py +++ b/asap-tools/experiments/experiment_run_clickhouse.py @@ -118,6 +118,11 @@ DEFAULT_MONITOR_INTERVAL_SECONDS = 1.0 +def _start_clickhouse_controller(controller_service, cfg, **kwargs): + """Start the ClickHouse planner using the configured punting setting.""" + controller_service.start(punting=cfg.controller.punting, **kwargs) + + def _inline_sql_queries_in_experiment_config(local_experiment_root_dir: str) -> None: """Enrich the saved experiment_params.yaml by inlining SQL from sql_file references. @@ -545,13 +550,14 @@ def main(cfg: DictConfig) -> None: controller_service = ControllerService( provider=provider, use_container=False, node_offset=node_offset ) - controller_service.start( + _start_clickhouse_controller( + controller_service, + cfg, controller_input_file=os.path.join( remote_controller_dir, "planner_input.yaml" ), streaming_engine="precompute", controller_remote_output_dir=remote_controller_dir, - punting=False, discovery_backend=DiscoveryBackend( type="clickhouse", url=clickhouse_url, diff --git a/asap-tools/experiments/experiment_run_e2e.py b/asap-tools/experiments/experiment_run_e2e.py index 343a861..6c82bd6 100644 --- a/asap-tools/experiments/experiment_run_e2e.py +++ b/asap-tools/experiments/experiment_run_e2e.py @@ -332,6 +332,15 @@ def main(cfg: DictConfig): experiment_output_dir=experiment_output_dir, local_experiment_dir=local_experiment_dir, ) + if cluster_data_service and config.check_exporter_and_queries_exist( + "cluster_data_exporter", cfg.experiment_params + ): + cluster_data_service.start( + config=exporter_config["exporter_list"]["cluster_data_exporter"], + experiment_output_dir=experiment_output_dir, + local_experiment_dir=local_experiment_dir, + num_nodes=num_nodes_in_experiment, + ) prometheus_service.start( experiment_output_dir=experiment_output_dir, local_experiment_dir=local_experiment_dir, @@ -400,8 +409,12 @@ def main(cfg: DictConfig): ) # Handle cluster data exporter for replaying cluster traces - if cluster_data_service and config.check_exporter_and_queries_exist( - "cluster_data_exporter", cfg.experiment_params + if ( + cluster_data_service + and experiment_mode != constants.SKETCHDB_EXPERIMENT_NAME + and config.check_exporter_and_queries_exist( + "cluster_data_exporter", cfg.experiment_params + ) ): cluster_data_service.start( config=exporter_config["exporter_list"]["cluster_data_exporter"], diff --git a/asap-tools/experiments/experiment_utils/config.py b/asap-tools/experiments/experiment_utils/config.py index 1c1f5a4..fe0650b 100644 --- a/asap-tools/experiments/experiment_utils/config.py +++ b/asap-tools/experiments/experiment_utils/config.py @@ -510,8 +510,9 @@ def read_workloads_config(experiment_params: DictConfig): def get_prometheus_data_ingestion_interval_ms(prometheus_config): - """Extract scrape interval from Prometheus configuration, returned in milliseconds.""" + """Extract Prometheus' global scrape interval in milliseconds.""" s = prometheus_config.scrape_interval + # ponytail: check ms before s — "100ms".endswith("s") is True and would misroute if s.endswith("ms"): return int(s[:-2]) diff --git a/asap-tools/experiments/experiment_utils/core.py b/asap-tools/experiments/experiment_utils/core.py index b334958..2bb2496 100644 --- a/asap-tools/experiments/experiment_utils/core.py +++ b/asap-tools/experiments/experiment_utils/core.py @@ -53,6 +53,13 @@ def read_exporter_config(experiment_params: DictConfig) -> Tuple[Optional[Dict], f"Missing keys in cluster_data_exporter section: {missing_keys}", ) + if "scrape_interval" in cde_config: + return ( + None, + "cluster_data_exporter no longer accepts 'scrape_interval'; " + "set prometheus.scrape_interval instead", + ) + # Validate provider provider = cde_config.provider if provider not in ["google", "alibaba"]: diff --git a/asap-tools/experiments/generate_prometheus_config.py b/asap-tools/experiments/generate_prometheus_config.py index 35d3e86..a988ee7 100644 --- a/asap-tools/experiments/generate_prometheus_config.py +++ b/asap-tools/experiments/generate_prometheus_config.py @@ -282,7 +282,7 @@ def main(args, experiment_config=None): scrape_config = { "job_name": "cluster_data_exporter", - "scrape_interval": "10s", # Fast scrape interval for high-resolution cluster data + "scrape_interval": config["global"]["scrape_interval"], "static_configs": [ { "targets": [target], @@ -290,6 +290,9 @@ def main(args, experiment_config=None): ], } + if "scrape_timeout" in cde_config: + scrape_config["scrape_timeout"] = cde_config["scrape_timeout"] + # Add metric_relabel_configs to only keep required metrics add_metric_relabel_configs( scrape_config, "cluster_data_exporter", experiment_config diff --git a/asap-tools/experiments/generate_victoriametrics_config.py b/asap-tools/experiments/generate_victoriametrics_config.py index 68582b7..4be7654 100644 --- a/asap-tools/experiments/generate_victoriametrics_config.py +++ b/asap-tools/experiments/generate_victoriametrics_config.py @@ -285,6 +285,9 @@ def main(args, experiment_config=None): "static_configs": [{"targets": targets}], } + if "scrape_timeout" in cluster_data_config: + scrape_job["scrape_timeout"] = cluster_data_config["scrape_timeout"] + add_metric_relabel_configs( scrape_job, "cluster_data_exporter", experiment_config ) diff --git a/asap-tools/experiments/tests/test_cluster_data_config.py b/asap-tools/experiments/tests/test_cluster_data_config.py new file mode 100644 index 0000000..cbdd39e --- /dev/null +++ b/asap-tools/experiments/tests/test_cluster_data_config.py @@ -0,0 +1,167 @@ +"""Regression tests for cluster-data scrape and ingestion configuration.""" + +import tempfile +import unittest +from types import SimpleNamespace +from pathlib import Path + +import yaml +from omegaconf import OmegaConf + +import generate_prometheus_config +import generate_victoriametrics_config +from experiment_utils.config import get_prometheus_data_ingestion_interval_ms + + +EXPERIMENTS_DIR = Path(__file__).resolve().parents[1] + + +class ClusterDataConfigTest(unittest.TestCase): + def _generate_config(self, experiment_config): + args = SimpleNamespace( + num_nodes=1, + node_offset=18, + output_dir=None, + copy_to_dir=None, + rule_files=None, + query_log_file=None, + prometheus_client_ip="10.10.1.19", + node_ip_prefix="10.10.1", + remote_write_url=None, + remote_write_metric_names=None, + remote_write_base_port=None, + parallelism=None, + scrape_interval="1s", + evaluation_interval="10s", + recording_rules_interval="5s", + ) + with tempfile.TemporaryDirectory() as output_dir: + args.output_dir = output_dir + generate_prometheus_config.main(args, experiment_config) + with open(f"{output_dir}/prometheus.yml") as config_file: + return yaml.safe_load(config_file) + + def _generate_vmagent_config(self, experiment_config): + args = SimpleNamespace( + num_nodes=1, + node_offset=18, + output_dir=None, + node_ip_prefix="10.10.1", + scrape_interval="1s", + remote_write_metric_names=None, + ) + with tempfile.TemporaryDirectory() as output_dir: + args.output_dir = output_dir + generate_victoriametrics_config.main(args, experiment_config) + with open(f"{output_dir}/vmagent_scrape.yml") as config_file: + return yaml.safe_load(config_file) + + def test_google_uses_global_scrape_interval(self): + with open( + EXPERIMENTS_DIR / "config/experiment_type/cluster_data_google.yaml" + ) as config_file: + experiment_config = yaml.safe_load(config_file) + + prometheus_config = self._generate_config(experiment_config) + cde_job = next( + job + for job in prometheus_config["scrape_configs"] + if job["job_name"] == "cluster_data_exporter" + ) + + self.assertEqual(cde_job["scrape_interval"], "1s") + self.assertEqual(cde_job["scrape_timeout"], "1s") + self.assertEqual( + get_prometheus_data_ingestion_interval_ms( + OmegaConf.create({"scrape_interval": "1s"}) + ), + 1000, + ) + + def test_alibaba_uses_global_scrape_interval(self): + with open( + EXPERIMENTS_DIR + / "config/experiment_type/cluster_data_alibaba_node_2021.yaml" + ) as config_file: + experiment_config = yaml.safe_load(config_file) + + prometheus_config = self._generate_config(experiment_config) + cde_job = next( + job + for job in prometheus_config["scrape_configs"] + if job["job_name"] == "cluster_data_exporter" + ) + + self.assertEqual(cde_job["scrape_interval"], "1s") + self.assertEqual( + get_prometheus_data_ingestion_interval_ms( + OmegaConf.create({"scrape_interval": "1s"}) + ), + 1000, + ) + + def test_cluster_data_does_not_define_scrape_interval(self): + with open( + EXPERIMENTS_DIR + / "config/experiment_type/cluster_data_alibaba_node_2022.yaml" + ) as config_file: + experiment_config = yaml.safe_load(config_file) + cde_config = experiment_config["exporters"]["exporter_list"][ + "cluster_data_exporter" + ] + self.assertNotIn("scrape_interval", cde_config) + + prometheus_config = self._generate_config(experiment_config) + cde_job = next( + job + for job in prometheus_config["scrape_configs"] + if job["job_name"] == "cluster_data_exporter" + ) + self.assertEqual(cde_job["scrape_interval"], "1s") + + def test_obsolete_cluster_data_interval_fails_loudly(self): + with open( + EXPERIMENTS_DIR + / "config/experiment_type/cluster_data_alibaba_node_2022.yaml" + ) as config_file: + experiment_config = yaml.safe_load(config_file) + experiment_config["exporters"]["exporter_list"]["cluster_data_exporter"][ + "scrape_interval" + ] = "60s" + + with self.assertRaisesRegex(ValueError, "no longer accepts"): + self._generate_config(experiment_config) + + def test_victoriametrics_uses_global_interval_and_timeout(self): + with open( + EXPERIMENTS_DIR / "config/experiment_type/cluster_data_google.yaml" + ) as config_file: + experiment_config = yaml.safe_load(config_file) + + vmagent_config = self._generate_vmagent_config(experiment_config) + self.assertEqual(vmagent_config["global"]["scrape_interval"], "1s") + cde_job = next( + job + for job in vmagent_config["scrape_configs"] + if job["job_name"] == "cluster_data_exporter" + ) + self.assertEqual(cde_job["scrape_timeout"], "1s") + + def test_cluster_data_repetitions_use_one_second_delay(self): + for config_path in EXPERIMENTS_DIR.glob( + "config/experiment_type/cluster_data_*.yaml" + ): + with self.subTest(config=config_path.name): + with open(config_path) as config_file: + experiment_config = yaml.safe_load(config_file) + self.assertTrue(experiment_config["query_groups"]) + self.assertTrue( + all( + group["repetition_delay_ms"] == 1000 + for group in experiment_config["query_groups"] + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/asap-tools/experiments/tests/test_experiment_run_clickhouse.py b/asap-tools/experiments/tests/test_experiment_run_clickhouse.py new file mode 100644 index 0000000..d39f8c2 --- /dev/null +++ b/asap-tools/experiments/tests/test_experiment_run_clickhouse.py @@ -0,0 +1,31 @@ +"""Regression tests for ClickHouse experiment controller wiring.""" + +import unittest +from unittest.mock import Mock + +from omegaconf import OmegaConf + +from experiment_run_clickhouse import _start_clickhouse_controller + + +class ClickHouseControllerConfigTest(unittest.TestCase): + def test_controller_receives_configured_punting_setting(self): + controller_service = Mock() + cfg = OmegaConf.create({"controller": {"punting": True}}) + + _start_clickhouse_controller( + controller_service, + cfg, + controller_input_file="planner_input.yaml", + streaming_engine="precompute", + ) + + controller_service.start.assert_called_once_with( + punting=True, + controller_input_file="planner_input.yaml", + streaming_engine="precompute", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/03-how-to-guides/operations/run-alibaba-cluster-data-experiment.md b/docs/03-how-to-guides/operations/run-alibaba-cluster-data-experiment.md index f556d2c..783f4db 100644 --- a/docs/03-how-to-guides/operations/run-alibaba-cluster-data-experiment.md +++ b/docs/03-how-to-guides/operations/run-alibaba-cluster-data-experiment.md @@ -89,9 +89,14 @@ sketchdb-cluster-data-exporter:latest The checked-in experiment configuration is `asap-tools/experiments/config/experiment_type/cluster_data_alibaba_node_2021.yaml`. It runs four global CPU quantiles, ten repetitions each, at 3x replay speed. +All cluster-data experiments use the global +`prometheus.scrape_interval: "1s"` setting in +`asap-tools/experiments/config/config.yaml`. Prometheus' cluster-data job and +controller ingestion planning both use that setting; the exporter config does +not define a second scrape interval. Query repetition delays are also 1 second. -`controller.punting` is set to `false` in -`asap-tools/experiments/config/config.yaml`. +Because `controller.punting` is intentionally required in the shared config, +the command below sets it explicitly to `false` for this experiment. ```bash cd /home/milind/Desktop/cmu/research/sketch_db_for_prometheus/code/ASAPQuery/asap-tools/experiments @@ -103,12 +108,97 @@ python3 experiment_run_e2e.py \ providers.cloudlab.hostname_suffix=scratch2.cloudmigration-PG0.utah.cloudlab.us \ providers.cloudlab.node_offset=18 \ providers.cloudlab.num_nodes=1 \ + controller.punting=false \ cluster_data_directory=/scratch/sketch_db_for_prometheus/cluster_traces ``` The default run takes about three minutes and tears down the services when it finishes. Use `flow.no_teardown=true` only when debugging service state. +## Run a Google Cluster-Data Experiment + +The Google trace is already split into compressed `task_usage` parts, so no +preprocessing is required. The benchmark data is under: + +```text +../../benchmarks/metrics_observability/data/google-cluster-data/ClusterData2011/clusterdata-2011-2/task_usage +``` + +The checked-in Google configuration uses `part_index: 0`, which is useful for +a one-node experiment. The first part is approximately 92 MB compressed and +381 MB uncompressed. Copy it to node19: + +```bash +GOOGLE_DATA="$PWD/../../benchmarks/metrics_observability/data/google-cluster-data/ClusterData2011/clusterdata-2011-2/task_usage" +CDE_HOST="node19.scratch2.cloudmigration-PG0.utah.cloudlab.us" + +ssh -o StrictHostKeyChecking=no "milindsr@$CDE_HOST" \ + 'mkdir -p /scratch/sketch_db_for_prometheus/cluster_traces' + +rsync -ah --info=progress2 -e 'ssh -o StrictHostKeyChecking=no' \ + "$GOOGLE_DATA/part-00000-of-00500.csv.gz" \ + "milindsr@$CDE_HOST:/scratch/sketch_db_for_prometheus/cluster_traces/" +``` + +Build the same exporter image used for Alibaba if it is not already present: + +```bash +ssh -o StrictHostKeyChecking=no "milindsr@$CDE_HOST" \ + 'cd /scratch/sketch_db_for_prometheus/code/asap-tools/data-sources/prometheus-exporters/cluster_data_exporter && + bash installation/install.sh' +``` + +Run the Google experiment from `asap-tools/experiments`: + +```bash +python3 experiment_run_e2e.py \ + experiment.name=google_cluster_data_offset18 \ + experiment_type=cluster_data_google \ + providers.cloudlab.username=milindsr \ + providers.cloudlab.hostname_suffix=scratch2.cloudmigration-PG0.utah.cloudlab.us \ + providers.cloudlab.node_offset=18 \ + providers.cloudlab.num_nodes=1 \ + controller.punting=false \ + cluster_data_directory=/scratch/sketch_db_for_prometheus/cluster_traces +``` + +This configuration runs both `sketchdb` and `baseline` modes. Google +`task_usage` produces a much larger `/metrics` response than the Alibaba +Node trace. Its scrape job still uses the global 1-second interval, while the +Google config keeps a 1-second scrape timeout for the large response. If the +Google scrape cannot complete within 1 second, adjust the timeout after +validating the available Prometheus capacity; do not add a second interval +setting to the exporter config. + +The default query is: + +```promql +quantile by () (0.5, google_mean_cpu_usage_rate_0) +``` + +For Google data, the exporter labels are `job_id`, `task_index`, and +`machine_id` (along with the scrape-added `instance` and `job` labels). To +change the aggregate column, use the corresponding metric name and update +the `metrics` entry in +`asap-tools/experiments/config/experiment_type/cluster_data_google.yaml`. +For example, `canonical-memory-usage` becomes +`google_canonical_memory_usage_0`. To group the result by a data label, change +`by ()` to e.g. `by (machine_id)` and include that label in the `metrics` +entry's label list. + +Because Google has separate baseline and sketchdb modes, the latency wrapper +can be used after the run: + +```bash +cd /home/milind/Desktop/cmu/research/sketch_db_for_prometheus/code/ASAPQuery/asap-tools/experiments/post_experiment +PYTHONPATH=/home/milind/Desktop/cmu/research/sketch_db_for_prometheus/code/ASAPQuery/asap-common/dependencies/py/promql_utilities \ +./run_compare_latencies.sh google_cluster_data_offset18 + +python3 analyze_monitor_output.py \ + /home/milind/Desktop/cmu/research/sketch_db_for_prometheus/code/ASAPQuery/experiment_outputs/google_cluster_data_offset18/sketchdb/remote_monitor_output/monitor_output.json \ + --print +``` + ## Change the query's aggregation column or grouping label The raw 2021 Node CSV has columns like: