From cb6c0d8a35c07775e4379b2fb5f605b3f561a55d Mon Sep 17 00:00:00 2001 From: qmy Date: Wed, 22 Jul 2026 14:04:09 +0800 Subject: [PATCH 1/2] fix(perf): include negative rate in run-dir regex so HTML report is generated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Open-loop runs that don't pin a fixed request rate write the run directory as ``rate_-1.0_number_`` (``-1.0`` is the Request Rate sentinel when rate is unset, e.g. workload_trace replay driven by trace arrival offsets). The loader regex had no sign in its character class, so the directory never matched and ``RunLoader.load_all`` dropped it — ``gen_perf_html_report`` then saw zero runs and logged "No benchmark runs found ... Skipping HTML report", silently not producing ``perf_report.html``. Allow an optional leading sign on the rate capture group. Verified by re-running a workload_trace open-loop (rate=-1.0) run against a vLLM endpoint: perf_report.html is now generated (previously skipped). Co-Authored-By: Claude --- evalscope/perf/utils/report/perf_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evalscope/perf/utils/report/perf_data.py b/evalscope/perf/utils/report/perf_data.py index 97b6ce196..7e16b2cb3 100644 --- a/evalscope/perf/utils/report/perf_data.py +++ b/evalscope/perf/utils/report/perf_data.py @@ -36,7 +36,7 @@ # Patterns for recognized run directory names _PARALLEL_RE = re.compile(r'^parallel_(\d+)_number_(\d+)$') -_RATE_RE = re.compile(r'^rate_([\d.]+)_number_(\d+)$') +_RATE_RE = re.compile(r'^rate_(-?[\d.]+)_number_(\d+)$') class RunLoader: From 70ce1c5cd0b6e9be113a5add40bb1309197eb0c8 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Wed, 22 Jul 2026 17:13:34 +0800 Subject: [PATCH 2/2] fix(perf): derive run mode from RunData fields, not dir-name regex Follow-up to the rate=-1.0 loader fix. RunData.name and perf_charts._x_axis re-parsed the run directory name with their own sign-less regexes, so an open-loop run with no fixed rate (rate_-1.0_number_M) was still mislabeled as "Parallel 0" and its sweep charts used a "Concurrency" x-axis. - perf_charts._x_axis: detect open-loop via RunData.is_open_loop; drop _RATE_DIR_RE - RunData.name: derive label from is_open_loop/rate; render the rate<0 sentinel as "Open-loop / Number M" instead of a meaningless "Rate -1.0 rps" - add network-free unit test tests/perf/test_perf_report_loader.py --- evalscope/perf/utils/perf_models.py | 15 ++-- evalscope/perf/utils/report/perf_charts.py | 4 +- tests/perf/test_perf_report_loader.py | 80 ++++++++++++++++++++++ 3 files changed, 89 insertions(+), 10 deletions(-) create mode 100644 tests/perf/test_perf_report_loader.py diff --git a/evalscope/perf/utils/perf_models.py b/evalscope/perf/utils/perf_models.py index 69a04a413..d7a1be7ab 100644 --- a/evalscope/perf/utils/perf_models.py +++ b/evalscope/perf/utils/perf_models.py @@ -434,14 +434,15 @@ def sort_key(self) -> float: def name(self) -> str: """Human-readable run label. - * ``parallel__number_`` directories → "Parallel N / Number M" - * ``rate__number_`` directories → "Rate R rps / Number M" + * closed-loop (``parallel__number_``) → "Parallel N / Number M" + * open-loop with a fixed rate → "Rate R rps / Number M" + * open-loop with no fixed rate (rate=-1) → "Open-loop / Number M" """ - import re - m = re.match(r'^rate_([\d.]+)_number_(\d+)$', self.dir_name) - if m: - return f'Rate {m.group(1)} rps / Number {m.group(2)}' - return f'Parallel {self.parallel} / Number {self.number}' + if not self.is_open_loop: + return f'Parallel {self.parallel} / Number {self.number}' + if self.rate is not None and self.rate < 0: + return f'Open-loop / Number {self.number}' + return f'Rate {self.rate} rps / Number {self.number}' @property def success_rate(self) -> float: diff --git a/evalscope/perf/utils/report/perf_charts.py b/evalscope/perf/utils/report/perf_charts.py index 835b3787f..5bcb04258 100644 --- a/evalscope/perf/utils/report/perf_charts.py +++ b/evalscope/perf/utils/report/perf_charts.py @@ -180,8 +180,6 @@ def histogram( # X-axis helper: adapts to open-loop (rate) vs closed-loop (concurrency) # --------------------------------------------------------------------------- -_RATE_DIR_RE = re.compile(r'^rate_[\d.]+_number_\d+$') - def _x_axis(runs: 'List[RunData]'): """Return *(xs, x_title)* appropriate for the run mode. @@ -191,7 +189,7 @@ def _x_axis(runs: 'List[RunData]'): * closed-loop (``parallel_*_number_*`` dirs) → xs = concurrency integers, x_title = ``'Concurrency'`` """ - is_open = any(_RATE_DIR_RE.match(r.dir_name) for r in runs) + is_open = any(r.is_open_loop for r in runs) if is_open: return [r.summary.request_rate for r in runs], 'Rate (req/s)' return [r.parallel for r in runs], 'Concurrency' diff --git a/tests/perf/test_perf_report_loader.py b/tests/perf/test_perf_report_loader.py new file mode 100644 index 000000000..10e896bec --- /dev/null +++ b/tests/perf/test_perf_report_loader.py @@ -0,0 +1,80 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. +"""Unit tests for perf HTML-report run discovery and labeling. + +Network-free: builds minimal fake run sub-directories on disk and exercises +``RunLoader.load_all`` plus the ``RunData``/chart helpers directly. + +Regression for open-loop runs with no fixed rate (``rate_-1.0_number_``), +which were previously dropped by the loader regex so no HTML report was +generated. +""" +import json +import os +import tempfile +import unittest + + +def _write_json(path: str, obj: object) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w', encoding='utf-8') as f: + json.dump(obj, f) + + +def _make_run(base: str, dir_name: str, *, request_rate: float) -> None: + """Create a minimal run sub-directory recognized by ``RunLoader``.""" + sub = os.path.join(base, dir_name) + _write_json(os.path.join(sub, 'benchmark_summary.json'), {'Request rate': request_rate}) + _write_json(os.path.join(sub, 'benchmark_percentile.json'), []) + _write_json(os.path.join(sub, 'benchmark_args.json'), {'api': 'openai'}) + + +class TestPerfReportLoader(unittest.TestCase): + + def setUp(self): + self.tmp = tempfile.mkdtemp() + _make_run(self.tmp, 'parallel_2_number_5', request_rate=-1.0) + _make_run(self.tmp, 'rate_1.5_number_4', request_rate=1.5) + _make_run(self.tmp, 'rate_-1.0_number_30', request_rate=-1.0) + + def tearDown(self): + import shutil + shutil.rmtree(self.tmp, ignore_errors=True) + + def _by_dir(self): + from evalscope.perf.utils.report.perf_data import RunLoader + runs = RunLoader.load_all(self.tmp, with_requests=False) + return {r.dir_name: r for r in runs} + + def test_open_loop_no_fixed_rate_is_discovered(self): + """rate_-1.0_number_30 must be loaded (previously silently dropped).""" + runs = self._by_dir() + self.assertIn('rate_-1.0_number_30', runs) + run = runs['rate_-1.0_number_30'] + self.assertTrue(run.is_open_loop) + self.assertEqual(run.name, 'Open-loop / Number 30') + + def test_fixed_rate_run_labeled_by_rate(self): + run = self._by_dir()['rate_1.5_number_4'] + self.assertTrue(run.is_open_loop) + self.assertEqual(run.name, 'Rate 1.5 rps / Number 4') + + def test_closed_loop_run_unaffected(self): + run = self._by_dir()['parallel_2_number_5'] + self.assertFalse(run.is_open_loop) + self.assertEqual(run.name, 'Parallel 2 / Number 5') + + def test_x_axis_detects_open_loop(self): + from evalscope.perf.utils.report import perf_charts + run = self._by_dir()['rate_-1.0_number_30'] + _, x_title = perf_charts._x_axis([run]) + self.assertEqual(x_title, 'Rate (req/s)') + + def test_x_axis_closed_loop(self): + from evalscope.perf.utils.report import perf_charts + run = self._by_dir()['parallel_2_number_5'] + _, x_title = perf_charts._x_axis([run]) + self.assertEqual(x_title, 'Concurrency') + + +if __name__ == '__main__': + unittest.main()