From 62567e01f7147c6e07efd6e1de1ef8cdcdca1450 Mon Sep 17 00:00:00 2001 From: Urvashi Tiwari Date: Wed, 24 Jun 2026 17:38:46 -0400 Subject: [PATCH 1/2] Removed examples from TB and added a tag to build TB from that repo to resolve Tb errors Signed-off-by: Urvashi Tiwari --- .../health/mi300_health_config.json | 12 +- cvs/tests/health/README.md | 1 - .../health/install/install_transferbench.py | 10 +- cvs/tests/health/transferbench_cvs.py | 142 ++---------------- docs/how-to/run-cvs-tests.rst | 1 - docs/install/cvs-install.rst | 1 - docs/reference/configuration-files/health.rst | 37 +---- 7 files changed, 32 insertions(+), 172 deletions(-) diff --git a/cvs/input/config_file/health/mi300_health_config.json b/cvs/input/config_file/health/mi300_health_config.json index 788ecb406..b6d17c1c5 100644 --- a/cvs/input/config_file/health/mi300_health_config.json +++ b/cvs/input/config_file/health/mi300_health_config.json @@ -9,9 +9,9 @@ }, "transferbench": { "path": "/home/{user-id}/INSTALL/TransferBench", - "example_tests_path": "/home/{user-id}/INSTALL/TransferBench/examples", "git_install_path": "/home/{user-id}/INSTALL/", "git_url": "https://github.com/ROCm/TransferBench.git", + "git_tag": "v1.67.00", "_comment_rocm_path": "ROCm installation path. Set the placeholder changeme to auto-detect from /opt/rocm or /opt/rocm/core-*", "rocm_path": "", "results": { @@ -26,15 +26,7 @@ "32_cu_local_copy": "1250.0", "32_cu_rem_read": "48.0", "32_cu_rem_write": "48.0", - "32_cu_rem_copy": "48.0", - "example_results": { - "test1": "47.1", - "test2": "48.4", - "test3_0_to_1": "31.9", - "test3_1_to_0": "38.9", - "test4": "1264", - "test6": "48.6" - } + "32_cu_rem_copy": "48.0" } }, "rvs": { diff --git a/cvs/tests/health/README.md b/cvs/tests/health/README.md index b8d568f8a..d4f4700e1 100644 --- a/cvs/tests/health/README.md +++ b/cvs/tests/health/README.md @@ -34,7 +34,6 @@ pytest -vvv -log-file=/tmp/agfhc_test.log -s ./tests/health/agfhc_cvs.py --clust ``` In the config file, cvs/input/config_file/health/mi300_health_config.json, change the value of parameters(/tmp/cvs): "path": "/tmp/cvs/INSTALL/TransferBench", -"example_tests_path": "/tmp/cvs/INSTALL/TransferBench/examples", "git_install_path": "/tmp/cvs/INSTALL/", to desired location. pytest -vvv --log-file=/tmp/test.log -s ./tests/health/install/install_agfhc.py --cluster_file input/cluster_file/cluster.json --config_file input/config_file/health/mi300_health_config.json --html=/var/www/html/cvs/agfhc.html --capture=tee-sys --self-contained-html diff --git a/cvs/tests/health/install/install_transferbench.py b/cvs/tests/health/install/install_transferbench.py index 296a4ba17..19270f06e 100644 --- a/cvs/tests/health/install/install_transferbench.py +++ b/cvs/tests/health/install/install_transferbench.py @@ -183,6 +183,7 @@ def test_install_transferbench(orch, config_dict): Steps: - Ensure git_install_path exists on all nodes. - Clone TransferBench repo under git_install_path on every node. + - Checkout the configured git tag. - Build on all nodes using detected ROCM_PATH and HIPCC. - Verify the build artifact is present on each node. @@ -193,6 +194,7 @@ def test_install_transferbench(orch, config_dict): config_dict (dict): Includes: - git_install_path: directory to clone/build - git_url: repository URL + - git_tag: git tag to checkout after clone """ globals.error_list = [] @@ -213,6 +215,13 @@ def test_install_transferbench(orch, config_dict): timeout=120, ) + tb_src = f'{git_install_path}/TransferBench' + git_tag = config_dict['git_tag'] + out_dict = orch.exec( + f"bash -c 'cd {tb_src} && git checkout {git_tag}'", + timeout=120, + ) + # Detect ROCm path and compiler rocm_path = detect_rocm_path(orch, config_dict.get('rocm_path', '')) hip_compiler = detect_hip_compiler(orch, rocm_path) @@ -221,7 +230,6 @@ def test_install_transferbench(orch, config_dict): # uses cwd-relative paths so cwd MUST be the source tree; we wrap the # build in `bash -c` explicitly to make the cwd dependency visible # at the call site rather than via an implicit `cd X; cmd` chain. - tb_src = f'{git_install_path}/TransferBench' out_dict = orch.exec( f"bash -c 'cd {tb_src} && ROCM_PATH={rocm_path} HIPCC={hip_compiler} make'", timeout=500, diff --git a/cvs/tests/health/transferbench_cvs.py b/cvs/tests/health/transferbench_cvs.py index a64cc6c18..236c38aac 100644 --- a/cvs/tests/health/transferbench_cvs.py +++ b/cvs/tests/health/transferbench_cvs.py @@ -152,7 +152,21 @@ def parse_tb_p2p_bw(out_dict, exp_dict): def parse_tb_scaling_bw(out_dict, exp_dict): for node in out_dict.keys(): log.info(f"^^^^^ {out_dict[node]} ^^^^^^") - match = re.search('Best\s+[0-9\.]+\(\s[0-9]+\)\s+[0-9\.]+\(\s[0-9]+\)\s+([0-9\.]+)', out_dict[node]) + # Best summary row: skip CPU00/CPU01 columns, capture GPU00 peak bandwidth. + # CU counts in parentheses may be padded (e.g. "( 8)" vs "( 32)"). + match = re.search( + r'(?m)^[ \t]*Best\s+' + r'(?:[0-9\.]+\(\s*[0-9]+\)\s+){2}' + r'([0-9\.]+)', + out_dict[node], + ) + if not match: + out = out_dict[node] + tail = 6000 + if len(out) > tail: + out = f"...[truncated {len(out) - tail} chars; Best row is usually near the end]...\n{out[-tail:]}" + fail_test(f"TransferBench scaling Best row GPU00 bandwidth not found on node {node}. Output: {out}") + continue gpu0_bw = float(match.group(1)) if float(gpu0_bw) < float(exp_dict['best_gpu0_bw']): fail_test( @@ -208,130 +222,6 @@ def parse_tb_schmoo_bw(out_dict, exp_dict): ) -def parse_tb_example_test_results(out_dict, exp_dict): - for node in out_dict.keys(): - # Test 1 results parse - match = re.search('(Test\s1:\n.*\n.*\n.*\n)', out_dict[node]) - if not match: - fail_test(f"Test 1 output section not found in TransferBench output on node {node}") - continue - test1_out = match.group(1) - match = re.search(r'(?:Transfer|Executor:\s+GPU)\s+[0-9]+\s+[|│]\s*([0-9\.]+)\s+GB/s', test1_out, re.I) - if not match: - fail_test(f"Transfer bandwidth pattern not found in Test 1 output on node {node}. Output: {test1_out}") - continue - test1_res = match.group(1) - if float(test1_res) < float(exp_dict['test1']): - fail_test( - f"Transfer Bench example test1 failed actual value {test1_res} is less than expected {exp_dict['test1']}" - ) - - # Test 2 results parse - match = re.search('(Test\s2:\n.*\n.*\n.*\n)', out_dict[node]) - if not match: - fail_test(f"Test 2 output section not found in TransferBench output on node {node}") - continue - test2_out = match.group(1) - match = re.search(r'(?:Transfer|Executor:\s+(?:GPU|DMA))\s+[0-9]+\s+[|│]\s*([0-9\.]+)\s+GB/s', test2_out, re.I) - if not match: - fail_test(f"Transfer bandwidth pattern not found in Test 2 output on node {node}. Output: {test2_out}") - continue - test2_res = match.group(1) - if float(test2_res) < float(exp_dict['test2']): - fail_test( - f"Transfer Bench example test2 failed actual value {test2_res} is less than expected {exp_dict['test2']}" - ) - - # Test 3 results parse - match = re.search('(Test\s3:.*?)(?=Test\s[456]:|##|$)', out_dict[node], re.DOTALL) - if not match: - fail_test(f"Test 3 output section not found in TransferBench output on node {node}") - continue - test3_out = match.group(1) - - match = re.search( - r'(?:Transfer|Executor:\s+GPU)\s+[0-9]+\s+[|│]\s*([0-9\.]+)\s+GB/s(?:[\s0-9\.\|a-z]+\s+G0\s*-\>)?', - test3_out, - re.I, - ) - if not match: - fail_test( - f"G0->G1 transfer bandwidth pattern not found in Test 3 output on node {node}. Output: {test3_out}" - ) - continue - test3_res_0_1 = match.group(1) - if float(test3_res_0_1) < float(exp_dict['test3_0_to_1']): - fail_test( - f"Transfer Bench example test3 failed actual value {test3_res_0_1} is less than expected {exp_dict['test3_0_to_1']}" - ) - - match = re.search( - r'(?:Transfer|Executor:\s+GPU)\s+[0-9]+\s+[|│]\s*([0-9\.]+)\s+GB/s(?:[\s0-9\.\|a-z]+\s+G1\s*-\>)?', - test3_out, - re.I, - ) - if not match: - fail_test( - f"G1->G0 transfer bandwidth pattern not found in Test 3 output on node {node}. Output: {test3_out}" - ) - continue - test3_res_1_0 = match.group(1) - if float(test3_res_1_0) < float(exp_dict['test3_1_to_0']): - fail_test( - f"Transfer Bench example test3 failed actual value {test3_res_1_0} is less than expected {exp_dict['test3_1_to_0']}" - ) - - # Test 4 results parse - match = re.search('(Test\s4:\n.*\n.*\n.*\n)', out_dict[node]) - if not match: - fail_test(f"Test 4 output section not found in TransferBench output on node {node}") - continue - test4_out = match.group(1) - match = re.search(r'(?:Transfer|Executor:\s+GPU)\s+[0-9]+\s+[|│]\s*([0-9\.]+)\s+GB/s', test4_out, re.I) - if not match: - fail_test(f"Transfer bandwidth pattern not found in Test 4 output on node {node}. Output: {test4_out}") - continue - test4_res = match.group(1) - if float(test4_res) < float(exp_dict['test4']): - fail_test( - f"Transfer Bench example test4 failed actual value {test4_res} is less than expected {exp_dict['test4']}" - ) - - # Test 5 CPU only .. skip - # Test 6 results parse - match = re.search('(Test\s6:\n.*\n.*\n.*\n)', out_dict[node]) - if not match: - fail_test(f"Test 6 output section not found in TransferBench output on node {node}") - continue - test6_out = match.group(1) - match = re.search(r'Executor:\s+GPU\s+[0-9]+\s+│\s*([0-9\.]+)\s+GB/s', test6_out, re.I) - if not match: - fail_test(f"Transfer bandwidth pattern not found in Test 6 output on node {node}. Output: {test6_out}") - continue - test6_res = match.group(1) - if float(test6_res) < float(exp_dict['test6']): - fail_test( - f"Transfer Bench example test6 failed actual value {test6_res} is less than expected {exp_dict['test6']}" - ) - - -def test_transfer_bench_example_tests_1_6_t(orch, config_dict): - globals.error_list = [] - log.info('Testcase Run TransferBench example tests 1-6') - path = config_dict['path'] - rocm_path = detect_rocm_path(orch, config_dict.get('rocm_path', '')) - log.info("%s", config_dict) - example_path = config_dict['example_tests_path'] - out_dict = orch.exec( - f"sudo bash -c 'export LD_LIBRARY_PATH={rocm_path}/lib:$LD_LIBRARY_PATH && echo \"LD_LIBRARY_PATH: $LD_LIBRARY_PATH\" && {path}/TransferBench {example_path}/example.cfg'", - timeout=(60 * 5), - ) - print_test_output(log, out_dict) - scan_test_results(out_dict) - parse_tb_example_test_results(out_dict, config_dict['results']['example_results']) - update_test_result() - - def test_transfer_bench_a2a( orch, config_dict, @@ -412,7 +302,7 @@ def test_transfer_bench_scaling( path = config_dict['path'] rocm_path = detect_rocm_path(orch, config_dict.get('rocm_path', '')) out_dict = orch.exec( - f"sudo bash -c 'export LD_LIBRARY_PATH={rocm_path}/lib:$LD_LIBRARY_PATH && echo \"LD_LIBRARY_PATH: $LD_LIBRARY_PATH\" && {path}/TransferBench scaling'", + f"sudo bash -c 'export LD_LIBRARY_PATH={rocm_path}/lib:$LD_LIBRARY_PATH && echo \"LD_LIBRARY_PATH: $LD_LIBRARY_PATH\" && GFX_TEMPORAL=3 GFX_UNROLL=32 {path}/TransferBench scaling'", timeout=(60 * 10), ) print_test_output(log, out_dict) diff --git a/docs/how-to/run-cvs-tests.rst b/docs/how-to/run-cvs-tests.rst index 9b5f18ffe..49320b6e0 100644 --- a/docs/how-to/run-cvs-tests.rst +++ b/docs/how-to/run-cvs-tests.rst @@ -267,7 +267,6 @@ You can list all available TransferBench test cases using the CLI: .. code:: text Available tests in transferbench_cvs: - - test_transfer_bench_example_tests_1_6_t - test_transfer_bench_a2a - test_transfer_bench_p2p - test_transfer_bench_healthcheck diff --git a/docs/install/cvs-install.rst b/docs/install/cvs-install.rst index 01121cdfb..db1da4f3b 100644 --- a/docs/install/cvs-install.rst +++ b/docs/install/cvs-install.rst @@ -241,7 +241,6 @@ Health - Under ``transferbench``: - - ``example_tests_path`` - ``git_install_path`` - Under ``rvs``: diff --git a/docs/reference/configuration-files/health.rst b/docs/reference/configuration-files/health.rst index 017e75bf3..a4a0ea95c 100644 --- a/docs/reference/configuration-files/health.rst +++ b/docs/reference/configuration-files/health.rst @@ -32,9 +32,9 @@ Here's a code snippet of the ``mi300_health_config.json`` file for reference: "transferbench": { "path": "/home/{user-id}/INSTALL/TransferBench", - "example_tests_path": "/home/{user-id}/INSTALL/TransferBench/examples", "git_install_path": "/home/{user-id}/INSTALL/", "git_url": "https://github.com/ROCm/TransferBench.git", + "git_tag": "v1.67.00", "_comment_rocm_path": "ROCm installation path. Set the placeholder changeme to auto-detect from /opt/rocm or /opt/rocm/core-*", "rocm_path": "", "results": @@ -50,16 +50,7 @@ Here's a code snippet of the ``mi300_health_config.json`` file for reference: "32_cu_local_copy": "1250.0", "32_cu_rem_read": "48.0", "32_cu_rem_write": "48.0", - "32_cu_rem_copy": "48.0", - "example_results": - { - "test1": "47.1", - "test2": "48.4", - "test3_0_to_1": "31.9", - "test3_1_to_0": "38.9", - "test4": "1264", - "test6": "48.6" - } + "32_cu_rem_copy": "48.0" } }, @@ -198,15 +189,15 @@ TransferBench * - ``path`` - ``/opt/amd/transferbench`` - Path where TransferBench is installed - * - ``example_tests_path`` - - ``/home/{user-id}/INSTALL`` |br| ``/TransferBench/examples`` - - Path where TransferBench examples are installed * - ``git_install_path`` - ``/home/{user-id}/INSTALL/`` - Path where the Git repo is installed * - ``git_url`` - `https://github.com/ROCm/TransferBench.git `_ - URL for Git repo + * - ``git_tag`` + - ``v1.67.00`` + - Git tag to checkout after cloning TransferBench * - ``rocm_path`` - ```` - Set the path of rocm @@ -243,24 +234,6 @@ TransferBench * - ``32_cu_rem_copy`` - 48.0 - Remote memory copy bandwidth (GB/s) using 32 CUs - * - ``test1`` - - 47.1 - - Specific benchmark result (likely bandwidth in GB/s) - * - ``test2`` - - 48.4 - - Another benchmark result - * - ``test3_0_to_1`` - - 31.9 - - Directional test from GPU 0 to GPU 1 - * - ``test3_1_to_0`` - - 38.9 - - Directional test from GPU 1 to GPU 0 - * - ``test4`` - - 1264 - - High-performance test result (possibly local memory) - * - ``test6`` - - 48.6 - - Additional benchmark result ROCm Validation Suite (RVS) --------------------------- From f8f967a6263f6d8fc7ac59f54f51b46f0fa7bf4e Mon Sep 17 00:00:00 2001 From: Urvashi Tiwari Date: Fri, 26 Jun 2026 15:23:01 -0400 Subject: [PATCH 2/2] Addressed the review comments Signed-off-by: Urvashi Tiwari --- .../health/install/install_transferbench.py | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/cvs/tests/health/install/install_transferbench.py b/cvs/tests/health/install/install_transferbench.py index 19270f06e..ecf4fe580 100644 --- a/cvs/tests/health/install/install_transferbench.py +++ b/cvs/tests/health/install/install_transferbench.py @@ -202,6 +202,11 @@ def test_install_transferbench(orch, config_dict): log.info('Testcase install transferbench') git_install_path = config_dict['git_install_path'] git_url = config_dict['git_url'] + git_tag = config_dict.get('git_tag') + if not git_tag: + fail_test('TransferBench config is missing the required "git_tag" field') + update_test_result() + return out_dict = orch.exec(f'ls -ld {git_install_path}') for node in out_dict.keys(): @@ -216,11 +221,25 @@ def test_install_transferbench(orch, config_dict): ) tb_src = f'{git_install_path}/TransferBench' - git_tag = config_dict['git_tag'] out_dict = orch.exec( f"bash -c 'cd {tb_src} && git checkout {git_tag}'", timeout=120, ) + scan_test_results(out_dict) + for node, output in out_dict.items(): + if re.search(r'error:|fatal:', output, re.I): + fail_test(f'git checkout {git_tag} failed on node {node}: {output.strip()}') + + out_dict = orch.exec( + f"bash -c 'cd {tb_src} && git describe --tags --exact-match'", + timeout=60, + ) + for node, output in out_dict.items(): + actual_tag = output.strip() + if actual_tag != git_tag: + fail_test( + f'TransferBench not at git tag {git_tag} on node {node} after checkout (git describe: {actual_tag!r})' + ) # Detect ROCm path and compiler rocm_path = detect_rocm_path(orch, config_dict.get('rocm_path', ''))