From 1c9dced7f0a8977962d8344813a9cb1398db627c Mon Sep 17 00:00:00 2001 From: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> Date: Thu, 16 Jul 2026 00:01:19 -0700 Subject: [PATCH 1/4] fix(perf): update disaggregated GB200 test configs Signed-off-by: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> --- .../test_lists/qa/llm_perf_disagg.yml | 6 -- .../test_lists/qa/llm_perf_multinode.txt | 1 - ...x1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml | 93 ------------------- ...x1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml | 4 +- 4 files changed, 2 insertions(+), 102 deletions(-) delete mode 100644 tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml diff --git a/tests/integration/test_lists/qa/llm_perf_disagg.yml b/tests/integration/test_lists/qa/llm_perf_disagg.yml index c6095b030e58..95e3d1e012c5 100644 --- a/tests/integration/test_lists/qa/llm_perf_disagg.yml +++ b/tests/integration/test_lists/qa/llm_perf_disagg.yml @@ -11,7 +11,6 @@ llm_perf_disagg: - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-UCX] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_128k8k_ctx1_pp4_gen8_pp4_bs2_eplb0_mtp0_con2-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) @@ -37,7 +36,6 @@ llm_perf_disagg: - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] TIMEOUT (120) @@ -73,14 +71,11 @@ llm_perf_disagg: - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL_kv-reuse] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-v32-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL] TIMEOUT (120) # GB200 accuracy cases - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_accuracy-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_accuracy-deepseek-r1-fp4_gpqa_diamond_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL] TIMEOUT (120) # GB200 stress cases - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_wideep_stress-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL] TIMEOUT (120) @@ -91,7 +86,6 @@ llm_perf_disagg: - perf/test_perf_sanity.py::test_e2e[aggr-ctx_only-gb200_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL] TIMEOUT (120) # GB200 aggregated gen_only - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_qwen3-235b-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL] TIMEOUT (120) - - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL] TIMEOUT (120) # 2: GB300 test cases - condition: diff --git a/tests/integration/test_lists/qa/llm_perf_multinode.txt b/tests/integration/test_lists/qa/llm_perf_multinode.txt index 1de41b361772..9add8de9cda6 100644 --- a/tests/integration/test_lists/qa/llm_perf_multinode.txt +++ b/tests/integration/test_lists/qa/llm_perf_multinode.txt @@ -4,7 +4,6 @@ perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-UCX] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_128k8k_ctx1_pp4_gen8_pp4_bs2_eplb0_mtp0_con2-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp0_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml deleted file mode 100644 index 74801acda6f8..000000000000 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ /dev/null @@ -1,93 +0,0 @@ -metadata: - model_name: deepseek_r1_0528_fp4_v2 - precision: fp4 - model_dir_name: DeepSeek-R1-0528-FP4-v2 - supported_gpus: - - GB200 - script_file: disaggr_torch.slurm - benchmark_type: 128k8k -slurm: - script_file: disaggr_torch.slurm - partition: - account: - job_time: 02:00:00 - job_name: unified-benchmark - extra_args: --gres=gpu:4 - numa_bind: true -benchmark: - mode: e2e - use_nv_sa_benchmark: false - multi_round: 10 - benchmark_ratio: 0.0 - streaming: true - concurrency_list: '64' - input_length: 131072 - output_length: 8192 - dataset_file: datasets/perf-ci/deepseek_r1-128k8k-20480-ratio-1_for_serve.json -hardware: - gpus_per_node: 4 - num_ctx_servers: 1 - num_gen_servers: 1 -environment: - container_mount: - container_image: - model_path: - trtllm_repo: '' - build_wheel: false - work_dir: - worker_env_var: TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 ENROOT_ALLOW_DEV=yes TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS=3 - server_env_var: TRTLLM_SERVER_DISABLE_GC=1 -profiling: - nsys_on: false -worker_config: - gen: - print_iter_log: true - max_batch_size: 2 - max_num_tokens: 8 - tensor_parallel_size: 32 - moe_expert_parallel_size: 32 - pipeline_parallel_size: 1 - context_parallel_size: 1 - enable_attention_dp: true - enable_lm_head_tp_in_adp: true - cuda_graph_config: - enable_padding: true - max_batch_size: 2 - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.6 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 131104 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: &id001 - decoding_type: MTP - max_draft_len: 3 - num_postprocess_workers: 4 - stream_interval: 20 - ctx: - print_iter_log: true - max_batch_size: 1 - max_num_tokens: 131104 - tensor_parallel_size: 1 - moe_expert_parallel_size: 1 - pipeline_parallel_size: 8 - context_parallel_size: 1 - enable_attention_dp: false - cuda_graph_config: null - kv_cache_config: - enable_block_reuse: false - free_gpu_memory_fraction: 0.3 - dtype: fp8 - moe_config: - backend: CUTEDSL - use_low_precision_moe_combine: true - cache_transceiver_config: - max_tokens_in_buffer: 131104 - backend: NIXL - disable_overlap_scheduler: true - speculative_config: *id001 diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml index 63f2ef1ae8f9..351e96d01d5e 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL.yaml @@ -52,13 +52,13 @@ worker_config: pipeline_parallel_size: 1 context_parallel_size: 1 enable_attention_dp: true - max_batch_size: 8 + max_batch_size: 32 max_num_tokens: 8192 max_seq_len: 121000 allreduce_strategy: MNNVL cuda_graph_config: enable_padding: true - max_batch_size: 8 + max_batch_size: 32 kv_cache_config: enable_block_reuse: false free_gpu_memory_fraction: 0.9 From d85782cfef99ee39870cc3b999dba54c4dd01348 Mon Sep 17 00:00:00 2001 From: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> Date: Thu, 16 Jul 2026 00:33:36 -0700 Subject: [PATCH 2/4] fix(ci): remove obsolete GB200 test stage Signed-off-by: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> --- jenkins/L0_Test.groovy | 9 --------- ..._sanity_ctx1_node2_gpu8_gen1_node8_gpu32.yml | 17 ----------------- ...gb200_multi_nodes_perf_sanity_node2_gpu8.yml | 1 - tests/integration/test_lists/waives.txt | 1 - 4 files changed, 28 deletions(-) delete mode 100644 tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node8_gpu32.yml diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 3f72edbf9e93..965363c1827d 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -5041,15 +5041,6 @@ def launchTestJobs(pipeline, testFilter) 36, 9 ) - // 10 Nodes - multiNodesSBSAConfigs += buildStageConfigs( - "GB200-40_GPUs-10_Nodes-PyTorch-Disagg-PerfSanity-CTX1-NODE2-GPU8-GEN1-NODE8-GPU32-Post-Merge", - "auto:gb200-flex", - "l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node8_gpu32", - 1, - 40, - 10 - ) // GB300 PerfSanity post-merge aggregated // 2 Nodes multiNodesSBSAConfigs += buildStageConfigs( diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node8_gpu32.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node8_gpu32.yml deleted file mode 100644 index 4f36687cabbb..000000000000 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node8_gpu32.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: 0.0.1 -l0_gb200_multi_nodes_perf_sanity_ctx1_node2_gpu8_gen1_node8_gpu32: -- condition: - ranges: - # 1 ctx worker with each 2 node and 8 GPUs - # 1 gen worker with each 8 node and 32 GPUs - system_gpu_count: - gte: 40 - lte: 40 - wildcards: - gpu: - - '*gb200*' - terms: - stage: post_merge - backend: pytorch - tests: - - perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (180) diff --git a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_node2_gpu8.yml b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_node2_gpu8.yml index b1d454c5ad34..f3f79c3f63ac 100644 --- a/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_node2_gpu8.yml +++ b/tests/integration/test_lists/test-db/l0_gb200_multi_nodes_perf_sanity_node2_gpu8.yml @@ -29,4 +29,3 @@ l0_gb200_multi_nodes_perf_sanity_node2_gpu8: # deepseek-r1-fp4 - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] TIMEOUT (180) - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL] TIMEOUT (180) - # - perf/test_perf_sanity.py::test_e2e[aggr_upload-ctx_only-gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL] TIMEOUT (180) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index ca09390043e3..68bcf565d1ea 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -416,7 +416,6 @@ perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_deepseek-v32-fp4_8k1k perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_kimi-k25-thinking-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6463814) perf/test_perf_sanity.py::test_e2e[disagg_upload-e2e-gb200_qwen3-235b-fp4_8k1k_con1024_ctx1_tp1_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6422339) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-NIXL] SKIP (https://nvbugs/6463814) -perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6422339) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6422339) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep8_eplb0_mtp0_ccb-NIXL] SKIP (https://nvbugs/6422339) perf/test_perf_sanity.py::test_e2e[disagg_upload-gen_only-gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL] SKIP (https://nvbugs/6422339) From c682318eb06df65ed8923be13108d0a7cf363ca1 Mon Sep 17 00:00:00 2001 From: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> Date: Thu, 16 Jul 2026 19:35:28 -0700 Subject: [PATCH 3/4] fix(perf): remove disabled multinode test cases Signed-off-by: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> --- tests/integration/test_lists/qa/llm_perf_multinode.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/integration/test_lists/qa/llm_perf_multinode.txt b/tests/integration/test_lists/qa/llm_perf_multinode.txt index 9add8de9cda6..77eec4fca9ad 100644 --- a/tests/integration/test_lists/qa/llm_perf_multinode.txt +++ b/tests/integration/test_lists/qa/llm_perf_multinode.txt @@ -30,7 +30,6 @@ perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_32k4k_con1_ perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep8_gen1_dep8_eplb0_mtp0_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_deepseek-v32-fp4_8k1k_con4096_ctx1_dep4_gen1_dep32_eplb256_mtp0_ccb-NIXL] @@ -123,11 +122,9 @@ perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_wideep_deepseek-r1-fp4_8k1k_ perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL_kv-reuse] -perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX] perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-v32-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL] -perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL] # GB300 wideep @@ -138,7 +135,6 @@ perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb300_wideep_deepseek-r1-fp4_ perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb300_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL] # accuracy cases -perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_accuracy-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_accuracy-deepseek-r1-fp4_gpqa_diamond_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL] # stress cases From 6a2504b9ea5b831e237b4a1207d9befbb075b353 Mon Sep 17 00:00:00 2001 From: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> Date: Thu, 16 Jul 2026 22:05:42 -0700 Subject: [PATCH 4/4] fix(perf): restore verified GB200 gen-only tests Signed-off-by: Lizhi Zhou <1432185+reasonsolo@users.noreply.github.com> --- tests/integration/test_lists/qa/llm_perf_disagg.yml | 4 ++++ tests/integration/test_lists/qa/llm_perf_multinode.txt | 3 +++ 2 files changed, 7 insertions(+) diff --git a/tests/integration/test_lists/qa/llm_perf_disagg.yml b/tests/integration/test_lists/qa/llm_perf_disagg.yml index 95e3d1e012c5..54540686da56 100644 --- a/tests/integration/test_lists/qa/llm_perf_disagg.yml +++ b/tests/integration/test_lists/qa/llm_perf_disagg.yml @@ -71,11 +71,14 @@ llm_perf_disagg: - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL_kv-reuse] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-v32-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL] TIMEOUT (120) # GB200 accuracy cases + - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_accuracy-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL] TIMEOUT (120) - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_accuracy-deepseek-r1-fp4_gpqa_diamond_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL] TIMEOUT (120) # GB200 stress cases - perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_wideep_stress-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL] TIMEOUT (120) @@ -86,6 +89,7 @@ llm_perf_disagg: - perf/test_perf_sanity.py::test_e2e[aggr-ctx_only-gb200_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL] TIMEOUT (120) # GB200 aggregated gen_only - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_qwen3-235b-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp0_con1_ccb-NIXL] TIMEOUT (120) + - perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_qwen3-235b-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_con2048_ccb-NIXL] TIMEOUT (120) # 2: GB300 test cases - condition: diff --git a/tests/integration/test_lists/qa/llm_perf_multinode.txt b/tests/integration/test_lists/qa/llm_perf_multinode.txt index 77eec4fca9ad..2e6b76a1961e 100644 --- a/tests/integration/test_lists/qa/llm_perf_multinode.txt +++ b/tests/integration/test_lists/qa/llm_perf_multinode.txt @@ -122,9 +122,11 @@ perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_wideep_deepseek-r1-fp4_8k1k_ perf/test_perf_sanity.py::test_e2e[disagg-e2e-gb200_wideep_deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_con1024_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL_kv-reuse] +perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-UCX] perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-r1-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-v32-fp4_1k1k_ctx1_gen1_dep32_bs32_eplb288_mtp0_con1024_ccb-NIXL] +perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep48_bs16_eplb288_mtp3_con12288_ccb-NIXL] # GB300 wideep @@ -135,6 +137,7 @@ perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb300_wideep_deepseek-r1-fp4_ perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb300_wideep_deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_con2048_ccb-NIXL] # accuracy cases +perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_accuracy-deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL] perf/test_perf_sanity.py::test_e2e[disagg-gen_only-gb200_wideep_accuracy-deepseek-r1-fp4_gpqa_diamond_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL] # stress cases