[TRTLLM-11229][infra] Save unittest subtest results periodically

jenkins/scripts/generate_timeout_xml.py

@@ -131,7 +131,7 @@ def main():
 
     try:
         with open(full_path, "r", encoding="utf-8") as f:
-            timeoutTests = [line.strip() for line in f if line.strip()]
+            timeoutTests = list(dict.fromkeys(line.strip() for line in f if line.strip()))
     except IOError as e:
         print(f"Error reading {full_path}: {e}")
         return

tests/integration/defs/test_unittests.py

@@ -135,8 +135,17 @@ def test_unittests_v2(llm_root, llm_venv, case: str, output_dir, request):
                                   f'results-sub-unittests-{case_fn}.xml')
 
     command = [
-        '-m', 'pytest', ignore_opt, "-vv", "--tb=short", "-rF",
-        "--timeout=2400", "--timeout-method=thread"
+        '-m',
+        'pytest',
+        ignore_opt,
+        "-vv",
+        "--tb=short",
+        "-rF",
+        "--timeout=2400",
+        "--timeout-method=thread",
+        "--periodic-junit",
+        "--periodic-batch-size=1",
+        "--periodic-save-unfinished-test",
     ]
     if test_prefix:
         command += [f"--test-prefix={test_prefix}"]
@@ -184,15 +193,16 @@ def run_command(cmd, num_workers=1):
 
     if num_workers == 1:
         # Do not bother with pytest-xdist at all if we don't need parallel execution
-        command += ["-p", "no:xdist", f"--junitxml={output_xml}"]
+        command += ["-p", "no:xdist", f"--periodic-junit-xmlpath={output_xml}"]
         passed = run_command(command)
     else:
         # Avoid .xml extension to prevent CI from reading failures from it
         parallel_output_xml = os.path.join(
             output_dir,
             f'parallel-sub-results-unittests-{case_fn}.xml.intermediate')
         parallel_command = command + [
-            "-n", f"{num_workers}", f"--junitxml={parallel_output_xml}"
+            "-n", f"{num_workers}",
+            f"--periodic-junit-xmlpath={parallel_output_xml}"
         ]
         passed = run_command(parallel_command, num_workers)
 
@@ -209,7 +219,8 @@ def run_command(cmd, num_workers=1):
                 f'retry-sub-results-unittests-{case_fn}.xml.intermediate')
             # Run failed case sequentially.
             retry_command = command + [
-                "-p", "no:xdist", '--lf', f"--junitxml={retry_output_xml}"
+                "-p", "no:xdist", '--lf',
+                f"--periodic-junit-xmlpath={retry_output_xml}"
             ]
             passed = run_command(retry_command)
 

tests/unittest/conftest.py

@@ -51,6 +51,37 @@ def pytest_configure(config):
     # Dump all threads' stacks when SIGALRM is received
     signal.signal(signal.SIGALRM, dump_threads)
 
+    # Register PeriodicJUnitXML when invoked from integration test_unittests.py
+    periodic = config.getoption("--periodic-junit", default=False)
+    periodic_junit_xmlpath = config.getoption("--periodic-junit-xmlpath",
+                                              default=None)
+    if periodic and periodic_junit_xmlpath:
+        from integration.defs.trt_test_alternative import (print_info,
+                                                           print_warning)
+        from integration.defs.utils.periodic_junit import PeriodicJUnitXML
+        periodic_interval = config.getoption("--periodic-interval")
+        periodic_batch_size = config.getoption("--periodic-batch-size")
+        periodic_save_unfinished_test = config.getoption(
+            "--periodic-save-unfinished-test", default=False)
+        xml_dir = os.path.dirname(periodic_junit_xmlpath)
+        if xml_dir:
+            os.makedirs(xml_dir, exist_ok=True)
+        reporter = PeriodicJUnitXML(
+            xmlpath=periodic_junit_xmlpath,
+            interval=periodic_interval,
+            batch_size=periodic_batch_size,
+            logger={
+                'info': print_info,
+                'warning': print_warning
+            },
+            save_unfinished_test=periodic_save_unfinished_test,
+        )
+        reporter.pytest_configure(config)
+        config.pluginmanager.register(reporter, 'periodic_junit')
+        print_info("PeriodicJUnitXML reporter registered (unittest)")
+        print_info(f"  XML path: {periodic_junit_xmlpath}")
+        print_info(f"  Batch size: {periodic_batch_size}")
+
 
 @pytest.hookimpl(wrapper=True)
 def pytest_runtest_protocol(item, nextitem):
@@ -118,6 +149,45 @@ def pytest_addoption(parser):
         "Specify a file containing a list of waives, one per line. After filtering collected tests, Pytest will "
         "apply the waive state specified by this file to the set of tests to be run.",
     )
+    # Periodic JUnit options: must be registered here so they are recognized when
+    # pytest is run with unittest paths (integration test_unittests.py spawns such a run).
+    parser.addoption(
+        "--periodic-junit",
+        action="store_true",
+        default=False,
+        help=
+        "Enable periodic JUnit XML reporter. Only used when invoked from integration test_unittests.",
+    )
+    parser.addoption(
+        "--periodic-interval",
+        action="store",
+        type=int,
+        default=18000,
+        help=
+        "Time interval in seconds between periodic saves. Only used with --periodic-junit.",
+    )
+    parser.addoption(
+        "--periodic-batch-size",
+        action="store",
+        type=int,
+        default=10,
+        help=
+        "Number of completed tests before triggering a periodic save. Only used with --periodic-junit.",
+    )
+    parser.addoption(
+        "--periodic-junit-xmlpath",
+        action="store",
+        default=None,
+        help=
+        "Path to the output XML file for periodic JUnit XML reporter. Only used with --periodic-junit.",
+    )
+    parser.addoption(
+        "--periodic-save-unfinished-test",
+        action="store_true",
+        default=False,
+        help=
+        "Save unfinished test name to unfinished_test.txt. Only used with --periodic-junit.",
+    )
 
 
 def apply_waives_ut(waives_file, items: list[pytest.Item], config):