CLI: Fix TAP compliance for test plan when using QUnit.test.only()

We correctly prevent registration, counting, and execution of non-only
tests after the first QUnit.test.only call. And we correctly undo
the execution and registration of any QUnit.test calls before the
first QUnit.test.only call.

But, we did not clear the test *count*. This left the test count
emitted in `runEnd` event, and thus in the "1..X" test plan printed
by the TAP reporter, inaccurate. This has implications for TAP-based
test runners (such as QTap) as those rely on this being accurate to
declare a run as finished, stop waiting for results, and stop the
browser process.

Author: Krinkle

src/core/module.js

@@ -153,6 +153,41 @@ function processModule (name, options, scope, modifiers = {}) {
   }
 }
 
+/**
+ * Clear the SuiteReport tree of all tests and leave only current module as child suite
+ *
+ * This should be called before defining the first module.only() or test.only()
+ * because otherwise:
+ * - `runEnd.testCounts` is too high.
+ * - UI (HtmlReporter) and TAP (TapReporter) display totals too high.
+ * - Test runners like QTap might timeout because the TAP plan
+ *   would be printed as "1..9" even if only 2 tests are run,
+ *   which means tap-finished will wait for 3-9.
+ */
+export function clearSuiteReports (currentModule) {
+  let childSuite = null;
+  let suiteReport = currentModule.suiteReport;
+  while (suiteReport) {
+    suiteReport.tests.length = 0;
+    const i = suiteReport.childSuites.indexOf(childSuite);
+    if (i === -1) {
+      suiteReport.childSuites.length = 0;
+    } else {
+      // Reduce in-place to just currentModule.suiteReport or its intermediary
+      suiteReport.childSuites.splice(0, i);
+      suiteReport.childSuites.splice(1);
+    }
+
+    if (suiteReport === globalSuiteReport) {
+      suiteReport = null;
+    } else {
+      childSuite = suiteReport;
+      currentModule = currentModule.parentModule;
+      suiteReport = (currentModule && currentModule.suiteReport) || globalSuiteReport;
+    }
+  }
+}
+
 let focused = false; // indicates that the "only" filter was used
 
 export function module (name, options, scope) {
@@ -165,15 +200,9 @@ module.only = function (...args) {
   if (!focused) {
     // Upon the first module.only() call,
     // delete any and all previously registered modules and tests.
-    //
-    // TODO: This does not clear SuiteReport, which means the empty modules are
-    // left behind and wrongly reported as "skipped" (skipped == total), and
-    // deleted tests wrongly count toward runEnd.testCounts as passing test
-    // (this.getFailedAssertions().length === 0).
-    // This is why /test/cli/fixtures/only-test-only-module-mix.tap.txt reports
-    // 1..3 instead of 1..1.
     config.modules.length = 0;
     config.queue.length = 0;
+    clearSuiteReports(config.currentModule);
 
     // Ignore any tests declared after this block within the same
     // module parent. https://github.com/qunitjs/qunit/issues/1645

src/core/test.js

@@ -5,6 +5,7 @@ import dump from './dump.js';
 import { emit } from './events.js';
 import { globalThis, setTimeout, clearTimeout, StringMap } from './globals.js';
 import Logger from './logger.js';
+import { clearSuiteReports } from './module.js';
 import Promise from './promise.js';
 import TestReport from './reports/test.js';
 import { extractStacktrace, sourceFromStacktrace } from './stacktrace.js';
@@ -912,6 +913,8 @@ function addOnlyTest (settings) {
   }
   if (!focused) {
     config.queue.length = 0;
+    clearSuiteReports(config.currentModule);
+
     focused = true;
   }
 

test/cli/fixtures/only-module-flat.tap.txt

@@ -12,8 +12,8 @@ not ok 1 module B > test B # TODO
   ...
 ok 2 module B > test C # SKIP
 ok 3 module B > test D
-1..4
-# pass 2
+1..3
+# pass 1
 # skip 1
 # todo 1
 # fail 0

test/cli/fixtures/only-module-then-test.tap.txt

@@ -3,8 +3,8 @@
 
 TAP version 13
 ok 1 module A > module B > test B
-1..2
-# pass 2
+1..1
+# pass 1
 # skip 0
 # todo 0
 # fail 0

test/cli/fixtures/only-module.tap.txt

@@ -16,8 +16,8 @@ ok 3 module B > Only this module should run > normal test
 ok 4 module D > test D
 ok 5 module E > module F > test F
 ok 6 module E > test E
-1..8
-# pass 6
+1..6
+# pass 4
 # skip 1
 # todo 1
 # fail 0

test/cli/fixtures/only-test-only-module-mix.tap.txt

@@ -9,8 +9,8 @@ not ok 1 global failure
     Error: No tests were run.
   ...
 Bail out! Error: No tests were run.
-1..3
-# pass 2
+1..1
+# pass 0
 # skip 0
 # todo 0
 # fail 1

test/cli/fixtures/only-test.tap.txt

@@ -4,8 +4,8 @@
 TAP version 13
 ok 1 run this test
 ok 2 all tests with only run
-1..3
-# pass 3
+1..2
+# pass 2
 # skip 0
 # todo 0
 # fail 0