feat: update timed out test counting logic

Since timed out tests can now be treated as either successes or
failures, update the logic for handling them. Note that before
`self.failed` and `self.timed_out` were separate, but now `self.failed`
includes `self.failed_timed_out`.

Author: eduardorittner

nextest-runner/src/reporter/events.rs

@@ -553,7 +553,7 @@ pub struct RunStats {
     /// The number of tests that passed on retry.
     pub flaky: usize,
 
-    /// The number of tests that failed.
+    /// The number of tests that failed. Includes `leaky_failed` and `failed_timed_out`.
     pub failed: usize,
 
     /// The number of failed tests that were slow.
@@ -592,7 +592,7 @@ impl RunStats {
 
     /// Returns count of tests that did not pass.
     pub fn failed_count(&self) -> usize {
-        self.failed + self.exec_failed + self.failed_timed_out
+        self.failed + self.exec_failed
     }
 
     /// Summarizes the stats as an enum at the end of a test run.
@@ -720,7 +720,21 @@ impl RunStats {
                     self.failed_slow += 1;
                 }
             }
-            ExecutionResult::Timeout { .. } => self.failed_timed_out += 1,
+            ExecutionResult::Timeout {
+                result: SlowTimeoutResult::Pass,
+            } => {
+                self.passed += 1;
+                self.passed_timed_out += 1;
+                if run_statuses.len() > 1 {
+                    self.flaky += 1;
+                }
+            }
+            ExecutionResult::Timeout {
+                result: SlowTimeoutResult::Fail,
+            } => {
+                self.failed += 1;
+                self.failed_timed_out += 1;
+            }
             ExecutionResult::ExecFail => self.exec_failed += 1,
         }
     }

nextest-runner/tests/integration/basic.rs

@@ -749,3 +749,95 @@ fn test_termination() -> Result<()> {
 
     Ok(())
 }
+
+#[test]
+fn test_timeout_result() -> Result<()> {
+    test_init();
+
+    let pcx = ParseContext::new(&PACKAGE_GRAPH);
+    let expr = Filterset::parse(
+        "test(/^test_slow_timeout/)".to_owned(),
+        &pcx,
+        FiltersetKind::Test,
+    )
+    .unwrap();
+    let test_filter = TestFilterBuilder::new(
+        RunIgnored::Only,
+        None,
+        TestFilterPatterns::default(),
+        vec![expr],
+    )
+    .unwrap();
+
+    let test_list = FIXTURE_TARGETS.make_test_list(
+        "with-timeout-success",
+        &test_filter,
+        &TargetRunner::empty(),
+    )?;
+    let config = load_config();
+    let profile = config
+        .profile("with-timeout-success")
+        .expect("with-timeout-success config is valid");
+    let build_platforms = BuildPlatforms::new_with_no_target().unwrap();
+    let profile = profile.apply_build_platforms(&build_platforms);
+
+    let runner = TestRunnerBuilder::default()
+        .build(
+            &test_list,
+            &profile,
+            vec![],
+            SignalHandlerKind::Noop,
+            InputHandlerKind::Noop,
+            DoubleSpawnInfo::disabled(),
+            TargetRunner::empty(),
+        )
+        .unwrap();
+
+    let (instance_statuses, run_stats) = execute_collect(runner);
+    assert_eq!(
+        run_stats.passed_timed_out, 3,
+        "3 tests timed out and passed"
+    );
+    for test_name in [
+        "test_slow_timeout",
+        "test_slow_timeout_2",
+        "test_slow_timeout_subprocess",
+    ] {
+        let (_, instance_value) = instance_statuses
+            .iter()
+            .find(|&(&(_, name), _)| name == test_name)
+            .unwrap_or_else(|| panic!("{test_name} should be present"));
+        let valid = match &instance_value.status {
+            InstanceStatus::Skipped(_) => panic!("{test_name} should have been run"),
+            InstanceStatus::Finished(run_statuses) => {
+                // This test should not have been retried since retries aren't configured.
+                assert_eq!(
+                    run_statuses.len(),
+                    1,
+                    "{test_name} should have been run exactly once",
+                );
+                let run_status = run_statuses.last_status();
+                // The test should have taken less than 5 seconds (most relevant for
+                // test_slow_timeout_subprocess -- without job objects it gets stuck on Windows
+                // until the subprocess exits.)
+                assert!(
+                    run_status.time_taken < Duration::from_secs(5),
+                    "{test_name} should have taken less than 5 seconds, actually took {:?}",
+                    run_status.time_taken
+                );
+                run_status.result
+                    == ExecutionResult::Timeout {
+                        result: SlowTimeoutResult::Pass,
+                    }
+            }
+        };
+        if !valid {
+            panic!(
+                "for test_slow_timeout, mismatch in status: expected timeout, actual {:?}",
+                instance_value.status
+            );
+        }
+    }
+
+    Ok(())
+}