[vendored-sync] dotnet-test-terminal-reporter: drift detected (#7)

[github-actions]

Automated drift report for vendored file src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporter.cs.

Source

• Upstream repo: microsoft/testfx
• Upstream path: src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Summary.cs
• Tracked ref: main
• Scope: reporter partial

Drift

• Baseline blob SHA: 9987fc30e3b7f0ab2948b10e3ef4572d83d87aa4
• Current blob SHA: 70615451a89acbc84f4ab235bb0957ee4061f6a0
• Baseline ref SHA : dba319b212caae2a325800de8a5570ebe787b06d
• Current ref SHA : f935d2d3f9ce1ab831a361041b0ceaf92ff73363

Links

• File history
• Baseline blob
• Current blob
• Upstream ref compare (whole-repo diff, may be noisy)

Upstream-only diff

--- src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Summary.cs@9987fc3
+++ src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Summary.cs@7061545
@@ -1,7 +1,5 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
-
-using Microsoft.Testing.Platform.Helpers;
 
 namespace Microsoft.Testing.Platform.OutputDevice.Terminal;
 
@@ -42,6 +40,17 @@
 
 List<TestProgressState> assemblies = [.. _assemblies.Values.OrderBy(static a => a.Id)];
 
+ // Retry attempt (second or later): skip straight to the sections the orchestrator does not restate. The
+ // verdict and counts below would describe the filtered subset this attempt re-ran rather than the run, so
+ // the reconciled retry summary owns them instead.
+ if (!_options.ShowRunSummary)
+ {
+ AppendSlowestTests(terminal, assemblies);
+ AppendHandshakeFailureRecap(terminal);
+ AppendErroredAssemblyRecap(terminal);
+ return;
+ }
+
 // Single-pass aggregation: compute all summary counters in one foreach instead of
 // 7 separate LINQ calls (Sum×5, Any×1, Count×1), saving 6 extra O(N) passes and
 // 7 LINQ enumerator allocations per test run.
@@ -49,7 +58,9 @@
 int totalFailedTests = 0;
 int totalSkippedTests = 0;
 int totalPassedTests = 0;
- int totalRetried = 0;
+ int totalRetriedTests = 0;
+ int totalRetriedExecutions = 0;
+ int totalFlakyTests = 0;
 bool anyAssemblyFailed = false;
 int failedAssembliesWithoutFailedTests = 0;
 
@@ -59,7 +70,9 @@
 totalFailedTests += assembly.FailedTests;
 totalSkippedTests += assembly.SkippedTests;
 totalPassedTests += assembly.PassedTests;
- totalRetried += assembly.RetriedFailedTests;
+ totalRetriedTests += assembly.RetriedTests;
+ totalRetriedExecutions += assembly.RetriedExecutions;
+ totalFlakyTests += assembly.FlakyTests;
 if (!assembly.Success)
 {
 anyAssemblyFailed = true;
@@ -119,7 +132,6 @@
 int failed = totalFailedTests;
 int passed = totalPassedTests;
 int skipped = totalSkippedTests;
- int retried = totalRetried;
 
 // Orchestrator-only: count assemblies that ended unsuccessfully without a failed test (crash / non-zero exit)
 // plus handshake failures. These are surfaced as an "error: N" line so they aren't hidden behind a zero
@@ -147,19 +159,8 @@
 }
 
 terminal.ResetColor();
- terminal.Append(totalText);
-
- // Orchestrator-only: when failed tests were retried, append "(+N retried)" after the total so the headline
- // count (which reflects the final attempt) is reconciled with the extra retried executions. retried is 0 for
- // the in-process host, so the total line stays byte-identical there.
- if (retried > 0)
- {
- terminal.SetColor(TerminalColor.DarkGray);
- terminal.Append($" (+{retried} {TerminalResources.Retried})");
- terminal.ResetColor();
- }
-
- terminal.AppendLine();
+ terminal.AppendLine(totalText);
+
 if (colorizeFailed)
 {
 terminal.SetColor(TerminalColor.DarkRed);
@@ -195,10 +196,16 @@
 {
 terminal.ResetColor();
 }
+
+ AppendRetrySummaryLines(terminal, totalFlakyTests, totalRetriedTests, totalRetriedExecutions);
 
 terminal.Append(durationText);
 AppendLongDuration(terminal, runDuration, wrapInParentheses: false, colorize: false);
 terminal.AppendLine();
+
+ // Optional "Flaky tests" section (on by default, suppressed by --show-flaky-tests off). No-op when nothing
+ // was retried, so the summary stays byte-identical for a run without retries.
+ AppendFlakyTests(terminal, assemblies);
 
 // Optional "Slowest tests" section (opt-in via --show-slowest-tests). Additive: no-op when the feature is
 // off, so the summary stays byte-identical for the default run.
@@ -214,197 +221,28 @@
 }
 
 /// <summary>
- /// Orchestrator overload (<c>dotnet test</c>): the multi-process orchestrator also knows each discovered test's
- /// uid, file path and line number. The shared discovery summary currently lists display names only, so those are
- /// accepted for signature parity. When <paramref name="displayName"/> is missing the <paramref name="uid"/> is used
- /// as the listed name; when neither is available the test is still counted (so the discovery total stays correct)
- /// but no blank entry is added to the summary.
+ /// Appends the retry accounting lines that sit between the skipped count and the duration:
+ /// <c>flaky: N</c> (tests that failed at least once but eventually passed) and
+ /// <c>retried: N tests, M extra runs</c>. Both are omitted entirely when nothing was retried, so a run without
+ /// retries keeps its historical summary byte-for-byte.
 /// </summary>
- internal void TestDiscovered(string executionId, string? displayName, string? uid, string? filePath, int? lineNumber)
+ private void AppendRetrySummaryLines(ITerminal terminal, int flakyTests, int retriedTests, int retriedExecutions)
 {
- // Prefer the display name, fall back to the uid so the discovered test is still listed by something.
- string? name = displayName ?? uid;
- if (name is not null)
- {
- TestDiscovered(executionId, name);
- return;
- }
-
- // No name available at all: still increment the discovered count so the discovery summary total stays
- // correct (in discovery mode TotalTests is computed from DiscoveredTests), but avoid adding a blank entry.
- if (!_assemblies.TryGetValue(executionId, out TestProgressState? asm))
- {
- throw ApplicationStateGuard.Unreachable();
- }
-
- asm.ReportDiscoveredTest(displayName: null);
- _terminalWithProgress.UpdateWorker(asm.SlotIndex);
- }
-
- internal void TestDiscovered(string executionId, string displayName)
- {
- if (!_assemblies.TryGetValue(executionId, out TestProgressState? asm))
- {
- throw ApplicationStateGuard.Unreachable();
- }
-
- // In discovery mode TotalTests is computed from DiscoveredTests; in execution mode it is computed from the
- // passed/skipped/failed tally as tests complete. So we only need to bump the discovered count here.
- asm.ReportDiscoveredTest(MakeControlCharactersVisible(displayName, true));
-
- _terminalWithProgress.UpdateWorker(asm.SlotIndex);
- }
-
- public void AppendTestDiscoverySummary(ITerminal terminal)
- {
- List<TestProgressState> assemblies = [.. _assemblies.Values.OrderBy(static a => a.Id)];
- terminal.AppendLine();
-
- int totalTests = assemblies.Sum(static a => a.TotalTests);
- bool runFailed = WasCancelled || totalTests < 1;
-
- if (_options.ShowAssembly)
- {
- // Orchestrator (dotnet test): a per-assembly "Discovered N tests in assembly - <link>" header followed by
- // the discovered test names, then a run-level total ("Discovered N tests." / "... in N assemblies.").
- foreach (TestProgressState assembly in assemblies)
- {
- terminal.Append(string.Format(CultureInfo.CurrentCulture, TerminalResources.DiscoveredTestsInAssembly, assembly.DiscoveredTests));
- terminal.Append(" - ");
- AppendAssemblyLinkTargetFrameworkAndArchitecture(terminal, assembly);
- terminal.AppendLine();
- foreach (string displayName in assembly.DiscoveredTestDisplayNames)
- {
- terminal.Append(SingleIndentation);
- terminal.AppendLine(displayName);
- }
-
- terminal.AppendLine();
- }
-
- terminal.SetColor(runFailed ? TerminalColor.DarkRed : TerminalColor.DarkGreen);
- terminal.AppendLine(assemblies.Count <= 1
- ? string.Format(CultureInfo.CurrentCulture, TerminalResources.DiscoveredTestsSummarySingular, totalTests)
- : string.Format(CultureInfo.CurrentCulture, TerminalResources.DiscoveredTestsSummary, totalTests, assemblies.Count));
- terminal.ResetColor();
- terminal.AppendLine();
-
- if (WasCancelled)
- {
- terminal.Append(TerminalResources.Aborted);
- terminal.AppendLine();
- }
-
- return;
- }
-
- // In-process host: the single "Test discovery summary: found N test(s)" format (unchanged shipping output).
- foreach (TestProgressState assembly in assemblies)
- {
- foreach (string displayName in assembly.DiscoveredTestDisplayNames)
- {
- terminal.Append(SingleIndentation);
- terminal.AppendLine(displayName);
- }
- }
-
- terminal.AppendLine();
-
- terminal.SetColor(runFailed ? TerminalColor.DarkRed : TerminalColor.DarkGreen);
- terminal.Append(string.Format(CultureInfo.CurrentCulture, TerminalResources.TestDiscoverySummarySingular, totalTests));
-
- if (assemblies.Count == 1)
+ // "flaky" is the headline value of retrying, so it is reported whenever it is non-zero unless the user
+ // explicitly turned the feature off.
+ if (flakyTests > 0 && _options.ShowFlakyTests)
+ {
+ terminal.SetColor(TerminalColor.DarkYellow);
+ terminal.AppendLine($"{SingleIndentation}{string.Format(CultureInfo.CurrentCulture, TerminalResources.FlakyLowercase, flakyTests)}");
+ terminal.ResetColor();
+ }
+
+ if (retriedTests > 0)
 {
 terminal.SetColor(TerminalColor.DarkGray);
- terminal.Append(" - ");
- terminal.ResetColor();
- AppendAssemblyLinkTargetFrameworkAndArchitecture(terminal, assemblies[0]);
- }
-
- terminal.ResetColor();
- terminal.AppendLine();
-
- if (WasCancelled)
- {
- terminal.Append(TerminalResources.Aborted);
- terminal.AppendLine();
- }
-
- string durationText = $"{SingleIndentation}{TerminalResources.DurationLowercase}: ";
- TimeSpan runDuration = _testExecutionStartTime != null && _testExecutionEndTime != null ? (_testExecutionEndTime - _testExecutionStartTime).Value : TimeSpan.Zero;
- terminal.Append(durationText);
- AppendLongDuration(terminal, runDuration, wrapInParentheses: false, colorize: false);
- terminal.AppendLine();
- }
-
- /// <summary>
- /// Appends the opt-in "Slowest tests" section, ranking the longest-running tests by their reported execution
- /// duration. For a single assembly a flat list is rendered; for the multi-assembly orchestrator each assembly
- /// gets its own sub-list so the ranking stays scoped per assembly. No-op when the feature is off or when no
- /// timed tests were recorded.
- /// </summary>
- private void AppendSlowestTests(ITerminal terminal, List<TestProgressState> assemblies)
- {
- int count = _options.SlowestTestsCount;
- if (count <= 0)
- {
- return;
- }
-
- if (_options.ShowAssembly && assemblies.Count > 1)
- {
- bool headerWritten = false;
- foreach (TestProgressState assembly in assemblies)
- {
- IReadOnlyList<(string DisplayName, TimeSpan Duration)> slowest = assembly.GetSlowestTests(count);
- if (slowest.Count == 0)
- {
- continue;
- }
-
- if (!headerWritten)
- {
- terminal.AppendLine();
- terminal.AppendLine(TerminalResources.SlowestTests);
- headerWritten = true;
- }
-
- terminal.Append(SingleIndentation);
- AppendAssemblyLinkTargetFrameworkAndArchitecture(terminal, assembly);
- terminal.AppendLine();
- foreach ((string displayName, TimeSpan duration) in slowest)
- {
- terminal.Append(DoubleIndentation);
- AppendSlowestTestLine(terminal, displayName, duration);
- }
- }
-
- return;
- }
-
- // Single assembly (in-process host, or the orchestrator with a single assembly): a flat list.
- IReadOnlyList<(string DisplayName, TimeSpan Duration)> tests = assemblies.Count == 1
- ? assemblies[0].GetSlowestTests(count)
- : [];
- if (tests.Count == 0)
- {
- return;
- }
-
- terminal.AppendLine();
- terminal.AppendLine(TerminalResources.SlowestTests);
- foreach ((string displayName, TimeSpan duration) in tests)
- {
- terminal.Append(SingleIndentation);
- AppendSlowestTestLine(terminal, displayName, duration);
- }

... (diff truncated; 14 more lines)

Local adaptation notes

The Microsoft.Testing.Platform terminal UI reporter, hard-forked into 'dotnet test'. testfx has since split TerminalTestReporter into partial files; the SDK keeps a single TerminalTestReporter.cs. Each upstream partial is tracked so any upstream reporter change pings the SDK to reconcile. This is a hard fork: local content intentionally diverges; the signal is 'upstream changed', not 'files differ'.

How to reconcile

1. Review the upstream changes and decide whether they should be ported.
2. Port the relevant changes to src/Cli/dotnet/Commands/Test/MTP/Terminal/TerminalTestReporter.cs.
3. Update eng/vendored-files.json entry dotnet-test-terminal-reporter (source index 7):
   • bump baseline_ref_sha to the new upstream ref SHA,
   • bump baseline_blob_sha to the new upstream blob SHA.
4. Close this issue once the reconciliation PR is merged.

If no port is needed (e.g. whitespace-only upstream change), still bump the baseline SHAs to silence future runs.