diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs index ea2fcba535..d695be8923 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs @@ -455,6 +455,10 @@ public async Task DisplayAfterHotReloadSessionEndAsync(CancellationToken cancell public async Task DisplayAfterSessionEndRunAsync(CancellationToken cancellationToken) { + // Under --server (e.g. `dotnet test` with `--server dotnettestcli`) the terminal device stays + // silent: discovered tests are streamed to the SDK through the dotnet-test pipe (see + // DotnetTestDataConsumer), and the SDK owns rendering — including building the --list-tests json + // document by combining the discovered tests from every test app into a single output. if (_isServerMode) { return; @@ -588,6 +592,10 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo { RoslynDebug.Assert(_terminalTestReporter is not null); cancellationToken.ThrowIfCancellationRequested(); + + // Under --server (e.g. `dotnet test` with `--server dotnettestcli`) the terminal device does not + // buffer or render anything: data flows to the SDK through the dotnet-test pipe instead, and the + // SDK is responsible for producing the output (including the --list-tests json document). if (_isServerMode) { return Task.CompletedTask; diff --git a/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/IPC/DotnetTestDataConsumer.cs b/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/IPC/DotnetTestDataConsumer.cs index 87acb5e7a7..8046483788 100644 --- a/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/IPC/DotnetTestDataConsumer.cs +++ b/src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/IPC/DotnetTestDataConsumer.cs @@ -54,6 +54,11 @@ public async Task ConsumeAsync(IDataProducer dataProducer, IData value, Cancella switch (testNodeDetails.State) { case TestStates.Discovered: + // Only stream the full discovery details (file location, method identifier, + // traits) when the consumer asked for them. We reuse the existing IsIDE flag + // for that — despite the name, it is the handshake signal a consumer sets when + // it wants the complete discovery object (e.g. an IDE, or the SDK when running + // `dotnet test --list-tests json`). Plain runs keep the payload minimal. TestFileLocationProperty? testFileLocationProperty = null; TestMethodIdentifierProperty? testMethodIdentifierProperty = null; TestMetadataProperty[] traits = []; diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeListTestsJsonTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeListTestsJsonTests.cs new file mode 100644 index 0000000000..0740e172dd --- /dev/null +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeListTestsJsonTests.cs @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests.DotnetTestPipe; + +/// +/// Validates the agreed design for --list-tests json under --server dotnettestcli +/// (the mode the .NET SDK always uses for dotnet test): the test app does not render +/// the JSON document itself. Instead it streams the discovered tests to the SDK over the dotnet-test +/// pipe, and the SDK is responsible for producing the JSON (combining the tests from every test app +/// into a single document). +/// +/// Built on the black-box harness introduced in +/// microsoft/testfx#9153, so the +/// behavior is exercised end-to-end against the real wire protocol. +/// +/// +[TestClass] +public class DotnetTestPipeListTestsJsonTests : AcceptanceTestBase +{ + private const string AssetName = "DotnetTestPipeListTestsJson"; + + public TestContext TestContext { get; set; } = null!; + + [TestMethod] + public async Task DotnetTestPipe_ListTestsJson_StreamsDiscoveredTestsOverPipeAndKeepsStdoutClean() + { + var testHost = TestInfrastructure.TestHost.LocateFrom( + AssetFixture.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent); + + // The SDK requests the full discovery object by advertising IsIDE in the handshake (the same + // signal it sets when running `dotnet test --list-tests json`), so the host streams the + // complete discovery details (file location, method identifier, traits) we assert below. + FakeDotnetTestSdkResult result = await FakeDotnetTestSdk.RunAsync( + testHost, extraArguments: "--list-tests json", isIde: true, cancellationToken: TestContext.CancellationToken); + + result.TestHostResult.AssertExitCodeIs(ExitCode.Success); + + // Under --server the test app must stay silent on stdout/stderr: the SDK owns rendering, + // including building the --list-tests json document. In particular the app must NOT print a + // JSON document itself, otherwise the SDK would receive duplicated/raw output. + Assert.AreEqual( + string.Empty, + result.TestHostResult.StandardOutput.Trim(), + $"Expected no stdout under --server dotnettestcli (the SDK renders the output).{Environment.NewLine}" + + $"Captured stdout:{Environment.NewLine}{result.TestHostResult.StandardOutput}"); + + Assert.AreEqual( + string.Empty, + result.TestHostResult.StandardError.Trim(), + $"Expected no stderr noise for a successful discovery.{Environment.NewLine}" + + $"Captured stderr:{Environment.NewLine}{result.TestHostResult.StandardError}"); + + // The discovered tests must instead be streamed to the SDK as DiscoveredTestMessages frames, + // carrying the full discovery object (file location, method identifier, traits) the SDK needs + // to build the --list-tests json document. + var discoveredTests = new Dictionary(StringComparer.Ordinal); + bool sawDiscoveredFrame = false; + foreach (RawMessage frame in result.ReceivedMessages) + { + if (frame.SerializerId != DotnetTestPipeProtocol.SerializerIds.DiscoveredTestMessages) + { + continue; + } + + sawDiscoveredFrame = true; + foreach (DiscoveredTest test in DotnetTestPipeProtocol.DecodeDiscoveredTests(frame.Body)) + { + Assert.IsNotNull(test.DisplayName, "Every discovered test must carry a display name."); + discoveredTests[test.DisplayName] = test; + } + } + + Assert.IsTrue(sawDiscoveredFrame, "Expected at least one DiscoveredTestMessages frame over the dotnet-test pipe."); + Assert.Contains("Test1", discoveredTests.Keys); + Assert.Contains("Test2", discoveredTests.Keys); + + // Regression guard: the full discovery details must keep flowing over the pipe so the SDK can + // build the --list-tests json document. If any of these fields stop being streamed the test + // fails, even though the display names alone would still arrive. + DiscoveredTest test1 = discoveredTests["Test1"]; + Assert.AreEqual("MyTests.cs", test1.FilePath); + Assert.AreEqual("MyNamespace", test1.Namespace); + Assert.AreEqual("MyTestClass", test1.TypeName); + Assert.AreEqual("Test1", test1.MethodName); + Assert.AreEqual("Smoke", test1.Traits["Category"]); + + DiscoveredTest test2 = discoveredTests["Test2"]; + Assert.AreEqual("MyTests.cs", test2.FilePath); + Assert.AreEqual("Test2", test2.MethodName); + Assert.AreEqual("Integration", test2.Traits["Category"]); + } + + public sealed class TestAssetFixture() : TestAssetFixtureBase() + { + private const string AssetCode = """ +#file DotnetTestPipeListTestsJson.csproj + + + $TargetFrameworks$ + enable + enable + Exe + true + preview + + + + + + + +#file Program.cs +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.TestFramework; + +public class Program +{ + public static async Task Main(string[] args) + { + ITestApplicationBuilder builder = await TestApplication.CreateBuilderAsync(args); + builder.RegisterTestFramework(_ => new TestFrameworkCapabilities(), (_, __) => new DiscoveringTestFramework()); + using ITestApplication app = await builder.BuildAsync(); + return await app.RunAsync(); + } +} + +public class DiscoveringTestFramework : ITestFramework, IDataProducer +{ + public string Uid => nameof(DiscoveringTestFramework); + public string Version => "2.0.0"; + public string DisplayName => nameof(DiscoveringTestFramework); + public string Description => nameof(DiscoveringTestFramework); + public Type[] DataTypesProduced => new[] { typeof(TestNodeUpdateMessage) }; + public Task IsEnabledAsync() => Task.FromResult(true); + public Task CreateTestSessionAsync(CreateTestSessionContext context) + => Task.FromResult(new CreateTestSessionResult() { IsSuccess = true }); + public Task CloseTestSessionAsync(CloseTestSessionContext context) + => Task.FromResult(new CloseTestSessionResult() { IsSuccess = true }); + + public async Task ExecuteRequestAsync(ExecuteRequestContext context) + { + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, + new TestNode() + { + Uid = "0", + DisplayName = "Test1", + Properties = new( + DiscoveredTestNodeStateProperty.CachedInstance, + new TestFileLocationProperty("MyTests.cs", new LinePositionSpan(new LinePosition(10, 1), new LinePosition(10, 5))), + new TestMethodIdentifierProperty("MyTestAssembly", "MyNamespace", "MyTestClass", "Test1", 0, [], "System.Void"), + new TestMetadataProperty("Category", "Smoke")), + })); + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, + new TestNode() + { + Uid = "1", + DisplayName = "Test2", + Properties = new( + DiscoveredTestNodeStateProperty.CachedInstance, + new TestFileLocationProperty("MyTests.cs", new LinePositionSpan(new LinePosition(20, 1), new LinePosition(20, 5))), + new TestMethodIdentifierProperty("MyTestAssembly", "MyNamespace", "MyTestClass", "Test2", 0, [], "System.Void"), + new TestMetadataProperty("Category", "Integration")), + })); + + context.Complete(); + } +} +"""; + + public string TargetAssetPath => GetAssetPath(AssetName); + + public override (string ID, string Name, string Code) GetAssetsToGenerate() => (AssetName, AssetName, + AssetCode + .PatchTargetFrameworks(TargetFrameworks.NetCurrent) + .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion)); + } +} diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeProtocol.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeProtocol.cs index f6ca6151e0..b120d5c9e1 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeProtocol.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeProtocol.cs @@ -220,6 +220,170 @@ public static (byte? SessionType, string? SessionUid, string? ExecutionId) Decod return (sessionType, sessionUid, executionId); } + /// + /// Decodes the tests carried by a frame, + /// including the full discovery details (file path, line number, namespace, type/method name, + /// parameter types and traits). Mirrors the wire layout produced by + /// DiscoveredTestMessagesSerializer: the body is a field-tagged object whose + /// DiscoveredTestMessageList field (id 3) holds a length-prefixed array of field-tagged + /// test messages. Unknown fields are skipped via their declared size, exactly like the product + /// deserializer. + /// + /// Decoding the whole object (not just the display name) lets the acceptance test catch + /// regressions where the SDK-facing metadata required to build the --list-tests json + /// document stops flowing over the pipe. + /// + /// + public static IReadOnlyList DecodeDiscoveredTests(byte[] body) + { + const ushort discoveredTestMessageListFieldId = 3; + const ushort uidFieldId = 1; + const ushort displayNameFieldId = 2; + const ushort filePathFieldId = 3; + const ushort lineNumberFieldId = 4; + const ushort namespaceFieldId = 5; + const ushort typeNameFieldId = 6; + const ushort methodNameFieldId = 7; + const ushort traitsFieldId = 8; + const ushort parameterTypeFullNamesFieldId = 9; + + List discoveredTests = []; + using MemoryStream stream = new(body, writable: false); + + ushort fieldCount = ReadUShort(stream); + for (int i = 0; i < fieldCount; i++) + { + ushort fieldId = ReadUShort(stream); + int fieldSize = ReadInt(stream); + + if (fieldId != discoveredTestMessageListFieldId) + { + // ExecutionId / InstanceId (or any future field): skip the whole payload. + stream.Seek(fieldSize, SeekOrigin.Current); + continue; + } + + int messageCount = ReadInt(stream); + for (int m = 0; m < messageCount; m++) + { + string? uid = null; + string? displayName = null; + string? filePath = null; + int? lineNumber = null; + string? @namespace = null; + string? typeName = null; + string? methodName = null; + string[] parameterTypeFullNames = []; + Dictionary traits = []; + + ushort messageFieldCount = ReadUShort(stream); + for (int f = 0; f < messageFieldCount; f++) + { + ushort messageFieldId = ReadUShort(stream); + int messageFieldSize = ReadInt(stream); + switch (messageFieldId) + { + case uidFieldId: + uid = ReadFixedSizeString(stream, messageFieldSize); + break; + case displayNameFieldId: + displayName = ReadFixedSizeString(stream, messageFieldSize); + break; + case filePathFieldId: + filePath = ReadFixedSizeString(stream, messageFieldSize); + break; + case lineNumberFieldId: + lineNumber = ReadInt(stream); + break; + case namespaceFieldId: + @namespace = ReadFixedSizeString(stream, messageFieldSize); + break; + case typeNameFieldId: + typeName = ReadFixedSizeString(stream, messageFieldSize); + break; + case methodNameFieldId: + methodName = ReadFixedSizeString(stream, messageFieldSize); + break; + case traitsFieldId: + foreach (KeyValuePair trait in ReadTraits(stream)) + { + traits[trait.Key] = trait.Value; + } + + break; + case parameterTypeFullNamesFieldId: + parameterTypeFullNames = ReadParameterTypeFullNames(stream); + break; + default: + stream.Seek(messageFieldSize, SeekOrigin.Current); + break; + } + } + + discoveredTests.Add(new DiscoveredTest( + uid, displayName, filePath, lineNumber, @namespace, typeName, methodName, parameterTypeFullNames, traits)); + } + } + + return discoveredTests; + } + + /// + /// Reads the Traits payload (id 8) of a discovered test message: a length-prefixed array + /// of field-tagged key/value pairs (Key id 1, Value id 2). + /// + private static IReadOnlyList> ReadTraits(Stream stream) + { + const ushort keyFieldId = 1; + const ushort valueFieldId = 2; + + int length = ReadInt(stream); + List> traits = []; + for (int i = 0; i < length; i++) + { + string? key = null; + string? value = null; + ushort fieldCount = ReadUShort(stream); + for (int f = 0; f < fieldCount; f++) + { + ushort fieldId = ReadUShort(stream); + int fieldSize = ReadInt(stream); + switch (fieldId) + { + case keyFieldId: + key = ReadFixedSizeString(stream, fieldSize); + break; + case valueFieldId: + value = ReadFixedSizeString(stream, fieldSize); + break; + default: + stream.Seek(fieldSize, SeekOrigin.Current); + break; + } + } + + traits.Add(new KeyValuePair(key ?? string.Empty, value ?? string.Empty)); + } + + return traits; + } + + /// + /// Reads the ParameterTypeFullNames payload (id 9) of a discovered test message: a + /// length-prefixed array of length-prefixed UTF-8 strings. + /// + private static string[] ReadParameterTypeFullNames(Stream stream) + { + int length = ReadInt(stream); + string[] parameterTypeFullNames = new string[length]; + for (int i = 0; i < length; i++) + { + parameterTypeFullNames[i] = ReadLengthPrefixedString(stream); + } + + return parameterTypeFullNames; + } + private static async Task TryReadExactlyAsync(Stream stream, Memory buffer, CancellationToken cancellationToken) { int totalRead = 0; @@ -299,3 +463,18 @@ private static void WriteLengthPrefixedString(Stream stream, string value) /// A raw decoded pipe frame (serializer id + body bytes, no further decoding). internal sealed record RawMessage(int SerializerId, byte[] Body); + +/// +/// A fully decoded discovered test message: the complete discovery object the SDK needs to build the +/// --list-tests json document (display name plus file/method location and traits). +/// +internal sealed record DiscoveredTest( + string? Uid, + string? DisplayName, + string? FilePath, + int? LineNumber, + string? Namespace, + string? TypeName, + string? MethodName, + string[] ParameterTypeFullNames, + IReadOnlyDictionary Traits);