From 89f761678c24582fdb76bacd909f5fd498f92024 Mon Sep 17 00:00:00 2001 From: Evangelink Date: Mon, 15 Jun 2026 15:37:08 +0200 Subject: [PATCH 1/2] Add black-box baseline tests for dotnet-test pipe protocol (prep for dotnet/sdk#51615) This PR introduces a hand-rolled `FakeDotnetTestSdk` harness plus three baseline acceptance tests that pin today's wire-format behavior of `Microsoft.Testing.Platform` running under `--server dotnettestcli`. The harness is intentionally black-box: it uses only `System.IO.Pipes.NamedPipeServerStream` from the BCL and re-implements the dotnet-test wire format (`int32 size | int32 serializerId | body`, with leading `ushort fieldCount` in handshake/event bodies) directly. This was necessary because the production IPC types under `src/Platform/Microsoft.Testing.Platform/IPC/*` are decorated with `[Microsoft.CodeAnalysis.Embedded]` and so cannot be referenced across assembly boundaries even with `InternalsVisibleTo`. As a side benefit the harness now also acts as a contract test for the wire format. The three tests: 1. `DotnetTestPipe_TestAppAdvertisesAndNegotiatesProtocolV100_Today` pins protocol negotiation: the test app advertises `1.0.0`, and a fake SDK that advertises `1.0.0` negotiates `1.0.0`. 2. `DotnetTestPipe_EmitsTestSessionStartAndEnd` pins that a single no-op session produces exactly one `TestSessionEvent` with `SessionType=TestSessionStart` and one with `TestSessionEnd`. 3. `DotnetTestPipe_ChildEmitsNoStdoutOrStderrForPassingRun_Baseline` pins that today the test app emits nothing to stdout/stderr under pipe mode for a passing no-op run (banner is already silenced via the existing `_isServerMode` check in `TerminalOutputDevice`). These baselines lock down the current behavior so subsequent work on dotnet/sdk#51615 (live stdout/stderr forwarding, `IOutputDevice` routing through the pipe, protocol bump to 1.0.1) can be reviewed against an explicit before/after. No product code changes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../DotnetTestPipeBaselineTests.cs | 177 +++++++++++ .../DotnetTestPipe/DotnetTestPipeProtocol.cs | 293 ++++++++++++++++++ .../DotnetTestPipe/FakeDotnetTestSdk.cs | 165 ++++++++++ .../DotnetTestPipe/FakeDotnetTestSdkResult.cs | 47 +++ 4 files changed, 682 insertions(+) create mode 100644 test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeBaselineTests.cs create mode 100644 test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeProtocol.cs create mode 100644 test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/FakeDotnetTestSdk.cs create mode 100644 test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/FakeDotnetTestSdkResult.cs diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeBaselineTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeBaselineTests.cs new file mode 100644 index 0000000000..322aa2cb9d --- /dev/null +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeBaselineTests.cs @@ -0,0 +1,177 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Text.RegularExpressions; + +namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests.DotnetTestPipe; + +/// +/// Baseline tests for the --server dotnettestcli pipe protocol that lock down today's +/// behavior so subsequent phases (quiet output device under pipe mode, protocol 1.0.1, live log +/// and stream forwarding) can be reviewed as targeted diffs against these assertions. +/// +/// Tracks dotnet/sdk#51615 and the +/// related microsoft/testfx#7161. +/// +/// +[TestClass] +public class DotnetTestPipeBaselineTests : AcceptanceTestBase +{ + private const string AssetName = "DotnetTestPipeBaselineTest"; + + private static readonly Regex BannerRegex = new(@"Microsoft\.Testing\.Platform v.+ \[.+\]", RegexOptions.Compiled); + + public TestContext TestContext { get; set; } = null!; + + [TestMethod] + public async Task DotnetTestPipe_TestAppAdvertisesAndNegotiatesProtocolV100_Today() + { + var testHost = TestInfrastructure.TestHost.LocateFrom( + AssetFixture.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent); + + FakeDotnetTestSdkResult result = await FakeDotnetTestSdk.RunAsync( + testHost, cancellationToken: TestContext.CancellationToken); + + Assert.IsNotNull(result.ReceivedHandshake, "Test app never sent a handshake message."); + Assert.IsNotNull(result.SentHandshakeReply); + + Assert.IsTrue( + result.ReceivedHandshake.TryGetValue(DotnetTestPipeProtocol.HandshakeProperties.SupportedProtocolVersions, out string? appVersions), + "Handshake from test app is missing SupportedProtocolVersions."); + Assert.AreEqual( + FakeDotnetTestSdk.DefaultSupportedProtocolVersions, + appVersions, + "BASELINE: today the test app advertises only protocol 1.0.0. When this assertion " + + "starts failing, it means the testfx side has been bumped to 1.0.1+ and the version " + + "negotiation tests should take over (Phase 2 of dotnet/sdk#51615)."); + + Assert.AreEqual( + FakeDotnetTestSdk.DefaultSupportedProtocolVersions, + result.NegotiatedProtocolVersion, + "Fake SDK should have selected '1.0.0' from the test app's advertised list."); + } + + [TestMethod] + public async Task DotnetTestPipe_EmitsTestSessionStartAndEnd() + { + var testHost = TestInfrastructure.TestHost.LocateFrom( + AssetFixture.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent); + + FakeDotnetTestSdkResult result = await FakeDotnetTestSdk.RunAsync( + testHost, cancellationToken: TestContext.CancellationToken); + + byte[] sessionEventTypes = + [ + .. result.MessagesWithSerializerId(DotnetTestPipeProtocol.SerializerIds.TestSessionEvent) + .Select(m => DotnetTestPipeProtocol.DecodeTestSessionEventBody(m.Body).SessionType) + .Where(t => t.HasValue) + .Select(t => t!.Value) + ]; + + Assert.HasCount(2, sessionEventTypes, $"Expected exactly two session events; got [{string.Join(", ", sessionEventTypes)}]."); + Assert.AreEqual(DotnetTestPipeProtocol.SessionEventTypes.TestSessionStart, sessionEventTypes[0]); + Assert.AreEqual(DotnetTestPipeProtocol.SessionEventTypes.TestSessionEnd, sessionEventTypes[1]); + } + + /// + /// BASELINE: under --server dotnettestcli the test app today emits nothing to stdout/stderr + /// for a passing run (the MTP banner is already silenced via _isServerMode in + /// TerminalOutputDevice, and our DummyTestFramework writes nothing of its own). + /// This test pins that contract so future changes can't accidentally start leaking output that + /// would compete with what the SDK is also rendering (the original symptom behind + /// dotnet/sdk#51615 and microsoft/testfx#7161). When this test fails, evaluate whether the + /// added stdout/stderr writes should instead flow through the pipe as structured messages. + /// + [TestMethod] + public async Task DotnetTestPipe_ChildEmitsNoStdoutOrStderrForPassingRun_Baseline() + { + var testHost = TestInfrastructure.TestHost.LocateFrom( + AssetFixture.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent); + + FakeDotnetTestSdkResult result = await FakeDotnetTestSdk.RunAsync( + testHost, cancellationToken: TestContext.CancellationToken); + + Assert.DoesNotMatchRegex( + BannerRegex.ToString(), + result.TestHostResult.StandardOutput, + $"BASELINE: under --server dotnettestcli the test app already silences its banner. " + + $"If this fails, something started re-emitting the banner under pipe mode.{Environment.NewLine}" + + $"Captured stdout:{Environment.NewLine}{result.TestHostResult.StandardOutput}"); + + Assert.AreEqual( + string.Empty, + result.TestHostResult.StandardOutput.Trim(), + $"BASELINE: today the test app emits nothing to stdout under pipe mode for a no-op " + + $"DummyTestFramework run. New text leaking here likely belongs in a pipe message " + + $"(Phase 2 of dotnet/sdk#51615 adds a LogMessage channel for this).{Environment.NewLine}" + + $"Captured stdout:{Environment.NewLine}{result.TestHostResult.StandardOutput}"); + + Assert.AreEqual( + string.Empty, + result.TestHostResult.StandardError.Trim(), + $"BASELINE: today the test app emits nothing to stderr under pipe mode for a passing run.{Environment.NewLine}" + + $"Captured stderr:{Environment.NewLine}{result.TestHostResult.StandardError}"); + } + + public sealed class TestAssetFixture() : TestAssetFixtureBase() + { + private const string AssetCode = """ +#file DotnetTestPipeBaselineTest.csproj + + + $TargetFrameworks$ + enable + enable + Exe + true + preview + + + + + + + +#file Program.cs +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Extensions.TestFramework; + +public class Program +{ + public static async Task Main(string[] args) + { + ITestApplicationBuilder builder = await TestApplication.CreateBuilderAsync(args); + builder.RegisterTestFramework(_ => new TestFrameworkCapabilities(), (_, __) => new DummyTestFramework()); + using ITestApplication app = await builder.BuildAsync(); + return await app.RunAsync(); + } +} + +public class DummyTestFramework : ITestFramework +{ + public string Uid => nameof(DummyTestFramework); + public string Version => "2.0.0"; + public string DisplayName => nameof(DummyTestFramework); + public string Description => nameof(DummyTestFramework); + public Task IsEnabledAsync() => Task.FromResult(true); + public Task CreateTestSessionAsync(CreateTestSessionContext context) + => Task.FromResult(new CreateTestSessionResult() { IsSuccess = true }); + public Task CloseTestSessionAsync(CloseTestSessionContext context) + => Task.FromResult(new CloseTestSessionResult() { IsSuccess = true }); + public Task ExecuteRequestAsync(ExecuteRequestContext context) + { + context.Complete(); + return Task.CompletedTask; + } +} +"""; + + public string TargetAssetPath => GetAssetPath(AssetName); + + public override (string ID, string Name, string Code) GetAssetsToGenerate() => (AssetName, AssetName, + AssetCode + .PatchTargetFrameworks(TargetFrameworks.NetCurrent) + .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion)); + } +} diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeProtocol.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeProtocol.cs new file mode 100644 index 0000000000..b161939f54 --- /dev/null +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeProtocol.cs @@ -0,0 +1,293 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Buffers; +using System.IO.Pipes; +using System.Text; + +namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests.DotnetTestPipe; + +/// +/// Black-box reader/writer for the --server dotnettestcli --dotnet-test-pipe wire +/// protocol. Implements the framing and the serializer IDs / field IDs documented in +/// src/Platform/Microsoft.Testing.Platform/ServerMode/DotnetTest/IPC/ObjectFieldIds.cs +/// without referencing any of the testfx internal types (which are [Embedded] and +/// therefore invisible to this assembly). +/// +/// Treating this as a contract reader makes the resulting tests true black-box assertions on the +/// wire format: any silent change to the message shape on the testfx side will be caught here. +/// +/// +internal static class DotnetTestPipeProtocol +{ + /// + /// Frame layout (matches NamedPipeBase): + /// + /// int32 totalPayloadLength // = sizeof(serializerId) + bodyLength + /// int32 serializerId + /// byte[] body + /// + /// + public static class SerializerIds + { + public const int VoidResponse = 0; + public const int TestHostCompletedRequest = 1; + public const int TestHostProcessPIDRequest = 2; + public const int CommandLineOptionMessages = 3; + public const int DiscoveredTestMessages = 5; + public const int TestResultMessages = 6; + public const int FileArtifactMessages = 7; + public const int TestSessionEvent = 8; + public const int HandshakeMessage = 9; + public const int TestInProgressMessages = 10; + } + + public static class HandshakeProperties + { + public const byte PID = 0; + public const byte Architecture = 1; + public const byte Framework = 2; + public const byte OS = 3; + public const byte SupportedProtocolVersions = 4; + public const byte HostType = 5; + public const byte ModulePath = 6; + public const byte ExecutionId = 7; + public const byte InstanceId = 8; + public const byte IsIDE = 9; + public const byte ExecutionMode = 10; + } + + public static class SessionEventTypes + { + public const byte TestSessionStart = 0; + public const byte TestSessionEnd = 1; + } + + public static class TestSessionEventFields + { + public const ushort SessionType = 1; + public const ushort SessionUid = 2; + public const ushort ExecutionId = 3; + } + + /// + /// Computes the OS-level named pipe name from a friendly identifier. Mirrors + /// NamedPipeServer.GetPipeName in testfx. + /// + public static string GetPipeName(string name) + { + bool isUnix = Path.DirectorySeparatorChar == '/'; + return isUnix + ? Path.Combine("/tmp", name) + : $"testingplatform.pipe.{name.Replace('\\', '.')}"; + } + + /// + /// Reads exactly one framed message from the pipe. Returns on EOF + /// (peer disconnected cleanly). + /// + public static async Task ReadFrameAsync(PipeStream stream, CancellationToken cancellationToken) + { + byte[] header = ArrayPool.Shared.Rent(sizeof(int)); + try + { + if (!await TryReadExactlyAsync(stream, header.AsMemory(0, sizeof(int)), cancellationToken).ConfigureAwait(false)) + { + return null; + } + + int totalPayloadLength = BitConverter.ToInt32(header, 0); + + byte[] payload = new byte[totalPayloadLength]; + if (!await TryReadExactlyAsync(stream, payload, cancellationToken).ConfigureAwait(false)) + { + return null; + } + + int serializerId = BitConverter.ToInt32(payload, 0); + byte[] body = new byte[totalPayloadLength - sizeof(int)]; + Array.Copy(payload, sizeof(int), body, 0, body.Length); + + return new RawMessage(serializerId, body); + } + finally + { + ArrayPool.Shared.Return(header); + } + } + + /// + /// Writes one framed message to the pipe. + /// + public static async Task WriteFrameAsync(PipeStream stream, int serializerId, byte[] body, CancellationToken cancellationToken) + { + int totalPayloadLength = sizeof(int) + body.Length; + byte[] header = BitConverter.GetBytes(totalPayloadLength); + byte[] id = BitConverter.GetBytes(serializerId); + + await stream.WriteAsync(header.AsMemory(0, sizeof(int)), cancellationToken).ConfigureAwait(false); + await stream.WriteAsync(id.AsMemory(0, sizeof(int)), cancellationToken).ConfigureAwait(false); + await stream.WriteAsync(body.AsMemory(0, body.Length), cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + + if (OperatingSystem.IsWindows() && stream is NamedPipeServerStream serverStream) + { + serverStream.WaitForPipeDrain(); + } + } + + /// + /// Decodes the body of a frame into its property + /// dictionary. Format: ushort fieldCount; (byte key, length-prefixed UTF-8 value) * fieldCount. + /// + public static Dictionary DecodeHandshakeBody(byte[] body) + { + Dictionary result = []; + using MemoryStream stream = new(body, writable: false); + + ushort count = ReadUShort(stream); + for (int i = 0; i < count; i++) + { + byte key = (byte)stream.ReadByte(); + string value = ReadLengthPrefixedString(stream); + result.Add(key, value); + } + + return result; + } + + /// + /// Encodes the body of a frame from a property + /// dictionary. + /// + public static byte[] EncodeHandshakeBody(Dictionary properties) + { + using MemoryStream stream = new(); + WriteUShort(stream, (ushort)properties.Count); + foreach (KeyValuePair kvp in properties) + { + stream.WriteByte(kvp.Key); + WriteLengthPrefixedString(stream, kvp.Value); + } + + return stream.ToArray(); + } + + /// + /// Decodes the body of a frame. + /// Format: ushort fieldCount; (ushort fieldId, int fieldSize, payload)*fieldCount + /// where payload shape is determined by fieldId. Returns null for fields that are absent. + /// + public static (byte? SessionType, string? SessionUid, string? ExecutionId) DecodeTestSessionEventBody(byte[] body) + { + byte? sessionType = null; + string? sessionUid = null; + string? executionId = null; + + using MemoryStream stream = new(body, writable: false); + ushort fieldCount = ReadUShort(stream); + for (int i = 0; i < fieldCount; i++) + { + ushort fieldId = ReadUShort(stream); + int fieldSize = ReadInt(stream); + + switch (fieldId) + { + case TestSessionEventFields.SessionType: + sessionType = (byte)stream.ReadByte(); + break; + case TestSessionEventFields.SessionUid: + sessionUid = ReadFixedSizeString(stream, fieldSize); + break; + case TestSessionEventFields.ExecutionId: + executionId = ReadFixedSizeString(stream, fieldSize); + break; + default: + // Unknown field id: skip forward. + stream.Seek(fieldSize, SeekOrigin.Current); + break; + } + } + + return (sessionType, sessionUid, executionId); + } + + private static async Task TryReadExactlyAsync(Stream stream, Memory buffer, CancellationToken cancellationToken) + { + int totalRead = 0; + while (totalRead < buffer.Length) + { + int read = await stream.ReadAsync(buffer.Slice(totalRead), cancellationToken).ConfigureAwait(false); + if (read == 0) + { + return false; + } + + totalRead += read; + } + + return true; + } + + private static ushort ReadUShort(Stream stream) + { + Span bytes = stackalloc byte[sizeof(ushort)]; + stream.ReadExactly(bytes); + return BitConverter.ToUInt16(bytes); + } + + private static int ReadInt(Stream stream) + { + Span bytes = stackalloc byte[sizeof(int)]; + stream.ReadExactly(bytes); + return BitConverter.ToInt32(bytes); + } + + private static void WriteUShort(Stream stream, ushort value) + { + Span bytes = stackalloc byte[sizeof(ushort)]; + BitConverter.TryWriteBytes(bytes, value); + stream.Write(bytes); + } + + private static string ReadLengthPrefixedString(Stream stream) + { + int length = ReadInt(stream); + return ReadFixedSizeString(stream, length); + } + + private static string ReadFixedSizeString(Stream stream, int length) + { + byte[] buffer = ArrayPool.Shared.Rent(length); + try + { + stream.ReadExactly(buffer, 0, length); + return Encoding.UTF8.GetString(buffer, 0, length); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + private static void WriteLengthPrefixedString(Stream stream, string value) + { + int byteCount = Encoding.UTF8.GetByteCount(value); + byte[] header = BitConverter.GetBytes(byteCount); + stream.Write(header, 0, header.Length); + + byte[] buffer = ArrayPool.Shared.Rent(byteCount); + try + { + int written = Encoding.UTF8.GetBytes(value, buffer); + stream.Write(buffer, 0, written); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } +} + +/// A raw decoded pipe frame (serializer id + body bytes, no further decoding). +internal sealed record RawMessage(int SerializerId, byte[] Body); diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/FakeDotnetTestSdk.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/FakeDotnetTestSdk.cs new file mode 100644 index 0000000000..a1c7830585 --- /dev/null +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/FakeDotnetTestSdk.cs @@ -0,0 +1,165 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Globalization; +using System.IO.Pipes; +using System.Runtime.InteropServices; + +namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests.DotnetTestPipe; + +/// +/// In-process stand-in for the dotnet test SDK side of the +/// --server dotnettestcli --dotnet-test-pipe protocol. Listens on a named pipe, performs +/// the same handshake negotiation the real SDK uses today, captures every message the test app +/// sends, and returns a . +/// +/// Built on a hand-rolled wire reader () so that this harness +/// is independent of testfx internal types and exercises the wire format itself. +/// +/// +internal static class FakeDotnetTestSdk +{ + /// Default fake-SDK advertised protocol version. Mirrors today's + /// ProtocolConstants.SupportedVersions on the SDK side. + public const string DefaultSupportedProtocolVersions = "1.0.0"; + + /// + /// Spins up a fake SDK pipe server, runs against it, and returns + /// everything observed during the run. + /// + public static async Task RunAsync( + global::Microsoft.Testing.TestInfrastructure.TestHost testHost, + string? extraArguments = null, + Dictionary? environmentVariables = null, + string supportedProtocolVersions = DefaultSupportedProtocolVersions, + bool isIde = false, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(testHost); + + string pipeId = Guid.NewGuid().ToString("N"); + string osPipeName = DotnetTestPipeProtocol.GetPipeName(pipeId); + + // CurrentUserOnly only exists on .NET Core; on .NET Framework we omit it. + PipeOptions options = PipeOptions.Asynchronous | PipeOptions.CurrentUserOnly; + + // .NET Framework's NamedPipeServerStream takes the pipe name without leading \\.\pipe\ + // — Windows adds it automatically. On Unix, .NET 6+ uses Unix domain sockets at the given + // path (matching what NamedPipeServer.GetPipeName produces with Path.Combine("/tmp", name)). + using NamedPipeServerStream stream = new(osPipeName, PipeDirection.InOut, maxNumberOfServerInstances: 1, PipeTransmissionMode.Byte, options); + + List received = []; + Dictionary? receivedHandshake = null; + Dictionary? sentHandshakeReply = null; + string? negotiatedVersion = null; + + Task hostRun = Task.Run( + async () => + { + string pipeArgs = $"--server dotnettestcli --dotnet-test-pipe {osPipeName}"; + string finalArgs = extraArguments is null ? pipeArgs : $"{pipeArgs} {extraArguments}"; + return await testHost.ExecuteAsync(finalArgs, environmentVariables, cancellationToken: cancellationToken).ConfigureAwait(false); + }, + cancellationToken); + + // Wait for the test app to connect. + await stream.WaitForConnectionAsync(cancellationToken).ConfigureAwait(false); + + // Read frames until the peer disconnects. + while (true) + { + RawMessage? frame = await DotnetTestPipeProtocol.ReadFrameAsync(stream, cancellationToken).ConfigureAwait(false); + if (frame is null) + { + break; + } + + received.Add(frame); + + if (frame.SerializerId == DotnetTestPipeProtocol.SerializerIds.HandshakeMessage) + { + receivedHandshake = DotnetTestPipeProtocol.DecodeHandshakeBody(frame.Body); + string selected = SelectHighestMutuallySupportedVersion(receivedHandshake, supportedProtocolVersions); + negotiatedVersion = selected; + sentHandshakeReply = BuildSdkHandshakeReply(selected, isIde); + byte[] replyBody = DotnetTestPipeProtocol.EncodeHandshakeBody(sentHandshakeReply); + await DotnetTestPipeProtocol.WriteFrameAsync(stream, DotnetTestPipeProtocol.SerializerIds.HandshakeMessage, replyBody, cancellationToken).ConfigureAwait(false); + } + else + { + // For every other request the SDK replies with a VoidResponse (empty body). + await DotnetTestPipeProtocol.WriteFrameAsync(stream, DotnetTestPipeProtocol.SerializerIds.VoidResponse, body: [], cancellationToken).ConfigureAwait(false); + } + } + + TestHostResult hostResult = await hostRun.ConfigureAwait(false); + + return new FakeDotnetTestSdkResult( + hostResult, + received, + receivedHandshake, + sentHandshakeReply, + negotiatedVersion); + } + + /// + /// Mirrors TestApplication.GetSupportedProtocolVersion on the SDK side: takes the + /// semicolon-separated list the test app advertised and returns the highest version that is + /// also in . Returns if none + /// match. + /// + public static string SelectHighestMutuallySupportedVersion(Dictionary handshakeProperties, string sdkSupportedVersions) + { + if (!handshakeProperties.TryGetValue(DotnetTestPipeProtocol.HandshakeProperties.SupportedProtocolVersions, out string? appVersions) + || string.IsNullOrWhiteSpace(appVersions)) + { + return string.Empty; + } + + HashSet sdkSet = new(sdkSupportedVersions.Split(';', StringSplitOptions.RemoveEmptyEntries), StringComparer.Ordinal); + + string? best = null; + Version? bestParsed = null; + foreach (string raw in appVersions.Split(';', StringSplitOptions.RemoveEmptyEntries)) + { + string candidate = raw.Trim(); + if (!sdkSet.Contains(candidate)) + { + continue; + } + + if (!Version.TryParse(candidate, out Version? parsed)) + { + best ??= candidate; + continue; + } + + if (bestParsed is null || parsed > bestParsed) + { + best = candidate; + bestParsed = parsed; + } + } + + return best ?? string.Empty; + } + + private static Dictionary BuildSdkHandshakeReply(string selectedVersion, bool isIde) + { + Dictionary properties = new(capacity: 6) + { + { DotnetTestPipeProtocol.HandshakeProperties.PID, Environment.ProcessId.ToString(CultureInfo.InvariantCulture) }, + { DotnetTestPipeProtocol.HandshakeProperties.Architecture, RuntimeInformation.ProcessArchitecture.ToString() }, + { DotnetTestPipeProtocol.HandshakeProperties.Framework, RuntimeInformation.FrameworkDescription }, + { DotnetTestPipeProtocol.HandshakeProperties.OS, RuntimeInformation.OSDescription }, + { DotnetTestPipeProtocol.HandshakeProperties.SupportedProtocolVersions, selectedVersion }, + }; + + if (isIde) + { + properties.Add(DotnetTestPipeProtocol.HandshakeProperties.IsIDE, bool.TrueString); + } + + return properties; + } +} diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/FakeDotnetTestSdkResult.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/FakeDotnetTestSdkResult.cs new file mode 100644 index 0000000000..f3df515849 --- /dev/null +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/FakeDotnetTestSdkResult.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests.DotnetTestPipe; + +/// +/// Captures everything the fake SDK observed during a single test-app run driven through the +/// --server dotnettestcli --dotnet-test-pipe protocol: the raw frames received on the +/// named pipe, the SDK-selected handshake reply, and the process-level . +/// Used by baseline tests to assert current behavior before any source change. +/// +internal sealed class FakeDotnetTestSdkResult +{ + public FakeDotnetTestSdkResult( + TestHostResult testHostResult, + IReadOnlyList receivedMessages, + Dictionary? receivedHandshake, + Dictionary? sentHandshakeReply, + string? negotiatedProtocolVersion) + { + TestHostResult = testHostResult; + ReceivedMessages = receivedMessages; + ReceivedHandshake = receivedHandshake; + SentHandshakeReply = sentHandshakeReply; + NegotiatedProtocolVersion = negotiatedProtocolVersion; + } + + /// The process-level result: exit code, captured stdout, captured stderr. + public TestHostResult TestHostResult { get; } + + /// All raw frames the test app sent over the pipe, in arrival order. + public IReadOnlyList ReceivedMessages { get; } + + /// The decoded handshake properties the test app sent (advertising its supported + /// protocol versions, PID, architecture, framework, OS, etc.). + public Dictionary? ReceivedHandshake { get; } + + /// The handshake the fake SDK replied with (carrying the selected protocol version). + public Dictionary? SentHandshakeReply { get; } + + /// The version the fake SDK selected from the test app's advertised list, or null/empty if none. + public string? NegotiatedProtocolVersion { get; } + + /// Returns all frames whose serializer ID matches . + public IEnumerable MessagesWithSerializerId(int serializerId) + => ReceivedMessages.Where(m => m.SerializerId == serializerId); +} From c78429ec2c4533d14a0569d8ba6ac4d13494e648 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Mon, 15 Jun 2026 16:47:54 +0200 Subject: [PATCH 2/2] Address PR review feedback on dotnet-test pipe baseline tests - Trim SDK supported-version tokens and project app versions via Select - Fix misleading CurrentUserOnly comment (harness targets $(NetCurrent)) - Advance stream past extra SessionType bytes to stay aligned - Remove unnecessary Task.Run wrapper around already-async ExecuteAsync - Make SelectHighestMutuallySupportedVersion private static - Add assertion message for SentHandshakeReply guard - Use Regex-typed DoesNotMatchRegex overload to keep Compiled flag Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../DotnetTestPipeBaselineTests.cs | 4 ++-- .../DotnetTestPipe/DotnetTestPipeProtocol.cs | 8 +++++++ .../DotnetTestPipe/FakeDotnetTestSdk.cs | 23 ++++++++----------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeBaselineTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeBaselineTests.cs index 322aa2cb9d..5692d5d408 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeBaselineTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeBaselineTests.cs @@ -33,7 +33,7 @@ public async Task DotnetTestPipe_TestAppAdvertisesAndNegotiatesProtocolV100_Toda testHost, cancellationToken: TestContext.CancellationToken); Assert.IsNotNull(result.ReceivedHandshake, "Test app never sent a handshake message."); - Assert.IsNotNull(result.SentHandshakeReply); + Assert.IsNotNull(result.SentHandshakeReply, "Fake SDK never sent a handshake reply — check that the handshake frame was received and decoded."); Assert.IsTrue( result.ReceivedHandshake.TryGetValue(DotnetTestPipeProtocol.HandshakeProperties.SupportedProtocolVersions, out string? appVersions), @@ -92,7 +92,7 @@ public async Task DotnetTestPipe_ChildEmitsNoStdoutOrStderrForPassingRun_Baselin testHost, cancellationToken: TestContext.CancellationToken); Assert.DoesNotMatchRegex( - BannerRegex.ToString(), + BannerRegex, result.TestHostResult.StandardOutput, $"BASELINE: under --server dotnettestcli the test app already silences its banner. " + $"If this fails, something started re-emitting the banner under pipe mode.{Environment.NewLine}" + diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeProtocol.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeProtocol.cs index b161939f54..f6ca6151e0 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeProtocol.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/DotnetTestPipeProtocol.cs @@ -195,6 +195,14 @@ public static (byte? SessionType, string? SessionUid, string? ExecutionId) Decod { case TestSessionEventFields.SessionType: sessionType = (byte)stream.ReadByte(); + + // SessionType is a single byte today, but advance past any extra bytes the + // wire format may carry so subsequent fields stay aligned. + if (fieldSize > 1) + { + stream.Seek(fieldSize - 1, SeekOrigin.Current); + } + break; case TestSessionEventFields.SessionUid: sessionUid = ReadFixedSizeString(stream, fieldSize); diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/FakeDotnetTestSdk.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/FakeDotnetTestSdk.cs index a1c7830585..fb98303e30 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/FakeDotnetTestSdk.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/DotnetTestPipe/FakeDotnetTestSdk.cs @@ -40,7 +40,8 @@ public static async Task RunAsync( string pipeId = Guid.NewGuid().ToString("N"); string osPipeName = DotnetTestPipeProtocol.GetPipeName(pipeId); - // CurrentUserOnly only exists on .NET Core; on .NET Framework we omit it. + // CurrentUserOnly hardens the pipe ACL so only the current user can connect. It is + // available here because this harness only targets .NET (Core) ($(NetCurrent)). PipeOptions options = PipeOptions.Asynchronous | PipeOptions.CurrentUserOnly; // .NET Framework's NamedPipeServerStream takes the pipe name without leading \\.\pipe\ @@ -53,14 +54,9 @@ public static async Task RunAsync( Dictionary? sentHandshakeReply = null; string? negotiatedVersion = null; - Task hostRun = Task.Run( - async () => - { - string pipeArgs = $"--server dotnettestcli --dotnet-test-pipe {osPipeName}"; - string finalArgs = extraArguments is null ? pipeArgs : $"{pipeArgs} {extraArguments}"; - return await testHost.ExecuteAsync(finalArgs, environmentVariables, cancellationToken: cancellationToken).ConfigureAwait(false); - }, - cancellationToken); + string pipeArgs = $"--server dotnettestcli --dotnet-test-pipe {osPipeName}"; + string finalArgs = extraArguments is null ? pipeArgs : $"{pipeArgs} {extraArguments}"; + Task hostRun = testHost.ExecuteAsync(finalArgs, environmentVariables, cancellationToken: cancellationToken); // Wait for the test app to connect. await stream.WaitForConnectionAsync(cancellationToken).ConfigureAwait(false); @@ -108,7 +104,7 @@ public static async Task RunAsync( /// also in . Returns if none /// match. /// - public static string SelectHighestMutuallySupportedVersion(Dictionary handshakeProperties, string sdkSupportedVersions) + private static string SelectHighestMutuallySupportedVersion(Dictionary handshakeProperties, string sdkSupportedVersions) { if (!handshakeProperties.TryGetValue(DotnetTestPipeProtocol.HandshakeProperties.SupportedProtocolVersions, out string? appVersions) || string.IsNullOrWhiteSpace(appVersions)) @@ -116,13 +112,14 @@ public static string SelectHighestMutuallySupportedVersion(Dictionary sdkSet = new(sdkSupportedVersions.Split(';', StringSplitOptions.RemoveEmptyEntries), StringComparer.Ordinal); + HashSet sdkSet = new( + sdkSupportedVersions.Split(';', StringSplitOptions.RemoveEmptyEntries).Select(static raw => raw.Trim()), + StringComparer.Ordinal); string? best = null; Version? bestParsed = null; - foreach (string raw in appVersions.Split(';', StringSplitOptions.RemoveEmptyEntries)) + foreach (string candidate in appVersions.Split(';', StringSplitOptions.RemoveEmptyEntries).Select(static raw => raw.Trim())) { - string candidate = raw.Trim(); if (!sdkSet.Contains(candidate)) { continue;