Expand OpenTelemetry support: conventions, resource, env-driven setup, MSTest spans

Directory.Packages.props

@@ -101,6 +101,7 @@
  <PackageVersion Include="Microsoft.Win32.Registry" Version="5.0.0" />
  <PackageVersion Include="Microsoft.WindowsAppSDK" Version="1.8.251003001" />
  <PackageVersion Include="OpenTelemetry" Version="1.15.3" />
+ <PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.15.3" />
  <PackageVersion Include="System.Memory" Version="$(SystemMemoryVersion)" />
  <PackageVersion Include="System.Text.Json" Version="10.0.8" />
  <PackageVersion Include="System.Threading.Tasks.Extensions" Version="$(SystemThreadingTasksExtensionsVersion)" />

src/Adapter/MSTest.TestAdapter/InternalAPI/InternalAPI.Unshipped.txt

@@ -60,3 +60,10 @@ static Microsoft.Testing.Extensions.RunSettingsProviderHelper.TryLoadRunSettings
 static Microsoft.VisualStudio.TestTools.UnitTesting.MSTestTestNodeConverter.ToEmptyResultTestNode(Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel.UnitTestElement! element, bool isTrxEnabled) -> Microsoft.Testing.Platform.Extensions.Messages.TestNode!
 static readonly Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.AdapterTestProperties.DependenciesProperty -> Microsoft.VisualStudio.TestPlatform.ObjectModel.TestProperty!
 static readonly Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.AdapterTestProperties.ResourceLocksProperty -> Microsoft.VisualStudio.TestPlatform.ObjectModel.TestProperty!
+Microsoft.VisualStudio.TestTools.UnitTesting.MSTestPlatformActivity
+Microsoft.VisualStudio.TestTools.UnitTesting.MSTestPlatformActivity.Dispose() -> void
+Microsoft.VisualStudio.TestTools.UnitTesting.MSTestPlatformActivity.MSTestPlatformActivity(Microsoft.Testing.Platform.Telemetry.IPlatformActivity! activity) -> void
+Microsoft.VisualStudio.TestTools.UnitTesting.MSTestPlatformActivity.RecordException(System.Exception! exception) -> void
+Microsoft.VisualStudio.TestTools.UnitTesting.MSTestPlatformActivity.SetFailed(string? description) -> void
+Microsoft.VisualStudio.TestTools.UnitTesting.MSTestPlatformActivity.SetTag(string! key, object? value) -> void
+static Microsoft.VisualStudio.TestTools.UnitTesting.MSTestPlatformActivity.TryEnable(System.IServiceProvider! serviceProvider) -> void

src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestPlatformActivity.cs

@@ -0,0 +1,51 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+#if !WINDOWS_UWP
+using Microsoft.Testing.Platform.Telemetry;
+using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter;
+
+namespace Microsoft.VisualStudio.TestTools.UnitTesting;
+
+/// <summary>
+/// Bridges the engine's dependency-free <see cref="IMSTestActivity"/> seam onto the platform's OpenTelemetry
+/// service, so MSTest fixture and test-method spans nest under the platform's test-case spans.
+/// </summary>
+[SuppressMessage("ApiDesign", "RS0030:Do not use banned APIs", Justification = "We can use MTP from this folder")]
+internal sealed class MSTestPlatformActivity(IPlatformActivity activity) : IMSTestActivity
+{
+ public void SetTag(string key, object? value)
+ => activity.SetTag(key, value);
+
+ public void RecordException(Exception exception)
+ => activity.RecordException(exception);
+
+ public void SetFailed(string? description)
+ => activity.SetStatus(PlatformActivityStatusCode.Error, description);
+
+ public void Dispose()
+ => activity.Dispose();
+
+ /// <summary>
+ /// Installs the tracing factory on <see cref="MSTestInstrumentation"/> when the platform has an OpenTelemetry
+ /// service registered. Does nothing (leaving MSTest tracing disabled and free) otherwise.
+ /// </summary>
+ internal static void TryEnable(IServiceProvider serviceProvider)
+ {
+ if (serviceProvider.GetService(typeof(IPlatformOpenTelemetryService)) is not IPlatformOpenTelemetryService otelService)
+ {
+ MSTestInstrumentation.SetActivityFactory(null);
+ return;
+ }
+
+ MSTestInstrumentation.SetActivityFactory((name, tags)
+ // A non-ambient span is required, not an optimization - see the remarks on MSTestInstrumentation.
+ // Because the span is not ambient it cannot pick a parent up from the ambient context either, so it
+ // is parented explicitly to the same test-framework span the platform's test-case spans use, which
+ // keeps everything in one trace.
+ => otelService.StartNonAmbientActivity(name, tags, otelService.TestFrameworkActivity?.Id) is { } platformActivity
+ ? new MSTestPlatformActivity(platformActivity)
+ : null);
+ }
+}
+#endif

src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestTestFramework.cs

@@ -55,6 +55,10 @@ public MSTestTestFramework(MSTestExtension extension, Func<IEnumerable<Assembly>
  _configuration = new(serviceProvider.GetConfiguration());
  _loggerFactory = serviceProvider.GetRequiredService<ILoggerFactory>();
  PlatformServiceProvider.Instance.AdapterTraceLogger = new MTPTraceLogger(_loggerFactory.CreateLogger("mstest-trace"));
+
+ // Let the engine emit fixture/test-method spans that nest under the platform's test-case spans. This is a
+ // no-op unless the OpenTelemetry extension is registered.
+ MSTestPlatformActivity.TryEnable(serviceProvider);
  }
 
  public string Uid => _extension.Uid;

src/Adapter/MSTestAdapter.PlatformServices/Execution/TestAssemblyInfo.cs

@@ -186,6 +186,16 @@ public async Task<TestResult> RunAssemblyInitializeAsync(TestContext testContext
  // Perform a check again.
  if (!IsAssemblyInitializeExecuted)
  {
+ // Assembly initialize can dominate the wall-clock time of a run (spinning up a database,
+ // a browser, ...). Giving it its own span is what lets you tell "the test is slow" apart
+ // from "the assembly fixture is slow".
+ using IMSTestActivity? activity = MSTestInstrumentation.IsEnabled
+ ? MSTestInstrumentation.StartFixtureActivity(
+ MSTestInstrumentation.ActivityNames.AssemblyInitialize,
+ "assembly_initialize",
+ AssemblyInitializeMethod.DeclaringType?.FullName,
+ Assembly.GetName().Name)
+ : null;
  try
  {
  AssemblyInitializationException = await FixtureMethodRunner.RunWithTimeoutAndCancellationAsync(
@@ -223,6 +233,13 @@ public async Task<TestResult> RunAssemblyInitializeAsync(TestContext testContext
  }
  finally
  {
+ // RunWithTimeoutAndCancellationAsync *returns* (rather than throws) the failure for the
+ // timeout and cancellation paths, so the catch above is not enough to mark the span failed.
+ if (AssemblyInitializationException is not null)
+ {
+ activity?.RecordException(AssemblyInitializationException);
+ }
+
  IsAssemblyInitializeExecuted = true;
  }
  }
@@ -273,6 +290,13 @@ private static TestFailedException GetTestFailedExceptionFromAssemblyInitializeE
  try
  {
  await _assemblyInfoExecuteSyncSemaphore.WaitAsync().ConfigureAwait(false);
+ using IMSTestActivity? activity = MSTestInstrumentation.IsEnabled
+ ? MSTestInstrumentation.StartFixtureActivity(
+ MSTestInstrumentation.ActivityNames.AssemblyCleanup,
+ "assembly_cleanup",
+ AssemblyCleanupMethod.DeclaringType?.FullName,
+ Assembly.GetName().Name)
+ : null;
  AssemblyCleanupException = await FixtureMethodRunner.RunWithTimeoutAndCancellationAsync(
  () => AssemblyCleanupMethod.InvokeAsFixtureMethodAsync(
  testContext,
@@ -283,6 +307,13 @@ private static TestFailedException GetTestFailedExceptionFromAssemblyInitializeE
  ExecutionContext,
  Resource.AssemblyCleanupWasCancelled,
  Resource.AssemblyCleanupTimedOut).ConfigureAwait(false);
+
+ // RunWithTimeoutAndCancellationAsync returns (rather than throws) the failure for the timeout and
+ // cancellation paths, so record it here while the span is still open.
+ if (AssemblyCleanupException is not null)
+ {
+ activity?.RecordException(AssemblyCleanupException);
+ }
  }
  catch (Exception ex)
  {

src/Adapter/MSTestAdapter.PlatformServices/Execution/TestClassInfo.Cleanup.cs

@@ -197,6 +197,14 @@ internal sealed partial class TestClassInfo
  timeout = localTimeout;
  }
 
+ using IMSTestActivity? activity = MSTestInstrumentation.IsEnabled
+ ? MSTestInstrumentation.StartFixtureActivity(
+ MSTestInstrumentation.ActivityNames.ClassCleanup,
+ "class_cleanup",
+ methodInfo.DeclaringType?.FullName,
+ methodInfo.DeclaringType?.Assembly.GetName().Name)
+ : null;
+
  TestFailedException? result = await FixtureMethodRunner.RunWithTimeoutAndCancellationAsync(
  () => methodInfo.InvokeAsFixtureMethodAsync(
  testContext,
@@ -208,6 +216,11 @@ internal sealed partial class TestClassInfo
  Resource.ClassCleanupWasCancelled,
  Resource.ClassCleanupTimedOut).ConfigureAwait(false);
 
+ if (result is not null)
+ {
+ activity?.RecordException(result);
+ }
+
  return result;
  }
 }

src/Adapter/MSTestAdapter.PlatformServices/Execution/TestClassInfo.Initializer.cs

@@ -279,6 +279,16 @@ async Task<TestResult> DoRunAsync()
  timeout = localTimeout;
  }
 
+ // One span per class-initialize method (including inherited ones), so a slow or throwing class fixture is
+ // attributable in the trace instead of being folded into the first test of the class.
+ using IMSTestActivity? activity = MSTestInstrumentation.IsEnabled
+ ? MSTestInstrumentation.StartFixtureActivity(
+ MSTestInstrumentation.ActivityNames.ClassInitialize,
+ "class_initialize",
+ methodInfo.DeclaringType?.FullName,
+ methodInfo.DeclaringType?.Assembly.GetName().Name)
+ : null;
+
  TestFailedException? result = await FixtureMethodRunner.RunWithTimeoutAndCancellationAsync(
  () => methodInfo.InvokeAsFixtureMethodAsync(
  testContext,
@@ -290,6 +300,11 @@ async Task<TestResult> DoRunAsync()
  Resource.ClassInitializeWasCancelled,
  Resource.ClassInitializeTimedOut).ConfigureAwait(false);
 
+ if (result is not null)
+ {
+ activity?.RecordException(result);
+ }
+
  return result;
  }
 }

src/Adapter/MSTestAdapter.PlatformServices/Execution/TestMethodInfo.Lifecycle.cs

@@ -260,14 +260,29 @@ private async SynchronizationContextPreservingTask<bool> RunTestInitializeMethod
  timeout = localTimeout;
  }
 
- return await InvokeFixtureMethodAsync(
+ using IMSTestActivity? activity = MSTestInstrumentation.IsEnabled
+ ? MSTestInstrumentation.StartFixtureActivity(
+ MSTestInstrumentation.ActivityNames.TestInitialize,
+ "test_initialize",
+ methodInfo.DeclaringType?.FullName,
+ methodInfo.DeclaringType?.Assembly.GetName().Name)
+ : null;
+
+ TestFailedException? result = await InvokeFixtureMethodAsync(
  methodInfo,
  classInstance,
  arguments: null,
  timeout,
  timeoutTokenSource,
  Resource.TestInitializeWasCancelled,
  Resource.TestInitializeTimedOut).ConfigureAwait(false);
+
+ if (result is not null)
+ {
+ activity?.RecordException(result);
+ }
+
+ return result;
  }
 
  private async SynchronizationContextPreservingTask<TestFailedException?> InvokeGlobalInitializeMethodAsync(MethodInfo methodInfo, TimeoutInfo? timeoutInfo, CancellationTokenSource? timeoutTokenSource)
@@ -288,14 +303,29 @@ private async SynchronizationContextPreservingTask<bool> RunTestInitializeMethod
  timeout = localTimeout;
  }
 
- return await InvokeFixtureMethodAsync(
+ using IMSTestActivity? activity = MSTestInstrumentation.IsEnabled
+ ? MSTestInstrumentation.StartFixtureActivity(
+ MSTestInstrumentation.ActivityNames.TestCleanup,
+ "test_cleanup",
+ methodInfo.DeclaringType?.FullName,
+ methodInfo.DeclaringType?.Assembly.GetName().Name)
+ : null;
+
+ TestFailedException? result = await InvokeFixtureMethodAsync(
  methodInfo,
  classInstance,
  arguments: null,
  timeout,
  timeoutTokenSource,
  Resource.TestCleanupWasCancelled,
  Resource.TestCleanupTimedOut).ConfigureAwait(false);
+
+ if (result is not null)
+ {
+ activity?.RecordException(result);
+ }
+
+ return result;
  }
 
  private async SynchronizationContextPreservingTask<TestFailedException?> InvokeGlobalCleanupMethodAsync(MethodInfo methodInfo, TimeoutInfo? timeoutInfo, CancellationTokenSource? timeoutTokenSource)