feat: Matrix integration tests

.github/actions/setup-dotnet/action.yml

@@ -5,6 +5,10 @@ inputs:
  description: 'Whether to run dotnet build after restore'
  required: false
  default: 'true'
+ configuration:
+ description: 'What configuration to run (e.g., Release or Debug)'
+ required: false
+ default: 'Release'
 runs:
  using: 'composite'
  steps:
@@ -31,4 +35,4 @@ runs:
  - name: Build with dotnet
  if: inputs.build == 'true'
  shell: bash
- run: dotnet build -p:ContinuousIntegrationBuild=True --configuration Release --no-restore
+ run: dotnet build -p:ContinuousIntegrationBuild=True --configuration ${{ inputs.configuration }} --no-restore

.github/workflows/Build-Test-And-Deploy.yaml

@@ -38,17 +38,17 @@ jobs:
 
  - name: Run .NET Tests
  id: run-dotnet-tests
- run: dotnet test --no-build --configuration Release --blame-hang-timeout 15m --blame-hang-dump-type full -l trx --results-directory ./TestResults
+ run: dotnet test --no-build --configuration Release --blame-hang-timeout 15m --blame-hang-dump-type full -l trx --results-directory ./TestResults --filter TestType!=Integration
  env:
  POCKETLOGGER_LOG_PATH: ${{ github.workspace }}/artifacts/logs/pocketlogger.log
 
  - name: Convert TRX to Playlist
  if: ${{ steps.run-dotnet-tests.outcome == 'failure' || failure() }}
  uses: BenjaminMichaelis/trx-to-vsplaylist@v4
  with:
- trx-file-path: './TestResults/*.trx'
- test-outcomes: 'Failed'
- artifact-name: 'linux-test-playlists'
+ trx-file-path: "./TestResults/*.trx"
+ test-outcomes: "Failed"
+ artifact-name: "linux-test-playlists"
 
  - name: Set up Docker Buildx
  uses: docker/setup-buildx-action@v4
@@ -127,24 +127,33 @@ jobs:
 
  - name: Run .NET Tests
  id: run-dotnet-tests
- run: dotnet test --no-build --configuration Release --blame-hang-timeout 15m --blame-hang-dump-type full -l trx --results-directory ./TestResults
+ run: dotnet test --no-build --configuration Release --blame-hang-timeout 15m --blame-hang-dump-type full -l trx --results-directory ./TestResults --filter TestType!=Integration
  env:
  POCKETLOGGER_LOG_PATH: ${{ github.workspace }}/artifacts/logs/pocketlogger.log
 
  - name: Convert TRX to Playlist
  if: ${{ steps.run-dotnet-tests.outcome == 'failure' || failure() }}
  uses: BenjaminMichaelis/trx-to-vsplaylist@v4
  with:
- trx-file-path: './TestResults/*.trx'
- test-outcomes: 'Failed'
- artifact-name: 'windows-test-playlists'
+ trx-file-path: "./TestResults/*.trx"
+ test-outcomes: "Failed"
+ artifact-name: "windows-test-playlists"
 
  integration-tests:
- # Integration tests use Playwright and are gated behind RunIntegrationTests=true.
- # IntegrationTestFactAttribute explicitly skips these tests on Linux, so a
- # Windows runner is required to actually execute them.
- runs-on: windows-latest
-
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [windows-latest, ubuntu-latest, macos-latest]
+ browser: [chromium, firefox, webkit]
+ exclude:
+ - os: windows-latest
+ browser: webkit
+ - os: ubuntu-latest
+ browser: webkit
+ - os: macos-latest
+ browser: firefox
+
+ runs-on: ${{ matrix.os }}
  steps:
  - uses: actions/checkout@v7
 
@@ -153,22 +162,25 @@ jobs:
 
  - name: Set up .NET environment
  uses: ./.github/actions/setup-dotnet
+ with:
+ configuration: Debug
 
  - name: Run .NET Integration Tests
  id: run-dotnet-integration-tests
- run: dotnet test --no-build --configuration Release --blame-hang-timeout 15m --blame-hang-dump-type full -l trx --results-directory ./TestResults
+ run: dotnet test --no-build --configuration Debug --blame-hang-timeout 15m --blame-hang-dump-type full -l trx --results-directory ./TestResults --filter TestType=Integration -- Playwright.BrowserName=${{ matrix.browser }}
  env:
  RunIntegrationTests: true
  POCKETLOGGER_LOG_PATH: ${{ github.workspace }}/artifacts/logs/pocketlogger.log
  TRYDOTNET_PREBUILDS_PATH: ${{ github.workspace }}/artifacts/trydotnet-prebuilds
+ Playwright_BrowserName: ${{ matrix.browser }}
 
  - name: Convert TRX to Playlist
  if: ${{ steps.run-dotnet-integration-tests.outcome == 'failure' || failure() }}
  uses: BenjaminMichaelis/trx-to-vsplaylist@v4
  with:
- trx-file-path: './TestResults/*.trx'
- test-outcomes: 'Failed'
- artifact-name: 'integration-test-playlists'
+ trx-file-path: "./TestResults/*.trx"
+ test-outcomes: "Failed"
+ artifact-name: "integration-test-playlists"
 
  deploy-development:
  if: github.ref == 'refs/heads/main'

src/Microsoft.TryDotNet.IntegrationTests/EditorTests.cs

@@ -15,6 +15,7 @@
 namespace Microsoft.TryDotNet.IntegrationTests;
 
 [LogToPocketLogger(FileNameEnvironmentVariable = "POCKETLOGGER_LOG_PATH")]
+[Trait("TestType", "Integration")]
 public class EditorTests : PlaywrightTestBase
 {
  public EditorTests(IntegratedServicesFixture services, ITestOutputHelper output) : base(services, output)
@@ -341,11 +342,13 @@ await editor.PressSequentiallyAsync(@"/////////////////////////
 
  }, new PageRunAndWaitForConsoleMessageOptions()
  {
- Predicate = message => message.Text.Contains("[MonacoEditorAdapter.setMarkers]"),
+ // Skip the initial empty setMarkers([]) call that Monaco fires to clear previous
+ // markers; wait for a non-empty call that actually contains diagnostic data.
+ Predicate = message => message.Text.Contains("[MonacoEditorAdapter.setMarkers]") && !message.Text.Contains(": []"),
  Timeout = Debugger.IsAttached ? 0.0f : (float)TimeSpan.FromMinutes(10).TotalMilliseconds
  });
 
- await Task.Delay(TimeSpan.FromSeconds(1));
+ await Task.Delay(TimeSpan.FromSeconds(3));
 
  var diagnosticMarker = page.Locator("div .squiggly-error");
  await diagnosticMarker.IsVisibleAsync();

src/Microsoft.TryDotNet.IntegrationTests/IntegratedServicesFixture.cs

@@ -1,6 +1,7 @@
 // Copyright (c) .NET Foundation and contributors. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using System;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.DotNet.Interactive.CSharpProject.Build;

src/Microsoft.TryDotNet.IntegrationTests/IntegrationTestFactAttribute.cs

@@ -1,33 +1,30 @@
-// Copyright (c) .NET Foundation and contributors. All rights reserved.
-// Licensed under the MIT license. See LICENSE file in the project root for full license information.
-
-using System;
-using System.Runtime.InteropServices;
-
-using Xunit;
-
-namespace Microsoft.TryDotNet.IntegrationTests
-{
- internal class IntegrationTestFactAttribute : FactAttribute
- {
- private const string EnvironmentVariableName = "RunIntegrationTests";
-
- public IntegrationTestFactAttribute(string? skipReason = null)
- {
- var variableValue = Environment.GetEnvironmentVariable(EnvironmentVariableName) ?? "false";
- switch (variableValue.ToLowerInvariant())
- {
- case "1":
- case "true":
- if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
- {
- Skip = string.IsNullOrWhiteSpace(skipReason) ? "Ignored on Linux" : skipReason;
- }
- break;
- default:
- Skip = $"Skipping integration tests because environment variable '{EnvironmentVariableName}' was not 'true' or '1'.";
- break;
- }
- }
- }
-}
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Runtime.InteropServices;
+
+using Xunit;
+
+namespace Microsoft.TryDotNet.IntegrationTests
+{
+ internal class IntegrationTestFactAttribute : FactAttribute
+ {
+ private const string EnvironmentVariableName = "RunIntegrationTests";
+
+ public IntegrationTestFactAttribute(string? skipReason = null)
+ {
+ var variableValue = Environment.GetEnvironmentVariable(EnvironmentVariableName) ?? "false";
+ switch (variableValue.ToLowerInvariant())
+ {
+ case "1":
+ case "true":
+ // Run tests
+ break;
+ default:
+ Skip = $"Skipping integration tests because environment variable '{EnvironmentVariableName}' was not 'true' or '1'.";
+ break;
+ }
+ }
+ }
+}

src/Microsoft.TryDotNet.IntegrationTests/Microsoft.TryDotNet.IntegrationTests.csproj

@@ -10,6 +10,10 @@
  <DebugType>embedded</DebugType>
  </PropertyGroup>
 
+ <ItemGroup>
+ <Using Include="Xunit" />
+ </ItemGroup>
+
  <ItemGroup>
  <PackageReference Include="Assent" />
  <PackageReference Include="AwesomeAssertions" />
@@ -71,6 +75,5 @@ public static class BuildProperties
  <Target Name="PublishTool" BeforeTargets="CoreCompile" Condition="'$(DisableArcade)' != '1'">
  <Exec Command="dotnet publish -o $(PublishLocation)" WorkingDirectory="$(MSBuildThisFileDirectory)..\Microsoft.TryDotNet" />
  </Target>
-
 </Project>
 

src/Microsoft.TryDotNet.IntegrationTests/PageExtensions.cs

@@ -96,7 +96,8 @@ public static async Task ClearMonacoEditor(this IPage page)
  var editor = page.Locator(@"textarea[role = ""textbox""]");
  await editor.IsVisibleAsync();
  await editor.FocusAsync();
- await editor.PressAsync("Control+a");
+ // Use ControlOrMeta+a so the shortcut works on both macOS (Cmd+A) and other platforms (Ctrl+A).
+ await editor.PressAsync("ControlOrMeta+a");
  await editor.PressAsync("Delete");
  }
 

src/Microsoft.TryDotNet.IntegrationTests/PlaywrightSession.cs

@@ -20,9 +20,24 @@ public PlaywrightSession(IPlaywright playwright, IBrowser browser)
 
  public IBrowser Browser { get; }
 
- public static async Task<PlaywrightSession> StartAsync()
+ public static async Task<PlaywrightSession> StartAsync(string? browserName = null)
  {
- var exitCode = Playwright.Program.Main(["install", "chromium"]);
+
+ string? selectedBrowser = browserName
+ ?? Environment.GetEnvironmentVariable("Playwright.BrowserName")
+ ?? Environment.GetEnvironmentVariable("Playwright_BrowserName");
+
+ int exitCode;
+ if (!string.IsNullOrWhiteSpace(selectedBrowser))
+ {
+ selectedBrowser = selectedBrowser.ToLowerInvariant();
+ exitCode = Playwright.Program.Main(["install", selectedBrowser]);
+ }
+ else
+ {
+ exitCode = Playwright.Program.Main(["install"]);
+ selectedBrowser = "chromium";
+ }
  if (exitCode is not 0)
  {
  throw new Exception($"Playwright exited with code {exitCode}");
@@ -37,7 +52,13 @@ public static async Task<PlaywrightSession> StartAsync()
  browserTypeLaunchOptions.Headless = false;
  }
 
- var browser = await session.Chromium.LaunchAsync(browserTypeLaunchOptions).Timeout(TimeSpan.FromMinutes(5), "Timeout launching browser");
+ IBrowser browser = selectedBrowser switch
+ {
+ "chromium" => await session.Chromium.LaunchAsync(browserTypeLaunchOptions).Timeout(TimeSpan.FromMinutes(5), "Timeout launching browser"),
+ "firefox" => await session.Firefox.LaunchAsync(browserTypeLaunchOptions).Timeout(TimeSpan.FromMinutes(5), "Timeout launching browser"),
+ "webkit" => await session.Webkit.LaunchAsync(browserTypeLaunchOptions).Timeout(TimeSpan.FromMinutes(5), "Timeout launching browser"),
+ _ => throw new ArgumentException($"Unknown browser '{selectedBrowser}'. Valid values: chromium, firefox, webkit.")
+ };
 
  return new PlaywrightSession(session, browser);
  }

src/Microsoft.TryDotNet.IntegrationTests/PlaywrightTestBase.cs

@@ -32,7 +32,10 @@ protected PlaywrightTestBase(
  protected async Task<IPage> NewPageAsync()
  {
  var playwright = await Services.GetPlaywrightAsync();
- return await playwright.Browser.NewPageAsync();
+ var page = await playwright.Browser.NewPageAsync();
+ // Firefox needs more time to reach NetworkIdle due to Blazor WASM loading.
+ page.SetDefaultNavigationTimeout(90_000f);
+ return page;
  }
 
  protected async Task<Uri> TryDotNetUrlAsync()

src/Microsoft.TryDotNet.IntegrationTests/TryDotNetJsIntegrationTests.cs

@@ -14,6 +14,7 @@
 namespace Microsoft.TryDotNet.IntegrationTests;
 
 [LogToPocketLogger(FileNameEnvironmentVariable = "POCKETLOGGER_LOG_PATH")]
+[Trait("TestType", "Integration")]
 public class TryDotNetJsIntegrationTests : PlaywrightTestBase
 {
  public TryDotNetJsIntegrationTests(IntegratedServicesFixture services, ITestOutputHelper output) : base(services, output)

src/Microsoft.TryDotNet.IntegrationTests/WasmRunnerTests.cs

@@ -12,6 +12,7 @@
 namespace Microsoft.TryDotNet.IntegrationTests;
 
 [LogToPocketLogger(FileNameEnvironmentVariable = "POCKETLOGGER_LOG_PATH")]
+[Trait("TestType", "Integration")]
 public class WasmRunnerTests : PlaywrightTestBase
 {
  public WasmRunnerTests(IntegratedServicesFixture services, ITestOutputHelper output) : base(services, output)