Helix proof of concept for unit tests

eng/Version.Details.xml

@@ -57,5 +57,9 @@ Note: if the Uri is a new place, you will need to add a subscription from that p
  <Uri>https://github.com/dotnet/arcade</Uri>
  <Sha>1d2edb248bc2dc37fa0669053b0c88f3bffffb26</Sha>
  </Dependency>
+ <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19156.20">
+ <Uri>https://github.com/dotnet/arcade</Uri>
+ <Sha>b94a591377451bda3ac80e597d7faff03a06c29e</Sha>
+ </Dependency>
  </ToolsetDependencies>
 </Dependencies>

eng/Versions.props

@@ -24,10 +24,18 @@
  <MicrosoftDotNetGenFacadesPackageVersion>1.0.0-beta.19156.20</MicrosoftDotNetGenFacadesPackageVersion>
  <MicrosoftDotNetXUnitExtensionsPackageVersion>2.4.0-beta.19156.20</MicrosoftDotNetXUnitExtensionsPackageVersion>
  </PropertyGroup>
- <!-- Additional dependencies -->
+ <!-- XUnit-related unchanging dependencies -->
  <PropertyGroup>
- <AccessibilityPackageVersion>4.6.0-alpha-27122-5</AccessibilityPackageVersion>
+ <XUnitVersion>2.4.1-pre.build.4059</XUnitVersion>
+ <XUnitAssertVersion>$(XUnitVersion)</XUnitAssertVersion>
+ <XUnitRunnerConsoleVersion>$(XUnitVersion)</XUnitRunnerConsoleVersion>
+ <XUnitRunnerVisualStudioVersion>$(XUnitVersion)</XUnitRunnerVisualStudioVersion>
+ <XUnitExtensibilityExecutionVersion>$(XUnitVersion)</XUnitExtensibilityExecutionVersion>
  <XUnitStaFactPackageVersion>0.3.5</XUnitStaFactPackageVersion>
+ </PropertyGroup>
+ <!-- Additional unchanging dependencies -->
+ <PropertyGroup>
+ <AccessibilityPackageVersion>4.6.0-alpha-27122-5</AccessibilityPackageVersion>
  <MoqPackageVersion>4.10.0</MoqPackageVersion>
  <MicrosoftTargetingPackNETFrameworkv472PackageVersion>1.0.0</MicrosoftTargetingPackNETFrameworkv472PackageVersion>
  </PropertyGroup>

eng/ci-helix-test.yml

@@ -0,0 +1,30 @@
+parameters:
+ # All parameters are required for our purposes (except HelixType); see send-to-helix.yml for details
+ RepoName: ''
+ HelixType: 'tests/default' 
+ HelixTargetQueues: '' 
+ HelixAccessToken: '' 
+ XUnitProjects: ''
+ DisplayNamePrefix: '' 
+
+steps:
+- template: /eng/common/templates/steps/send-to-helix.yml
+ parameters:
+ HelixSource: ${{ parameters.RepoName }}
+ HelixType: ${{ parameters.HelixType }} 
+ HelixBuild: $(Build.BuildNumber)
+ HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
+ HelixAccessToken: ${{ parameters.HelixAccessToken }} # only defined for internal CI
+ Creator: ${{ parameters.RepoName }} # required for public / external (when there is no access token)
+ XUnitProjects: ${{ parameters.XUnitProjects }} 
+ XUnitPublishTargetFramework: netcoreapp3.0 # Whatever tfm will work for our test projects for `dotnet publish`
+ XUnitRuntimeTargetFramework: netcoreapp2.0 # Whatever tfm to pick from the xunit package, it must exist in that package or the helix job will fail
+ XUnitRunnerVersion: '2.4.1' # Should match XUnitRunnerConsoleVersion in eng/Versions.props
+ IncludeDotNetCli: true
+ DotNetCliPackageType: 'sdk'
+ DotNetCliVersion: '3.0.100-preview-010184' # MUST be official release: https://dotnet.microsoft.com/download/dotnet-core/3.0 ; does not need to match sdk we build against
+ EnableXUnitReporter: true
+ WaitForWorkItemCompletion: true
+ DisplayNamePrefix: ${{ parameters.DisplayNamePrefix }}
+ condition: succeeded()
+ continueOnError: false

eng/ci.yml

@@ -1,10 +1,8 @@
 parameters:
-
- # Needed because agent pool can't be read from a user-defined variable (Azure DevOps limitation)
- agentPool: dotnet-external-temp-vs2019
-
- # Needed because runAsPublic is used in template expressions, which can't read from user-defined variables
- runAsPublic: true
+ # This section is required because agent pool can't be read from a user-defined variable (Azure DevOps limitation)
+ agentPool: dotnet-external-temp-vs2019 
+ runAsPublic: true 
+ repoName: dotnet/winforms 
 
 jobs:
 - template: /eng/common/templates/jobs/jobs.yml
@@ -15,7 +13,7 @@ jobs:
  enablePublishTestResults: true
  enablePublishBuildAssets: true
  enableTelemetry: true
- helixRepo: dotnet/winforms
+ helixRepo: $(repoName)
 
  jobs:
  - job: Windows_NT
@@ -44,16 +42,19 @@ jobs:
  value: ''
 
  # needed for helix jobs
- - name: _HelixAgentPool
+ - name: _UnitTestHelixAgentPool
  value: 'Windows.10.Amd64.Open'
  - name: _WinformsControlsTestBinDir
  value: $(BUILD.SOURCESDIRECTORY)\artifacts\bin\WinformsControlsTest\$(_BuildConfig)\netcoreapp3.0
  - name: _WinformsFuncTestBinDir
  value: $(BUILD.SOURCESDIRECTORY)\artifacts\bin\System.Windows.Forms.Func.Tests\$(_BuildConfig)\netcoreapp3.0
  - name: _HelixStagingDir
  value: $(BUILD.STAGINGDIRECTORY)\helix\functests
+ - name: _HelixToken
+ value: ''
 
- # Override some values if we're building internally
+
+ # Override some values if we're building internally (not public)
  - ${{ if eq(parameters.runAsPublic, 'false') }}:
 
  # note: You have to use list syntax here (-name: value) or you will get errors about declaring the same variable multiple times
@@ -68,9 +69,8 @@ jobs:
  - name: _PublishBlobFeedUrl
  value: https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json
 
- # note: Even though they are referenced here, user defined variables (like $(_SignType)) are not resolved 
- # until the agent is running on the machine. They can be overridden any time before they are resolved,
- # like in the job matrix below (see Build_Debug)
+ # note: Even though they are referenced here, user defined variables (like $(_SignType)) are not resolved until the agent
+ # is running on the machine. They can be overridden any time before they are resolved, like in the job matrix below (see Build_Debug)
  - name: _SignArgs
  value: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) /p:Sign=$(_Sign)
  - name: _PublishArgs
@@ -82,8 +82,11 @@ jobs:
  - name: _OfficialBuildIdArgs
  value: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
 
- - name: _HelixAgentPool
+ # Windows.81.Amd64 will fail on Debug.Assert in TryGetSystemMetricsForDpi in UnsafeNativeMethods
+ - name: _UnitTestHelixAgentPool
  value: 'Windows.10.Amd64'
+ - name: _HelixToken
+ value: '$(helix-token)'
 
  strategy:
  matrix:
@@ -109,15 +112,21 @@ jobs:
  /bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\Build.binlog 
  displayName: Build
 
- # Run unit tests and rename binlog
- - script: eng\cibuild.cmd
- -configuration $(_BuildConfig)
- $(_OfficialBuildIdArgs)
- -test
- /bl:$(BUILD.SOURCESDIRECTORY)\artifacts\log\$(_BuildConfig)\Test.binlog
- displayName: Run Unit Tests 
-
+ # Run unit tests on Helix
+ - template: /eng/ci-helix-test.yml
+ parameters:
+ RepoName: ${{ parameters.repoName }}
+ HelixType: 'tests/unit'
+ HelixTargetQueues: $(_UnitTestHelixAgentPool)
+ HelixAccessToken: $(_HelixToken) # only defined for internal CI
+ # Should be $(BUILD.SOURCESDIRECTOR)System.Windows.Forms/src/**/*.Tests.csproj, but cannot until https://github.com/dotnet/arcade/issues/2156 is fixed
+ XUnitProjects: '$(BUILD.SOURCESDIRECTORY)\src\System.Windows.Forms\tests\UnitTests\System.Windows.Forms.Tests.csproj;$(BUILD.SOURCESDIRECTORY)\src\System.Windows.Forms.Design\tests\UnitTests\System.Windows.Forms.Design.Tests.csproj;$(BUILD.SOURCESDIRECTORY)\src\System.Windows.Forms.Design.Editors\tests\UnitTests\System.Windows.Forms.Design.Editors.Tests.csproj'
+ DisplayNamePrefix: 'Send Unit Tests (Windows.10) to Helix'
+
  # Run integration tests and rename binlog
+ # These are not ready to run on Helix because:
+ # 1. Helix doesn't build projects (it only uses publish); we need .dotnet and also bin from build
+ # 2. we need an interactive pool for SendKeys to work
  - script: eng\cibuild.cmd
  -configuration $(_BuildConfig)
  $(_OfficialBuildIdArgs)
@@ -150,4 +159,4 @@ jobs:
  # Run component governance detection (only for release; only for internal)
  - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
  displayName: Component Governance Detection
- condition: and(succeeded(), eq(variables['_BuildConfig'], 'Release'))
+ condition: and(succeeded(), eq(variables['_BuildConfig'], 'Release'))

eng/common/templates/steps/send-to-helix.yml

@@ -1,32 +1,33 @@
 parameters:
- HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/
- HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/'
- HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number
- HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/api/2018-03-14/info/queues for a list of queues
- HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group
- HelixPreCommands: '' # optional -- commands to run before Helix work item execution
- HelixPostCommands: '' # optional -- commands to run after Helix work item execution
- WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects
- WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects
- WorkItemTimeout: '' # optional -- a timeout in seconds for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects
- CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload
- XUnitProjects: '' # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true
- XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects
- XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner
- XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
- IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
- DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
- DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
- EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control
- WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
- IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set
- Creator: '' # optional -- if the build is external, use this to specify who is sending the job
- condition: succeeded() # optional -- condition for step to execute; defaults to succeeded()
- continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false
+ HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/
+ HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/'
+ HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number
+ HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/api/2018-03-14/info/queues for a list of queues
+ HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group
+ HelixPreCommands: '' # optional -- commands to run before Helix work item execution
+ HelixPostCommands: '' # optional -- commands to run after Helix work item execution
+ WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects
+ WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects
+ WorkItemTimeout: '' # optional -- a timeout in seconds for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects
+ CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload
+ XUnitProjects: '' # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true
+ XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects
+ XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner
+ XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
+ IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
+ DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
+ DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
+ EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control
+ WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
+ IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set
+ Creator: '' # optional -- if the build is external, use this to specify who is sending the job
+ DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO 
+ condition: succeeded() # optional -- condition for step to execute; defaults to succeeded()
+ continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false
 
 steps:
  - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"'
- displayName: Send job to Helix (Windows)
+ displayName: ${{ parameters.DisplayNamePrefix }} (Windows)
  env:
  BuildConfig: $(_BuildConfig)
  HelixSource: ${{ parameters.HelixSource }}
@@ -54,7 +55,7 @@ steps:
  condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT'))
  continueOnError: ${{ parameters.continueOnError }}
  - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog
- displayName: Send job to Helix (Unix)
+ displayName: ${{ parameters.DisplayNamePrefix }} (Unix)
  env:
  BuildConfig: $(_BuildConfig)
  HelixSource: ${{ parameters.HelixSource }}

global.json

@@ -9,6 +9,7 @@
  "version": "3.0.100-preview-010024"
  },
  "msbuild-sdks": {
- "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19156.20"
+ "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19156.20",
+ "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19156.20"
  }
 }

src/Common/tests/InternalUtilitiesForTests/InternalUtilitiesForTests.csproj

@@ -10,7 +10,7 @@
  <ItemGroup>
  <PackageReference Include="Moq" Version="$(MoqPackageVersion)" />
  <PackageReference Include="xunit.stafact" Version="$(XUnitStaFactPackageVersion)" />
- <PackageReference Include="Microsoft.DotNet.XUnitExtensions" Version="$(MicrosoftDotNetXUnitExtensionsPackageVersion)" />
+ <PackageReference Include="Microsoft.DotNet.XUnitExtensions" Version="$(MicrosoftDotNetXUnitExtensionsPackageVersion)" />
  <PackageReference Include="Microsoft.Win32.Registry" Version="$(MicrosoftWin32RegistryPackageVersion)" />
  </ItemGroup>
 </Project>

src/System.Windows.Forms.Design.Editors/tests/UnitTests/System.Windows.Forms.Design.Editors.Tests.csproj

@@ -5,8 +5,8 @@
  <AssemblyName>System.Windows.Forms.Design.Editors.Tests</AssemblyName>
  </PropertyGroup>
 
- <ItemGroup>
- <PackageReference Include="Moq" Version="$(MoqPackageVersion)" />
+ <ItemGroup>
+ <PackageReference Include="Moq" Version="$(MoqPackageVersion)" />
  <PackageReference Include="xunit.stafact" Version="$(XUnitStaFactPackageVersion)" />
  </ItemGroup>
 

src/System.Windows.Forms.Design/tests/UnitTests/System.Windows.Forms.Design.Tests.csproj

@@ -6,6 +6,11 @@
  <LangVersion>7.3</LangVersion>
  </PropertyGroup>
 
+ <ItemGroup>
+ <PackageReference Include="Moq" Version="$(MoqPackageVersion)" />
+ <PackageReference Include="xunit.stafact" Version="$(XUnitStaFactPackageVersion)" />
+ </ItemGroup>
+
  <ItemGroup>
  <ProjectReference Include="..\..\src\System.Windows.Forms.Design.csproj" />
  <Compile Include="..\..\..\Common\tests\CommonTestHelper.cs" Link="Common\CommonTestHelper.cs" />

src/System.Windows.Forms/tests/IntegrationTests/System.Windows.Forms.IntegrationTests/System.Windows.Forms.IntegrationTests.csproj

@@ -13,6 +13,7 @@
 
  <ItemGroup>
  <ProjectReference Include="..\..\..\src\System.Windows.Forms.csproj" />
+ <ProjectReference Include="..\WinformsControlsTest\WinformsControlsTest.csproj" />
  </ItemGroup>
 
 </Project>

src/System.Windows.Forms/tests/IntegrationTests/System.Windows.Forms.IntegrationTests/TestHelpers.cs

@@ -34,15 +34,15 @@ public static Process StartProcess(string byPathFromBinToExe)
  throw new ArgumentException(nameof(byPathFromBinToExe) + " must end in a .exe");
  }
 
- var dotnetPath = DotNetPath();
- if (!Directory.Exists(dotnetPath))
- {
- throw new DirectoryNotFoundException(dotnetPath + " directory cannot be found.");
+ var dotnetPath = DotNetPath();
+ if (!Directory.Exists(dotnetPath))
+ {
+ throw new DirectoryNotFoundException(dotnetPath + " directory cannot be found.");
  }
 
  ProcessStartInfo startInfo = new ProcessStartInfo();
  startInfo.FileName = Path.Combine(BinPath(), byPathFromBinToExe.Trim('\\'));
- startInfo.EnvironmentVariables["DOTNET_ROOT"] = dotnetPath;
+ startInfo.EnvironmentVariables["DOTNET_ROOT"] = dotnetPath;	// required
  // ...
 
  return StartProcess(startInfo);