diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 89ef6eae9eb6..73d5b5f2eef2 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -19,13 +19,13 @@ - + https://github.com/dotnet/arcade - 0992a0f871c48ccc32eaa49ad89fea8022b8f597 + 25da4e8f08fab05a39c27443df99b96d738f3c82 - + https://github.com/dotnet/arcade - 0992a0f871c48ccc32eaa49ad89fea8022b8f597 + 25da4e8f08fab05a39c27443df99b96d738f3c82 diff --git a/eng/Versions.props b/eng/Versions.props index 5c4a79e0b988..5af6c110290d 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -8,7 +8,7 @@ - 8.0.0-beta.26410.9 + 8.0.0-beta.26431.7 8.0.0-rtm.26281.3 8.0.0-rtm.26281.3 diff --git a/eng/common/Get-GitHubAppToken.ps1 b/eng/common/Get-GitHubAppToken.ps1 index 648035a3ecfc..ea776bd6bc28 100644 --- a/eng/common/Get-GitHubAppToken.ps1 +++ b/eng/common/Get-GitHubAppToken.ps1 @@ -72,25 +72,34 @@ $digestBytes = $sha256.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($signi $digestBase64 = [Convert]::ToBase64String($digestBytes) Write-Host "Signing JWT with key '$KeyName' in vault '$KeyVaultName'..." +$previousNativeCommandErrorPreference = $PSNativeCommandUseErrorActionPreference try { - $signatureUrl = az keyvault key sign ` + # Azure CLI can emit non-fatal Python warnings to stderr even when signing succeeds. + # Use the exit code to determine success for this invocation. + $PSNativeCommandUseErrorActionPreference = $false + $signatureBase64 = az keyvault key sign ` --vault-name $KeyVaultName ` --name $KeyName ` --algorithm RS256 ` --digest $digestBase64 ` - --query value ` + --query signature ` --output tsv ` --only-show-errors + $signExitCode = $LASTEXITCODE } catch { Write-PipelineTelemetryError -Category 'Build' -Message "Failed to sign the JWT via Key Vault (key '$KeyName', vault '$KeyVaultName'): $_. Verify the service connection identity has the 'Key Vault Crypto User' role (Sign action) on the key." exit 1 } -if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($signatureUrl)) { - Write-PipelineTelemetryError -Category 'Build' -Message "'az keyvault key sign' exited with code $LASTEXITCODE for key '$KeyName' in vault '$KeyVaultName'. Verify the service connection identity has the 'Key Vault Crypto User' role (Sign action) on the key." +finally { + $PSNativeCommandUseErrorActionPreference = $previousNativeCommandErrorPreference +} +if ($signExitCode -ne 0 -or [string]::IsNullOrWhiteSpace($signatureBase64)) { + Write-PipelineTelemetryError -Category 'Build' -Message "'az keyvault key sign' exited with code $signExitCode for key '$KeyName' in vault '$KeyVaultName'. Verify the service connection identity has the 'Key Vault Crypto User' role (Sign action) on the key." exit 1 } -$jwt = "$signingInput.$($signatureUrl.Trim())" +$signatureUrl = $signatureBase64.Trim().TrimEnd('=').Replace('+', '-').Replace('/', '_') +$jwt = "$signingInput.$signatureUrl" $headers = @{ Authorization = "Bearer $jwt" @@ -101,27 +110,38 @@ $headers = @{ Write-Host "Looking up installation for '$InstallationOwner'..." try { - $installations = @() + $installations = [System.Collections.Generic.List[object]]::new() $page = 1 do { - $pageInstallations = @(Invoke-RestMethod ` + $pageResponse = Invoke-RestMethod ` -Uri "https://api.github.com/app/installations?per_page=100&page=$page" ` -Headers $headers ` - -Method Get) - $installations += $pageInstallations + -Method Get + $pageInstallationCount = 0 + foreach ($installation in $pageResponse) { + $installations.Add($installation) + $pageInstallationCount++ + } $page++ - } while ($pageInstallations.Count -eq 100) + } while ($pageInstallationCount -eq 100) } catch { Write-PipelineTelemetryError -Category 'Build' -Message "Failed to list GitHub App installations: $_. The signed JWT may be invalid or the App's Client ID ('$AppClientId') may be incorrect." exit 1 } -$installation = $installations | Where-Object { $_.account.login -ieq $InstallationOwner } | Select-Object -First 1 -if (-not $installation) { +$matchingInstallations = @($installations | Where-Object { $_.account.login -ieq $InstallationOwner }) +if ($matchingInstallations.Count -eq 0) { $found = ($installations | ForEach-Object { $_.account.login }) -join ', ' Write-PipelineTelemetryError -Category 'Build' -Message "No installation found for '$InstallationOwner'. App is installed on: $found" exit 1 } +if ($matchingInstallations.Count -ne 1) { + $matchingIds = ($matchingInstallations | ForEach-Object { $_.id }) -join ', ' + Write-PipelineTelemetryError -Category 'Build' -Message "Found multiple installations for '$InstallationOwner': $matchingIds" + exit 1 +} +$installation = $matchingInstallations[0] +Write-Host "Using installation $($installation.id) for '$($installation.account.login)'." try { $tokenResponse = Invoke-RestMethod ` diff --git a/eng/common/sdl/extract-artifact-packages.ps1 b/eng/common/sdl/extract-artifact-packages.ps1 deleted file mode 100644 index f031ed5b25e9..000000000000 --- a/eng/common/sdl/extract-artifact-packages.ps1 +++ /dev/null @@ -1,82 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where artifact packages are stored - [Parameter(Mandatory=$true)][string] $ExtractPath # Full path to directory where the packages will be extracted -) - -$ErrorActionPreference = 'Stop' -Set-StrictMode -Version 2.0 - -$disableConfigureToolsetImport = $true - -function ExtractArtifacts { - if (!(Test-Path $InputPath)) { - Write-Host "Input Path does not exist: $InputPath" - ExitWithExitCode 0 - } - $Jobs = @() - Get-ChildItem "$InputPath\*.nupkg" | - ForEach-Object { - $Jobs += Start-Job -ScriptBlock $ExtractPackage -ArgumentList $_.FullName - } - - foreach ($Job in $Jobs) { - Wait-Job -Id $Job.Id | Receive-Job - } -} - -try { - # `tools.ps1` checks $ci to perform some actions. Since the SDL - # scripts don't necessarily execute in the same agent that run the - # build.ps1/sh script this variable isn't automatically set. - $ci = $true - . $PSScriptRoot\..\tools.ps1 - - $ExtractPackage = { - param( - [string] $PackagePath # Full path to a NuGet package - ) - - if (!(Test-Path $PackagePath)) { - Write-PipelineTelemetryError -Category 'Build' -Message "Input file does not exist: $PackagePath" - ExitWithExitCode 1 - } - - $RelevantExtensions = @('.dll', '.exe', '.pdb') - Write-Host -NoNewLine 'Extracting ' ([System.IO.Path]::GetFileName($PackagePath)) '...' - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId - - Add-Type -AssemblyName System.IO.Compression.FileSystem - - [System.IO.Directory]::CreateDirectory($ExtractPath); - - try { - $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) - - $zip.Entries | - Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | - ForEach-Object { - $TargetPath = Join-Path -Path $ExtractPath -ChildPath (Split-Path -Path $_.FullName) - [System.IO.Directory]::CreateDirectory($TargetPath); - - $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.FullName - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile) - } - } - catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 - } - finally { - $zip.Dispose() - } - } - Measure-Command { ExtractArtifacts } -} -catch { - Write-Host $_ - Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ - ExitWithExitCode 1 -} diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml index c103b0445a81..5840fe2fff85 100644 --- a/eng/common/templates-official/job/onelocbuild.yml +++ b/eng/common/templates-official/job/onelocbuild.yml @@ -9,11 +9,20 @@ parameters: GithubPat: $(BotAccount-dotnet-bot-repo-PAT) # Service connection for WIF-based Entra authentication to ceapex feeds (replaces CeapexPat). - # When set, dnceng/internal builds acquire a federated Entra token instead of using a PAT. - # All other projects (e.g. DevDiv, public), where this dnceng-scoped service connection does not - # exist, and any pipeline that sets this to '' fall back to PAT-based auth via the CeapexPat parameter. + # The internal and DevDiv projects each provide a project-scoped connection with this name. + # Other projects, and any pipeline that sets this to '', fall back to CeapexPat. CeapexServiceConnection: 'dnceng-onelocbuild-ceapex' + # GitHub App authentication for the OneLoc check-in PR. + # dnceng/internal and DevDiv/DevDiv are enabled by default with their project-scoped service + # connections. Other projects must explicitly opt in after provisioning equivalent infrastructure. + UseGitHubAppAuthentication: true + UseGitHubAppAuthenticationInOtherProjects: false + GitHubAppServiceConnection: 'dnceng-oneloc-githubapp' + GitHubAppClientId: 'Iv23lijBU8x3gc9lDOc9' + GitHubAppKeyVaultName: 'EngKeyVault' + GitHubAppKeyName: 'oneloc-localization-app-key' + SourcesDirectory: $(System.DefaultWorkingDirectory) CreatePr: true AutoCompletePr: false @@ -74,15 +83,30 @@ jobs: displayName: Generate LocProject.json condition: ${{ parameters.condition }} - # Acquire an Entra token for ceapex feed access via WIF (dnceng/internal only). - # All other projects use PAT-based auth, since the ceapex service connection is scoped to dnceng/internal. - - ${{ if and(ne(parameters.CeapexServiceConnection, ''), eq(variables['System.TeamProject'], 'internal')) }}: + # Acquire an Entra token for ceapex feed access in the supported internal and DevDiv projects. + - ${{ if and(ne(parameters.CeapexServiceConnection, ''), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'))) }}: - template: /eng/common/templates-official/steps/get-federated-access-token.yml parameters: federatedServiceConnection: ${{ parameters.CeapexServiceConnection }} outputVariableName: 'CeapexEntraToken' condition: ${{ parameters.condition }} + # Mint a short-lived GitHub App installation token for the loc check-in PR. Use the connection + # provisioned in each supported project; other projects must explicitly opt in and override it. + - ${{ if and(eq(parameters.RepoType, 'gitHub'), eq(parameters.UseGitHubAppAuthentication, true), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.UseGitHubAppAuthenticationInOtherProjects, true))) }}: + - template: /eng/common/templates-official/steps/get-github-app-token.yml + parameters: + ${{ if and(eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.GitHubAppServiceConnection, 'dnceng-oneloc-githubapp')) }}: + azureSubscription: 'devdiv-oneloc-githubapp' + ${{ else }}: + azureSubscription: ${{ parameters.GitHubAppServiceConnection }} + keyVaultName: ${{ parameters.GitHubAppKeyVaultName }} + keyName: ${{ parameters.GitHubAppKeyName }} + appClientId: ${{ parameters.GitHubAppClientId }} + installationOwner: ${{ parameters.GitHubOrg }} + outputVariableName: 'GitHubAppInstallationToken' + condition: ${{ parameters.condition }} + - task: OneLocBuild@2 displayName: OneLocBuild env: @@ -99,13 +123,16 @@ jobs: ${{ if eq(parameters.RepoType, 'gitHub') }}: isShouldReusePrSelected: ${{ parameters.ReusePr }} packageSourceAuth: patAuth - ${{ if and(ne(parameters.CeapexServiceConnection, ''), eq(variables['System.TeamProject'], 'internal')) }}: + ${{ if and(ne(parameters.CeapexServiceConnection, ''), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'))) }}: patVariable: $(CeapexEntraToken) - ${{ if or(eq(parameters.CeapexServiceConnection, ''), ne(variables['System.TeamProject'], 'internal')) }}: + ${{ if or(eq(parameters.CeapexServiceConnection, ''), and(ne(variables['System.TeamProject'], 'internal'), ne(variables['System.TeamProject'], 'DevDiv'))) }}: patVariable: ${{ parameters.CeapexPat }} ${{ if eq(parameters.RepoType, 'gitHub') }}: repoType: ${{ parameters.RepoType }} - gitHubPatVariable: "${{ parameters.GithubPat }}" + ${{ if and(eq(parameters.UseGitHubAppAuthentication, true), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.UseGitHubAppAuthenticationInOtherProjects, true))) }}: + gitHubPatVariable: "$(GitHubAppInstallationToken)" + ${{ else }}: + gitHubPatVariable: "${{ parameters.GithubPat }}" ${{ if ne(parameters.MirrorRepo, '') }}: isMirrorRepoSelected: true gitHubOrganization: ${{ parameters.GitHubOrg }} diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml index acb64b39a693..1695b6dcf37f 100644 --- a/eng/common/templates-official/steps/source-build.yml +++ b/eng/common/templates-official/steps/source-build.yml @@ -20,13 +20,12 @@ steps: - script: | df -h - # If building on the internal project, the artifact feeds variable may be available (usually only if needed) - # In that case, call the feed setup script to add internal feeds corresponding to public ones. + # If building on the dnceng internal project, call the feed setup script to add internal feeds corresponding to public ones. # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those # changes. internalRestoreArgs= - if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then + if [ '${{ eq(variables['System.TeamProject'], 'internal') }}' = 'True' ]; then # Temporarily work around https://github.com/dotnet/arcade/issues/7709 chmod +x $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh # Authenticate to internal feeds using the build identity (System.AccessToken) diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 89f1b31bc55c..7152ff120adf 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -109,18 +109,6 @@ jobs: displayName: Trim the version from the NuGet packages continueOnError: ${{ parameters.sdlContinueOnError }} - - powershell: eng/common/sdl/extract-artifact-packages.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts - -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts - displayName: Extract Blob Artifacts - continueOnError: ${{ parameters.sdlContinueOnError }} - - - powershell: eng/common/sdl/extract-artifact-packages.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts - -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts - displayName: Extract Package Artifacts - continueOnError: ${{ parameters.sdlContinueOnError }} - - ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}: - powershell: eng/common/sdl/extract-artifact-archives.ps1 -InputPath $(Build.ArtifactStagingDirectory)\artifacts diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml index 77584083d3c8..1a8a07ca0168 100644 --- a/eng/common/templates/job/onelocbuild.yml +++ b/eng/common/templates/job/onelocbuild.yml @@ -9,11 +9,20 @@ parameters: GithubPat: $(BotAccount-dotnet-bot-repo-PAT) # Service connection for WIF-based Entra authentication to ceapex feeds (replaces CeapexPat). - # When set, dnceng/internal builds acquire a federated Entra token instead of using a PAT. - # All other projects (e.g. DevDiv, public), where this dnceng-scoped service connection does not - # exist, and any pipeline that sets this to '' fall back to PAT-based auth via the CeapexPat parameter. + # The internal and DevDiv projects each provide a project-scoped connection with this name. + # Other projects, and any pipeline that sets this to '', fall back to CeapexPat. CeapexServiceConnection: 'dnceng-onelocbuild-ceapex' + # GitHub App authentication for the OneLoc check-in PR. + # dnceng/internal and DevDiv/DevDiv are enabled by default with their project-scoped service + # connections. Other projects must explicitly opt in after provisioning equivalent infrastructure. + UseGitHubAppAuthentication: true + UseGitHubAppAuthenticationInOtherProjects: false + GitHubAppServiceConnection: 'dnceng-oneloc-githubapp' + GitHubAppClientId: 'Iv23lijBU8x3gc9lDOc9' + GitHubAppKeyVaultName: 'EngKeyVault' + GitHubAppKeyName: 'oneloc-localization-app-key' + SourcesDirectory: $(System.DefaultWorkingDirectory) CreatePr: true AutoCompletePr: false @@ -71,15 +80,30 @@ jobs: displayName: Generate LocProject.json condition: ${{ parameters.condition }} - # Acquire an Entra token for ceapex feed access via WIF (dnceng/internal only). - # All other projects use PAT-based auth, since the ceapex service connection is scoped to dnceng/internal. - - ${{ if and(ne(parameters.CeapexServiceConnection, ''), eq(variables['System.TeamProject'], 'internal')) }}: + # Acquire an Entra token for ceapex feed access in the supported internal and DevDiv projects. + - ${{ if and(ne(parameters.CeapexServiceConnection, ''), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'))) }}: - template: /eng/common/templates/steps/get-federated-access-token.yml parameters: federatedServiceConnection: ${{ parameters.CeapexServiceConnection }} outputVariableName: 'CeapexEntraToken' condition: ${{ parameters.condition }} + # Mint a short-lived GitHub App installation token for the loc check-in PR. Use the connection + # provisioned in each supported project; other projects must explicitly opt in and override it. + - ${{ if and(eq(parameters.RepoType, 'gitHub'), eq(parameters.UseGitHubAppAuthentication, true), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.UseGitHubAppAuthenticationInOtherProjects, true))) }}: + - template: /eng/common/templates/steps/get-github-app-token.yml + parameters: + ${{ if and(eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.GitHubAppServiceConnection, 'dnceng-oneloc-githubapp')) }}: + azureSubscription: 'devdiv-oneloc-githubapp' + ${{ else }}: + azureSubscription: ${{ parameters.GitHubAppServiceConnection }} + keyVaultName: ${{ parameters.GitHubAppKeyVaultName }} + keyName: ${{ parameters.GitHubAppKeyName }} + appClientId: ${{ parameters.GitHubAppClientId }} + installationOwner: ${{ parameters.GitHubOrg }} + outputVariableName: 'GitHubAppInstallationToken' + condition: ${{ parameters.condition }} + - task: OneLocBuild@2 displayName: OneLocBuild env: @@ -96,13 +120,16 @@ jobs: ${{ if eq(parameters.RepoType, 'gitHub') }}: isShouldReusePrSelected: ${{ parameters.ReusePr }} packageSourceAuth: patAuth - ${{ if and(ne(parameters.CeapexServiceConnection, ''), eq(variables['System.TeamProject'], 'internal')) }}: + ${{ if and(ne(parameters.CeapexServiceConnection, ''), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'))) }}: patVariable: $(CeapexEntraToken) - ${{ if or(eq(parameters.CeapexServiceConnection, ''), ne(variables['System.TeamProject'], 'internal')) }}: + ${{ if or(eq(parameters.CeapexServiceConnection, ''), and(ne(variables['System.TeamProject'], 'internal'), ne(variables['System.TeamProject'], 'DevDiv'))) }}: patVariable: ${{ parameters.CeapexPat }} ${{ if eq(parameters.RepoType, 'gitHub') }}: repoType: ${{ parameters.RepoType }} - gitHubPatVariable: "${{ parameters.GithubPat }}" + ${{ if and(eq(parameters.UseGitHubAppAuthentication, true), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.UseGitHubAppAuthenticationInOtherProjects, true))) }}: + gitHubPatVariable: "$(GitHubAppInstallationToken)" + ${{ else }}: + gitHubPatVariable: "${{ parameters.GithubPat }}" ${{ if ne(parameters.MirrorRepo, '') }}: isMirrorRepoSelected: true gitHubOrganization: ${{ parameters.GitHubOrg }} diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index 91da8aaee5f8..c1e517f38d02 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -20,13 +20,12 @@ steps: - script: | df -h - # If building on the internal project, the artifact feeds variable may be available (usually only if needed) - # In that case, call the feed setup script to add internal feeds corresponding to public ones. + # If building on the dnceng internal project, call the feed setup script to add internal feeds corresponding to public ones. # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those # changes. internalRestoreArgs= - if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then + if [ '${{ eq(variables['System.TeamProject'], 'internal') }}' = 'True' ]; then # Temporarily work around https://github.com/dotnet/arcade/issues/7709 chmod +x $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh # Authenticate to internal feeds using the build identity (System.AccessToken) diff --git a/global.json b/global.json index 9d27f1e21c99..4327c4c7856b 100644 --- a/global.json +++ b/global.json @@ -6,7 +6,7 @@ "dotnet": "8.0.126" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.26410.9", + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.26431.7", "Microsoft.Build.NoTargets": "3.5.0", "Microsoft.Build.Traversal": "2.0.34" }