From d12363c952a58b261ae80e4e21bd9bb4fe3037bd Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Tue, 30 Jun 2026 15:47:47 -0500 Subject: [PATCH 1/2] [xaprepare] Move cgmanifest.json generation from xaprepare to YAML Continues the incremental removal of `xaprepare` by moving the `cgmanifest.json` generator out of the C# step `Step_GenerateCGManifest` and into a YAML `pwsh:` step that runs only on CI. The file is consumed exclusively by the Azure DevOps Component Governance Detection task (auto-injected on internal pipelines by `eng/common/core-templates/job/job.yml`). It is not consumed by any in-repo build target, so generating it from the C# Prepare step is no longer necessary. We cannot drop the file outright: `eng/common` ships CG infrastructure but it is not yet wired to auto-discover submodules in our pipelines, so the explicit `cgmanifest.json` is still the source of truth. Files: * Added: `build-tools/automation/yaml-templates/generate-cgmanifest.yaml` * Modified: `build-windows-steps.yaml`, `build-macos-steps.yaml`, `build-linux-steps.yaml`, `commercial-build.yaml` (call the new template after the Prepare/`make jenkins` step) * Modified: `Scenario_Standard.cs` (drop registration) * Modified: `GitRunner.cs` (remove now-unused `ConfigList` and `SubmoduleStatus` methods) * Deleted: `Step_GenerateCGManifest.cs` Verification: byte-for-byte diff of the YAML-generated `cgmanifest.json` against the previous C# output on Debug: Baseline=3144 New=3144 Total byte differences: 0 Precedent PRs in this stream: #11568, #11580, #11608, #11613, #11631, #11731, #11732, #11733, #11737, #11740, #11760, #11803, #11821. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../yaml-templates/build-linux-steps.yaml | 5 + .../yaml-templates/build-macos-steps.yaml | 5 + .../yaml-templates/build-windows-steps.yaml | 4 + .../yaml-templates/commercial-build.yaml | 5 + .../yaml-templates/generate-cgmanifest.yaml | 124 ++++++++++++++ .../xaprepare/Scenarios/Scenario_Standard.cs | 1 - .../Steps/Step_GenerateCGManifest.cs | 160 ------------------ .../xaprepare/ToolRunners/GitRunner.cs | 50 ------ 8 files changed, 143 insertions(+), 211 deletions(-) create mode 100644 build-tools/automation/yaml-templates/generate-cgmanifest.yaml delete mode 100644 build-tools/xaprepare/xaprepare/Steps/Step_GenerateCGManifest.cs diff --git a/build-tools/automation/yaml-templates/build-linux-steps.yaml b/build-tools/automation/yaml-templates/build-linux-steps.yaml index c85d5d7d021..8794026f1f2 100644 --- a/build-tools/automation/yaml-templates/build-linux-steps.yaml +++ b/build-tools/automation/yaml-templates/build-linux-steps.yaml @@ -47,6 +47,11 @@ steps: displayName: make jenkins retryCountOnTaskFailure: 1 +- template: /build-tools/automation/yaml-templates/generate-cgmanifest.yaml + parameters: + configuration: $(XA.Build.Configuration) + xaSourcePath: ${{ parameters.xaSourcePath }} + - script: make create-nupkgs CONFIGURATION=$(XA.Build.Configuration) workingDirectory: ${{ parameters.xaSourcePath }} displayName: make create-nupkgs diff --git a/build-tools/automation/yaml-templates/build-macos-steps.yaml b/build-tools/automation/yaml-templates/build-macos-steps.yaml index edcaffbee53..53aa5042eda 100644 --- a/build-tools/automation/yaml-templates/build-macos-steps.yaml +++ b/build-tools/automation/yaml-templates/build-macos-steps.yaml @@ -50,6 +50,11 @@ steps: displayName: make jenkins retryCountOnTaskFailure: 1 +- template: /build-tools/automation/yaml-templates/generate-cgmanifest.yaml + parameters: + configuration: $(XA.Build.Configuration) + xaSourcePath: ${{ parameters.xaSourcePath }} + - script: make create-installers CONFIGURATION=$(XA.Build.Configuration) MSBUILD_ARGS='${{ parameters.makeMSBuildArgs }}' workingDirectory: ${{ parameters.xaSourcePath }} displayName: make create-installers diff --git a/build-tools/automation/yaml-templates/build-windows-steps.yaml b/build-tools/automation/yaml-templates/build-windows-steps.yaml index e3b83ed4de8..26c3d3100f7 100644 --- a/build-tools/automation/yaml-templates/build-windows-steps.yaml +++ b/build-tools/automation/yaml-templates/build-windows-steps.yaml @@ -38,6 +38,10 @@ steps: projects: Xamarin.Android.sln arguments: '-c $(XA.Build.Configuration) -t:Prepare --no-restore -bl:$(System.DefaultWorkingDirectory)\bin\Build$(XA.Build.Configuration)\dotnet-build-prepare.binlog' +- template: /build-tools/automation/yaml-templates/generate-cgmanifest.yaml + parameters: + configuration: $(XA.Build.Configuration) + # Build Xamarin.Android and configure local workloads to test improved local build loop - template: /build-tools/automation/yaml-templates/run-dotnet-preview.yaml parameters: diff --git a/build-tools/automation/yaml-templates/commercial-build.yaml b/build-tools/automation/yaml-templates/commercial-build.yaml index 94ccd2ff676..dcf040f8e1d 100644 --- a/build-tools/automation/yaml-templates/commercial-build.yaml +++ b/build-tools/automation/yaml-templates/commercial-build.yaml @@ -35,6 +35,11 @@ steps: workingDirectory: ${{ parameters.xaSourcePath }} displayName: make jenkins +- template: /build-tools/automation/yaml-templates/generate-cgmanifest.yaml + parameters: + configuration: $(XA.Build.Configuration) + xaSourcePath: ${{ parameters.xaSourcePath }} + - task: CodeQL3000Finalize@0 displayName: CodeQL 3000 Finalize condition: and(succeededOrFailed(), eq(variables['Codeql.Enabled'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) diff --git a/build-tools/automation/yaml-templates/generate-cgmanifest.yaml b/build-tools/automation/yaml-templates/generate-cgmanifest.yaml new file mode 100644 index 00000000000..dae769b0689 --- /dev/null +++ b/build-tools/automation/yaml-templates/generate-cgmanifest.yaml @@ -0,0 +1,124 @@ +# Generates bin/Build$(Configuration)/cgmanifest.json from the repository's +# git submodules. The file is consumed by the Azure DevOps Component +# Governance Detection task (auto-injected on internal pipelines by +# eng/common/core-templates/job/job.yml). This replaces the previous +# xaprepare Step_GenerateCGManifest step. + +parameters: + configuration: $(XA.Build.Configuration) + xaSourcePath: $(System.DefaultWorkingDirectory) + +steps: +- pwsh: | + $ErrorActionPreference = 'Stop' + Set-Location -LiteralPath '${{ parameters.xaSourcePath }}' + + # Parse .gitmodules entries that live under external/ (matches the + # previous C# behavior which filtered on "submodule.external/"). + $configLines = & git config --blob HEAD:.gitmodules --list + if ($LASTEXITCODE -ne 0) { + throw "git config --list failed with exit code $LASTEXITCODE" + } + + $statusLines = & git submodule status + if ($LASTEXITCODE -ne 0) { + throw "git submodule status failed with exit code $LASTEXITCODE" + } + + # Build a map of submodule id -> @{ Path; Url } + $prefix = 'submodule.external/' + $entries = [ordered]@{} + foreach ($line in $configLines) { + if (-not $line.StartsWith($prefix)) { continue } + $eq = $line.IndexOf('=') + if ($eq -lt 0) { continue } + $lastDot = $line.LastIndexOf('.', $eq) + if ($lastDot -lt 0) { continue } + $id = $line.Substring($prefix.Length, $lastDot - $prefix.Length) + $key = $line.Substring($lastDot, $eq - $lastDot) + $value = $line.Substring($eq + 1) + if (-not $entries.Contains($id)) { + $entries[$id] = @{ Path = $null; Url = $null } + } + if ($key -eq '.path') { + $entries[$id].Path = $value + } elseif ($key -eq '.url') { + # Strip trailing ".git" to match the previous C# behavior. + if ($value.EndsWith('.git')) { + $value = $value.Substring(0, $value.Length - 4) + } + $entries[$id].Url = $value + } + } + + # Map submodule path -> commit hash via `git submodule status` output. + # Status lines are " [ (...)]", where is + # ' ', '-', or '+'. The previous C# matched the path inside the + # status line and extracted SHA between index 1 and the path; we do + # the same with a regex to match byte-for-byte. + $hashByPath = @{} + foreach ($line in $statusLines) { + if ([string]::IsNullOrEmpty($line)) { continue } + if ($line -match '^.([0-9a-fA-F]{40})\s+(\S+)') { + $hashByPath[$matches[2]] = $matches[1] + } + } + + # Build registration objects, then sort by repositoryUrl + # (ordinal, case-insensitive) to match Step_GenerateCGManifest. + $registrations = @() + foreach ($id in $entries.Keys) { + $path = $entries[$id].Path + $url = $entries[$id].Url + if ($null -eq $path -or $null -eq $url) { continue } + $hash = '' + if ($hashByPath.ContainsKey($path)) { + $hash = $hashByPath[$path] + } + $registrations += [pscustomobject]@{ Url = $url; Hash = $hash } + } + $registrations = $registrations | + Sort-Object -Property @{ Expression = { $_.Url.ToLowerInvariant() } } + + # Hand-write the JSON to byte-match Step_GenerateCGManifest.Execute + # (4-space indent, trailing newline after the closing brace). + $nl = [Environment]::NewLine + $sb = [System.Text.StringBuilder]::new() + [void]$sb.Append('{').Append($nl) + [void]$sb.Append(' "$schema": "https://json.schemastore.org/component-detection-manifest.json",').Append($nl) + [void]$sb.Append(' "version": 1,').Append($nl) + [void]$sb.Append(' "registrations": [').Append($nl) + + $first = $true + foreach ($entry in $registrations) { + if ($first) { + $first = $false + } else { + [void]$sb.Append(',').Append($nl) + } + [void]$sb.Append(' {').Append($nl) + [void]$sb.Append(' "component": {').Append($nl) + [void]$sb.Append(' "type": "git",').Append($nl) + [void]$sb.Append(' "git": {').Append($nl) + [void]$sb.Append(' "commitHash": "').Append($entry.Hash).Append('",').Append($nl) + [void]$sb.Append(' "repositoryUrl": "').Append($entry.Url).Append('"').Append($nl) + [void]$sb.Append(' }').Append($nl) + [void]$sb.Append(' }').Append($nl) + [void]$sb.Append(' }') + } + + [void]$sb.Append($nl) + [void]$sb.Append(' ]').Append($nl) + [void]$sb.Append('}').Append($nl) + + $outDir = Join-Path '${{ parameters.xaSourcePath }}' "bin/Build${{ parameters.configuration }}" + New-Item -ItemType Directory -Path $outDir -Force | Out-Null + $outPath = Join-Path $outDir 'cgmanifest.json' + + # Write UTF-8 without BOM to match the previous File.CreateText output. + $utf8NoBom = [System.Text.UTF8Encoding]::new($false) + [System.IO.File]::WriteAllText($outPath, $sb.ToString(), $utf8NoBom) + + Write-Host "Wrote $outPath with $($registrations.Count) registration(s)." + displayName: Generate cgmanifest.json + continueOnError: false diff --git a/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs b/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs index 06cfc3086df..5947f2f0d2b 100644 --- a/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs +++ b/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs @@ -19,7 +19,6 @@ protected override void AddSteps (Context context) throw new ArgumentNullException (nameof (context)); Steps.Add (new Step_GenerateFiles (atBuildStart: true)); - Steps.Add (new Step_GenerateCGManifest ()); } protected override void AddEndSteps (Context context) diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_GenerateCGManifest.cs b/build-tools/xaprepare/xaprepare/Steps/Step_GenerateCGManifest.cs deleted file mode 100644 index 53dd72a13e6..00000000000 --- a/build-tools/xaprepare/xaprepare/Steps/Step_GenerateCGManifest.cs +++ /dev/null @@ -1,160 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; - -namespace Xamarin.Android.Prepare -{ - partial class Step_GenerateCGManifest : Step - { - public Step_GenerateCGManifest () - : base ("Generate cgmanifest.json") - {} - - protected override async Task Execute (Context context) - { - var git = new GitRunner (context); - var gitSubmoduleInfo = await git.ConfigList (new[]{"--blob", "HEAD:.gitmodules"}); - var gitSubmoduleStatus = await git.SubmoduleStatus (); - var gitSubmodules = GitSubmoduleInfo.GetGitSubmodules (gitSubmoduleInfo, gitSubmoduleStatus) - .OrderBy (e => e.RepositoryUrl, StringComparer.OrdinalIgnoreCase); - - var jsonPath = Path.Combine (Configurables.Paths.BuildBinDir, "cgmanifest.json"); - using var json = File.CreateText (jsonPath); - - json.WriteLine ("{"); - json.WriteLine (" \"$schema\": \"https://json.schemastore.org/component-detection-manifest.json\","); - json.WriteLine (" \"version\": 1,"); - json.WriteLine (" \"registrations\": ["); - - bool first = true; - - foreach (var entry in gitSubmodules) { - if (first) { - first = false; - } else { - json.WriteLine (","); - } - - json.WriteLine ($" {{"); - json.WriteLine ($" \"component\": {{"); - json.WriteLine ($" \"type\": \"git\","); - json.WriteLine ($" \"git\": {{"); - json.WriteLine ($" \"commitHash\": \"{entry.CommitHash}\","); - json.WriteLine ($" \"repositoryUrl\": \"{entry.RepositoryUrl}\""); - json.WriteLine ($" }}"); - json.WriteLine ($" }}"); - json.Write ($" }}"); - } - - json.WriteLine (); - json.WriteLine (" ]"); - json.WriteLine ("}"); - - return true; - } - } - - sealed class GitSubmoduleInfo - { - public string Name { - get { - const string github = "github.com/"; - int i = RepositoryUrl.IndexOf (github, StringComparison.OrdinalIgnoreCase); - if (i >= 0) - return RepositoryUrl.Substring (i + github.Length); - return RepositoryUrl; - } - } - - public string RepositoryUrl { get; private set; } = String.Empty; - public string CommitHash { get; private set; } = String.Empty; - public string LocalPath { get; private set; } = String.Empty; - - GitSubmoduleInfo () - { - } - - const string Submodule = "submodule.external/"; - - public static IEnumerable GetGitSubmodules (List? config, List? submoduleStatus) - { - if (config == null) { - yield break; - } - - string? entryId = null; - string? path = null; - string? url = null; - - foreach (var line in config) { - if (!line.StartsWith (Submodule, StringComparison.Ordinal)) - continue; - - string? id = GetSubmoduleId (line); - if (id != entryId) { - if (url != null && path != null) - yield return CreateSubmoduleInfo (url, path, submoduleStatus); - - entryId = id; - path = null; - url = null; - } - - const string Path = ".path="; - const string Url = ".url="; - const string Git = ".git"; - - int pathIndex = line.IndexOf (Path, StringComparison.Ordinal); - if (pathIndex > 0) { - path = line.Substring (pathIndex + Path.Length); - continue; - } - - int urlIndex = line.IndexOf (Url, StringComparison.Ordinal); - if (urlIndex > 0) { - int start = urlIndex + Url.Length; - int count = line.Length - start; - if (line.EndsWith (Git, StringComparison.Ordinal)) - count -= Git.Length; - url = line.Substring (start, count); - continue; - } - } - - if (url != null && path != null) - yield return CreateSubmoduleInfo (url, path, submoduleStatus); - } - - static GitSubmoduleInfo CreateSubmoduleInfo (string url, string path, List? submoduleStatus) - { - string commitHash = String.Empty; - - if (submoduleStatus != null) { - foreach (var e in submoduleStatus) { - int pi = e.IndexOf (path, StringComparison.OrdinalIgnoreCase); - if (pi < 1 || e [pi - 1] != ' ') - continue; - commitHash = e.Substring (1, pi - 2); - break; - } - } - - return new GitSubmoduleInfo { - LocalPath = path, - RepositoryUrl = url, - CommitHash = commitHash, - }; - } - - static string? GetSubmoduleId (string line) - { - int eq = line.IndexOf ('='); - if (eq < 0) - return null; - int lastDot = line.LastIndexOf ('.', eq); - return line.Substring (Submodule.Length, lastDot - Submodule.Length); - } - } -} diff --git a/build-tools/xaprepare/xaprepare/ToolRunners/GitRunner.cs b/build-tools/xaprepare/xaprepare/ToolRunners/GitRunner.cs index 1fe8f8cf149..214956f34fe 100644 --- a/build-tools/xaprepare/xaprepare/ToolRunners/GitRunner.cs +++ b/build-tools/xaprepare/xaprepare/ToolRunners/GitRunner.cs @@ -95,31 +95,6 @@ public async Task Clone (string url, string destinationDirectoryPath) return await RunGit (runner, $"clone-{dirName}"); } - public async Task?> SubmoduleStatus (string? workingDirectory = null) - { - string runnerWorkingDirectory = DetermineRunnerWorkingDirectory (workingDirectory); - - var runner = CreateGitRunner (runnerWorkingDirectory);; - runner.AddArgument ("submodule"); - runner.AddArgument ("status"); - - var lines = new List (); - - bool success = await RunTool ( - () => { - using (var outputSink = (OutputSink)SetupOutputSink (runner)) { - outputSink.LineCallback = (string? line) => lines.Add (line ?? String.Empty); - return runner.Run (); - } - } - ); - - if (!success) - return null; - - return lines; - } - public async Task SubmoduleUpdate (string? workingDirectory = null, bool init = true, bool recursive = true) { string runnerWorkingDirectory = DetermineRunnerWorkingDirectory (workingDirectory); @@ -229,31 +204,6 @@ public string GetTopCommitHash (string? workingDirectory = null, bool shortHash return parserState.Entries; } - public async Task?> ConfigList (string[] fileOptions, string? workingDirectory = null) - { - var runner = CreateGitRunner (workingDirectory); - runner.AddArgument ("config"); - foreach (var opt in fileOptions) - runner.AddArgument (opt); - runner.AddArgument ("--list"); - - var lines = new List (); - - bool success = await RunTool ( - () => { - using (var outputSink = (OutputSink)SetupOutputSink (runner)) { - outputSink.LineCallback = (string? line) => lines.Add (line ?? String.Empty); - return runner.Run (); - } - } - ); - - if (!success) - return null; - - return lines; - } - public async Task IsRepoUrlHttps (string workingDirectory) { if (!Directory.Exists (workingDirectory)) From e1d6492940135dec69b61d78b13ca2bc411c658b Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Tue, 30 Jun 2026 16:01:04 -0500 Subject: [PATCH 2/2] Simplify generate-cgmanifest.yaml using ConvertTo-Json Replace the hand-rolled StringBuilder JSON emission with PowerShell's `ConvertTo-Json`. The output is no longer byte-identical to the previous C# output (2-space indent instead of 4, and slightly different whitespace), but it is still valid JSON matching the component-detection-manifest schema. The Azure DevOps Component Governance Detection task parses the file, so formatting does not matter. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../yaml-templates/generate-cgmanifest.yaml | 61 +++++++------------ 1 file changed, 23 insertions(+), 38 deletions(-) diff --git a/build-tools/automation/yaml-templates/generate-cgmanifest.yaml b/build-tools/automation/yaml-templates/generate-cgmanifest.yaml index dae769b0689..24d19b6be5f 100644 --- a/build-tools/automation/yaml-templates/generate-cgmanifest.yaml +++ b/build-tools/automation/yaml-templates/generate-cgmanifest.yaml @@ -53,9 +53,7 @@ steps: # Map submodule path -> commit hash via `git submodule status` output. # Status lines are " [ (...)]", where is - # ' ', '-', or '+'. The previous C# matched the path inside the - # status line and extracted SHA between index 1 and the path; we do - # the same with a regex to match byte-for-byte. + # ' ', '-', or '+'. $hashByPath = @{} foreach ($line in $statusLines) { if ([string]::IsNullOrEmpty($line)) { continue } @@ -64,8 +62,8 @@ steps: } } - # Build registration objects, then sort by repositoryUrl - # (ordinal, case-insensitive) to match Step_GenerateCGManifest. + # Build registration objects, sorted by repositoryUrl (ordinal, + # case-insensitive) to keep the output stable across runs. $registrations = @() foreach ($id in $entries.Keys) { $path = $entries[$id].Path @@ -77,47 +75,34 @@ steps: } $registrations += [pscustomobject]@{ Url = $url; Hash = $hash } } - $registrations = $registrations | - Sort-Object -Property @{ Expression = { $_.Url.ToLowerInvariant() } } - - # Hand-write the JSON to byte-match Step_GenerateCGManifest.Execute - # (4-space indent, trailing newline after the closing brace). - $nl = [Environment]::NewLine - $sb = [System.Text.StringBuilder]::new() - [void]$sb.Append('{').Append($nl) - [void]$sb.Append(' "$schema": "https://json.schemastore.org/component-detection-manifest.json",').Append($nl) - [void]$sb.Append(' "version": 1,').Append($nl) - [void]$sb.Append(' "registrations": [').Append($nl) + $registrations = @($registrations | + Sort-Object -Property @{ Expression = { $_.Url.ToLowerInvariant() } } | + ForEach-Object { + [ordered]@{ + component = [ordered]@{ + type = 'git' + git = [ordered]@{ + commitHash = $_.Hash + repositoryUrl = $_.Url + } + } + } + }) - $first = $true - foreach ($entry in $registrations) { - if ($first) { - $first = $false - } else { - [void]$sb.Append(',').Append($nl) - } - [void]$sb.Append(' {').Append($nl) - [void]$sb.Append(' "component": {').Append($nl) - [void]$sb.Append(' "type": "git",').Append($nl) - [void]$sb.Append(' "git": {').Append($nl) - [void]$sb.Append(' "commitHash": "').Append($entry.Hash).Append('",').Append($nl) - [void]$sb.Append(' "repositoryUrl": "').Append($entry.Url).Append('"').Append($nl) - [void]$sb.Append(' }').Append($nl) - [void]$sb.Append(' }').Append($nl) - [void]$sb.Append(' }') + $doc = [ordered]@{ + '$schema' = 'https://json.schemastore.org/component-detection-manifest.json' + version = 1 + registrations = $registrations } - - [void]$sb.Append($nl) - [void]$sb.Append(' ]').Append($nl) - [void]$sb.Append('}').Append($nl) + $json = $doc | ConvertTo-Json -Depth 10 $outDir = Join-Path '${{ parameters.xaSourcePath }}' "bin/Build${{ parameters.configuration }}" New-Item -ItemType Directory -Path $outDir -Force | Out-Null $outPath = Join-Path $outDir 'cgmanifest.json' - # Write UTF-8 without BOM to match the previous File.CreateText output. + # Write UTF-8 without BOM. $utf8NoBom = [System.Text.UTF8Encoding]::new($false) - [System.IO.File]::WriteAllText($outPath, $sb.ToString(), $utf8NoBom) + [System.IO.File]::WriteAllText($outPath, $json, $utf8NoBom) Write-Host "Wrote $outPath with $($registrations.Count) registration(s)." displayName: Generate cgmanifest.json