diff --git a/build-tools/automation/yaml-templates/build-linux-steps.yaml b/build-tools/automation/yaml-templates/build-linux-steps.yaml index 6f4156976e9..d6469ef6eea 100644 --- a/build-tools/automation/yaml-templates/build-linux-steps.yaml +++ b/build-tools/automation/yaml-templates/build-linux-steps.yaml @@ -47,6 +47,11 @@ steps: displayName: make jenkins retryCountOnTaskFailure: 1 +- template: /build-tools/automation/yaml-templates/generate-cgmanifest.yaml + parameters: + configuration: $(XA.Build.Configuration) + xaSourcePath: ${{ parameters.xaSourcePath }} + - script: make create-nupkgs CONFIGURATION=$(XA.Build.Configuration) workingDirectory: ${{ parameters.xaSourcePath }} displayName: make create-nupkgs diff --git a/build-tools/automation/yaml-templates/build-macos-steps.yaml b/build-tools/automation/yaml-templates/build-macos-steps.yaml index edcaffbee53..53aa5042eda 100644 --- a/build-tools/automation/yaml-templates/build-macos-steps.yaml +++ b/build-tools/automation/yaml-templates/build-macos-steps.yaml @@ -50,6 +50,11 @@ steps: displayName: make jenkins retryCountOnTaskFailure: 1 +- template: /build-tools/automation/yaml-templates/generate-cgmanifest.yaml + parameters: + configuration: $(XA.Build.Configuration) + xaSourcePath: ${{ parameters.xaSourcePath }} + - script: make create-installers CONFIGURATION=$(XA.Build.Configuration) MSBUILD_ARGS='${{ parameters.makeMSBuildArgs }}' workingDirectory: ${{ parameters.xaSourcePath }} displayName: make create-installers diff --git a/build-tools/automation/yaml-templates/build-windows-steps.yaml b/build-tools/automation/yaml-templates/build-windows-steps.yaml index e3b83ed4de8..26c3d3100f7 100644 --- a/build-tools/automation/yaml-templates/build-windows-steps.yaml +++ b/build-tools/automation/yaml-templates/build-windows-steps.yaml @@ -38,6 +38,10 @@ steps: projects: Xamarin.Android.sln arguments: '-c $(XA.Build.Configuration) -t:Prepare --no-restore -bl:$(System.DefaultWorkingDirectory)\bin\Build$(XA.Build.Configuration)\dotnet-build-prepare.binlog' +- template: /build-tools/automation/yaml-templates/generate-cgmanifest.yaml + parameters: + configuration: $(XA.Build.Configuration) + # Build Xamarin.Android and configure local workloads to test improved local build loop - template: /build-tools/automation/yaml-templates/run-dotnet-preview.yaml parameters: diff --git a/build-tools/automation/yaml-templates/commercial-build.yaml b/build-tools/automation/yaml-templates/commercial-build.yaml index 94ccd2ff676..dcf040f8e1d 100644 --- a/build-tools/automation/yaml-templates/commercial-build.yaml +++ b/build-tools/automation/yaml-templates/commercial-build.yaml @@ -35,6 +35,11 @@ steps: workingDirectory: ${{ parameters.xaSourcePath }} displayName: make jenkins +- template: /build-tools/automation/yaml-templates/generate-cgmanifest.yaml + parameters: + configuration: $(XA.Build.Configuration) + xaSourcePath: ${{ parameters.xaSourcePath }} + - task: CodeQL3000Finalize@0 displayName: CodeQL 3000 Finalize condition: and(succeededOrFailed(), eq(variables['Codeql.Enabled'], 'true'), eq(variables['Build.SourceBranch'], 'refs/heads/main')) diff --git a/build-tools/automation/yaml-templates/generate-cgmanifest.yaml b/build-tools/automation/yaml-templates/generate-cgmanifest.yaml new file mode 100644 index 00000000000..24d19b6be5f --- /dev/null +++ b/build-tools/automation/yaml-templates/generate-cgmanifest.yaml @@ -0,0 +1,109 @@ +# Generates bin/Build$(Configuration)/cgmanifest.json from the repository's +# git submodules. The file is consumed by the Azure DevOps Component +# Governance Detection task (auto-injected on internal pipelines by +# eng/common/core-templates/job/job.yml). This replaces the previous +# xaprepare Step_GenerateCGManifest step. + +parameters: + configuration: $(XA.Build.Configuration) + xaSourcePath: $(System.DefaultWorkingDirectory) + +steps: +- pwsh: | + $ErrorActionPreference = 'Stop' + Set-Location -LiteralPath '${{ parameters.xaSourcePath }}' + + # Parse .gitmodules entries that live under external/ (matches the + # previous C# behavior which filtered on "submodule.external/"). + $configLines = & git config --blob HEAD:.gitmodules --list + if ($LASTEXITCODE -ne 0) { + throw "git config --list failed with exit code $LASTEXITCODE" + } + + $statusLines = & git submodule status + if ($LASTEXITCODE -ne 0) { + throw "git submodule status failed with exit code $LASTEXITCODE" + } + + # Build a map of submodule id -> @{ Path; Url } + $prefix = 'submodule.external/' + $entries = [ordered]@{} + foreach ($line in $configLines) { + if (-not $line.StartsWith($prefix)) { continue } + $eq = $line.IndexOf('=') + if ($eq -lt 0) { continue } + $lastDot = $line.LastIndexOf('.', $eq) + if ($lastDot -lt 0) { continue } + $id = $line.Substring($prefix.Length, $lastDot - $prefix.Length) + $key = $line.Substring($lastDot, $eq - $lastDot) + $value = $line.Substring($eq + 1) + if (-not $entries.Contains($id)) { + $entries[$id] = @{ Path = $null; Url = $null } + } + if ($key -eq '.path') { + $entries[$id].Path = $value + } elseif ($key -eq '.url') { + # Strip trailing ".git" to match the previous C# behavior. + if ($value.EndsWith('.git')) { + $value = $value.Substring(0, $value.Length - 4) + } + $entries[$id].Url = $value + } + } + + # Map submodule path -> commit hash via `git submodule status` output. + # Status lines are " [ (...)]", where is + # ' ', '-', or '+'. + $hashByPath = @{} + foreach ($line in $statusLines) { + if ([string]::IsNullOrEmpty($line)) { continue } + if ($line -match '^.([0-9a-fA-F]{40})\s+(\S+)') { + $hashByPath[$matches[2]] = $matches[1] + } + } + + # Build registration objects, sorted by repositoryUrl (ordinal, + # case-insensitive) to keep the output stable across runs. + $registrations = @() + foreach ($id in $entries.Keys) { + $path = $entries[$id].Path + $url = $entries[$id].Url + if ($null -eq $path -or $null -eq $url) { continue } + $hash = '' + if ($hashByPath.ContainsKey($path)) { + $hash = $hashByPath[$path] + } + $registrations += [pscustomobject]@{ Url = $url; Hash = $hash } + } + $registrations = @($registrations | + Sort-Object -Property @{ Expression = { $_.Url.ToLowerInvariant() } } | + ForEach-Object { + [ordered]@{ + component = [ordered]@{ + type = 'git' + git = [ordered]@{ + commitHash = $_.Hash + repositoryUrl = $_.Url + } + } + } + }) + + $doc = [ordered]@{ + '$schema' = 'https://json.schemastore.org/component-detection-manifest.json' + version = 1 + registrations = $registrations + } + $json = $doc | ConvertTo-Json -Depth 10 + + $outDir = Join-Path '${{ parameters.xaSourcePath }}' "bin/Build${{ parameters.configuration }}" + New-Item -ItemType Directory -Path $outDir -Force | Out-Null + $outPath = Join-Path $outDir 'cgmanifest.json' + + # Write UTF-8 without BOM. + $utf8NoBom = [System.Text.UTF8Encoding]::new($false) + [System.IO.File]::WriteAllText($outPath, $json, $utf8NoBom) + + Write-Host "Wrote $outPath with $($registrations.Count) registration(s)." + displayName: Generate cgmanifest.json + continueOnError: false diff --git a/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs b/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs index 06cfc3086df..5947f2f0d2b 100644 --- a/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs +++ b/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs @@ -19,7 +19,6 @@ protected override void AddSteps (Context context) throw new ArgumentNullException (nameof (context)); Steps.Add (new Step_GenerateFiles (atBuildStart: true)); - Steps.Add (new Step_GenerateCGManifest ()); } protected override void AddEndSteps (Context context) diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_GenerateCGManifest.cs b/build-tools/xaprepare/xaprepare/Steps/Step_GenerateCGManifest.cs deleted file mode 100644 index 53dd72a13e6..00000000000 --- a/build-tools/xaprepare/xaprepare/Steps/Step_GenerateCGManifest.cs +++ /dev/null @@ -1,160 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; - -namespace Xamarin.Android.Prepare -{ - partial class Step_GenerateCGManifest : Step - { - public Step_GenerateCGManifest () - : base ("Generate cgmanifest.json") - {} - - protected override async Task Execute (Context context) - { - var git = new GitRunner (context); - var gitSubmoduleInfo = await git.ConfigList (new[]{"--blob", "HEAD:.gitmodules"}); - var gitSubmoduleStatus = await git.SubmoduleStatus (); - var gitSubmodules = GitSubmoduleInfo.GetGitSubmodules (gitSubmoduleInfo, gitSubmoduleStatus) - .OrderBy (e => e.RepositoryUrl, StringComparer.OrdinalIgnoreCase); - - var jsonPath = Path.Combine (Configurables.Paths.BuildBinDir, "cgmanifest.json"); - using var json = File.CreateText (jsonPath); - - json.WriteLine ("{"); - json.WriteLine (" \"$schema\": \"https://json.schemastore.org/component-detection-manifest.json\","); - json.WriteLine (" \"version\": 1,"); - json.WriteLine (" \"registrations\": ["); - - bool first = true; - - foreach (var entry in gitSubmodules) { - if (first) { - first = false; - } else { - json.WriteLine (","); - } - - json.WriteLine ($" {{"); - json.WriteLine ($" \"component\": {{"); - json.WriteLine ($" \"type\": \"git\","); - json.WriteLine ($" \"git\": {{"); - json.WriteLine ($" \"commitHash\": \"{entry.CommitHash}\","); - json.WriteLine ($" \"repositoryUrl\": \"{entry.RepositoryUrl}\""); - json.WriteLine ($" }}"); - json.WriteLine ($" }}"); - json.Write ($" }}"); - } - - json.WriteLine (); - json.WriteLine (" ]"); - json.WriteLine ("}"); - - return true; - } - } - - sealed class GitSubmoduleInfo - { - public string Name { - get { - const string github = "github.com/"; - int i = RepositoryUrl.IndexOf (github, StringComparison.OrdinalIgnoreCase); - if (i >= 0) - return RepositoryUrl.Substring (i + github.Length); - return RepositoryUrl; - } - } - - public string RepositoryUrl { get; private set; } = String.Empty; - public string CommitHash { get; private set; } = String.Empty; - public string LocalPath { get; private set; } = String.Empty; - - GitSubmoduleInfo () - { - } - - const string Submodule = "submodule.external/"; - - public static IEnumerable GetGitSubmodules (List? config, List? submoduleStatus) - { - if (config == null) { - yield break; - } - - string? entryId = null; - string? path = null; - string? url = null; - - foreach (var line in config) { - if (!line.StartsWith (Submodule, StringComparison.Ordinal)) - continue; - - string? id = GetSubmoduleId (line); - if (id != entryId) { - if (url != null && path != null) - yield return CreateSubmoduleInfo (url, path, submoduleStatus); - - entryId = id; - path = null; - url = null; - } - - const string Path = ".path="; - const string Url = ".url="; - const string Git = ".git"; - - int pathIndex = line.IndexOf (Path, StringComparison.Ordinal); - if (pathIndex > 0) { - path = line.Substring (pathIndex + Path.Length); - continue; - } - - int urlIndex = line.IndexOf (Url, StringComparison.Ordinal); - if (urlIndex > 0) { - int start = urlIndex + Url.Length; - int count = line.Length - start; - if (line.EndsWith (Git, StringComparison.Ordinal)) - count -= Git.Length; - url = line.Substring (start, count); - continue; - } - } - - if (url != null && path != null) - yield return CreateSubmoduleInfo (url, path, submoduleStatus); - } - - static GitSubmoduleInfo CreateSubmoduleInfo (string url, string path, List? submoduleStatus) - { - string commitHash = String.Empty; - - if (submoduleStatus != null) { - foreach (var e in submoduleStatus) { - int pi = e.IndexOf (path, StringComparison.OrdinalIgnoreCase); - if (pi < 1 || e [pi - 1] != ' ') - continue; - commitHash = e.Substring (1, pi - 2); - break; - } - } - - return new GitSubmoduleInfo { - LocalPath = path, - RepositoryUrl = url, - CommitHash = commitHash, - }; - } - - static string? GetSubmoduleId (string line) - { - int eq = line.IndexOf ('='); - if (eq < 0) - return null; - int lastDot = line.LastIndexOf ('.', eq); - return line.Substring (Submodule.Length, lastDot - Submodule.Length); - } - } -} diff --git a/build-tools/xaprepare/xaprepare/ToolRunners/GitRunner.cs b/build-tools/xaprepare/xaprepare/ToolRunners/GitRunner.cs index 1fe8f8cf149..214956f34fe 100644 --- a/build-tools/xaprepare/xaprepare/ToolRunners/GitRunner.cs +++ b/build-tools/xaprepare/xaprepare/ToolRunners/GitRunner.cs @@ -95,31 +95,6 @@ public async Task Clone (string url, string destinationDirectoryPath) return await RunGit (runner, $"clone-{dirName}"); } - public async Task?> SubmoduleStatus (string? workingDirectory = null) - { - string runnerWorkingDirectory = DetermineRunnerWorkingDirectory (workingDirectory); - - var runner = CreateGitRunner (runnerWorkingDirectory);; - runner.AddArgument ("submodule"); - runner.AddArgument ("status"); - - var lines = new List (); - - bool success = await RunTool ( - () => { - using (var outputSink = (OutputSink)SetupOutputSink (runner)) { - outputSink.LineCallback = (string? line) => lines.Add (line ?? String.Empty); - return runner.Run (); - } - } - ); - - if (!success) - return null; - - return lines; - } - public async Task SubmoduleUpdate (string? workingDirectory = null, bool init = true, bool recursive = true) { string runnerWorkingDirectory = DetermineRunnerWorkingDirectory (workingDirectory); @@ -229,31 +204,6 @@ public string GetTopCommitHash (string? workingDirectory = null, bool shortHash return parserState.Entries; } - public async Task?> ConfigList (string[] fileOptions, string? workingDirectory = null) - { - var runner = CreateGitRunner (workingDirectory); - runner.AddArgument ("config"); - foreach (var opt in fileOptions) - runner.AddArgument (opt); - runner.AddArgument ("--list"); - - var lines = new List (); - - bool success = await RunTool ( - () => { - using (var outputSink = (OutputSink)SetupOutputSink (runner)) { - outputSink.LineCallback = (string? line) => lines.Add (line ?? String.Empty); - return runner.Run (); - } - } - ); - - if (!success) - return null; - - return lines; - } - public async Task IsRepoUrlHttps (string workingDirectory) { if (!Directory.Exists (workingDirectory))