Skip to content
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
using System;
using System.Collections.Immutable;
using System.Collections.Generic;
using System.IO;
using System.Security.Cryptography.X509Certificates;
Expand All@@ -14,13 +15,39 @@ public class DependabotProxy : IDependabotProxy
/// <summary>
/// Represents configurations for package registries.
/// </summary>
/// <param name="Type">The type of package registry.</param>
/// <param name="URL">The URL of the package registry.</param>
public record class RegistryConfig(string Type, string URL);
public class RegistryConfig
{
/// <summary>
/// The type of the package registry.
/// </summary>
public string Type { get; init; } = "";

/// <summary>
/// The URL of the package registry.
/// </summary>
public string URL { get; init; } = "";

/// <summary>
/// A boolean indicating whether this registry replaces the base registry.
/// </summary>
[JsonProperty("replaces-base")]
public bool ReplacesBase { get; init; } = false;
};

public string Address { get; }

public HashSet<string> RegistryURLs { get; } = [];
/// <summary>
/// A dictionary mapping registry URLs to a boolean indicating whether they replace the base registry.
/// </summary>
private readonly Dictionary<string, bool> registryMapping = [];

private ImmutableHashSet<string>? registryURLs;
public ImmutableHashSet<string> RegistryURLs =>
registryURLs ??= registryMapping.Keys.ToImmutableHashSet();

private ImmutableHashSet<string>? registryBaseURLs;
public ImmutableHashSet<string> RegistryBaseURLs =>
registryBaseURLs ??= registryMapping.Where(kvp => kvp.Value).Select(kvp => kvp.Key).ToImmutableHashSet();

public string? CertificatePath { get; private set; }

Expand DownExpand Up@@ -65,7 +92,7 @@ private DependabotProxy(IDependabotProxyConfiguration config, ILogger logger, Te
}

logger.LogInfo($"Found private registry at '{registry.URL}'");
RegistryURLs.Add(registry.URL);
registryMapping.AddOrUpdateToLatest(registry.URL, registry.ReplacesBase);
}
}
}
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -56,7 +56,6 @@ internal static class EnvironmentVariableNames

/// <summary>
/// Specifies the NuGet feeds to use for fallback NuGet dependency fetching. The value is a space-separated list of feed URLs.
/// The default value is `https://api.nuget.org/v3/index.json`.
/// </summary>
public const string FallbackNugetFeeds = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK";

Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -10,13 +10,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching
{
internal sealed partial class FeedManager : IDisposable
{
internal const string PublicNugetOrgFeed = "https://api.nuget.org/v3/index.json";
private const string PublicNugetOrg = "nuget.org";
private const string PublicDotNugetOrg = $".{PublicNugetOrg}";
internal const string PublicApiNugetOrgFeed = $"https://api{PublicDotNugetOrg}/v3/index.json";

private readonly ILogger logger;
private readonly IDotNet dotnet;
private readonly IFileProvider fileProvider;
private readonly DependencyDirectory emptyPackageDirectory;
private readonly ImmutableHashSet<string> privateRegistryFeeds;
private readonly bool hasPrivateRegistryBaseFeeds;
private readonly ImmutableHashSet<string> privateRegistryBaseFeeds;
private readonly IFeedManagerIO feedManagerIo;

/// <summary>
Expand DownExpand Up@@ -72,14 +76,33 @@ internal sealed partial class FeedManager : IDisposable
/// </summary>
public ImmutableHashSet<string> ReachableFallbackFeeds => lazyReachableFallbackFeeds.Value;

private readonly Lazy<ImmutableHashSet<string>> lazyReachableDefaultFeeds;

/// <summary>
/// Gets the list of default NuGet feeds that are configured in the environment.
/// This is either the public NuGet feed or a set of feeds specified by the environment.
/// </summary>
public ImmutableHashSet<string> DefaultFeeds { get; init; }

/// <summary>
/// Gets the list of reachable default NuGet feeds.
/// </summary>
public ImmutableHashSet<string> ReachableDefaultFeeds => lazyReachableDefaultFeeds.Value;

public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider, IFeedManagerIO feedManagerIo)
{
this.logger = logger;
this.dotnet = dotnet;
this.fileProvider = fileProvider;
this.feedManagerIo = feedManagerIo;
privateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? [];
privateRegistryFeeds = dependabotProxy?.RegistryURLs ?? [];
HasPrivateRegistryFeeds = privateRegistryFeeds.Count > 0;
privateRegistryBaseFeeds = dependabotProxy?.RegistryBaseURLs ?? [];
hasPrivateRegistryBaseFeeds = privateRegistryBaseFeeds.Count > 0;

DefaultFeeds = hasPrivateRegistryBaseFeeds
? privateRegistryBaseFeeds
: [PublicApiNugetOrgFeed];
emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger);

lazyExplicitFeeds = new Lazy<ImmutableHashSet<string>>(GetExplicitFeeds);
Expand All@@ -96,13 +119,28 @@ public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotP
var reachableFallbackFeeds = GetReachableFallbackNugetFeeds();
return reachableFallbackFeeds.ToImmutableHashSet();
});
lazyReachableDefaultFeeds = new Lazy<ImmutableHashSet<string>>(() => CheckSpecifiedFeeds(DefaultFeeds));
}

public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider)
: this(logger, dotnet, dependabotProxy, fileProvider, new FeedManagerIO(logger, dependabotProxy))
{
}

private bool IsNugetOrgFeed(string url)
{
try
{
var uri = new Uri(url);
return uri.Host.EndsWith(PublicDotNugetOrg, StringComparison.InvariantCultureIgnoreCase) ||
string.Equals(uri.Host, PublicNugetOrg, StringComparison.InvariantCultureIgnoreCase);
}
catch (UriFormatException)
{
return false;
}
}

private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
{
var results = getNugetFeeds();
Expand All@@ -124,10 +162,18 @@ private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds)
continue;
}

if (!string.IsNullOrWhiteSpace(url))
if (hasPrivateRegistryBaseFeeds && IsNugetOrgFeed(url))
{
yield return url;
// Use private registry base feeds.
foreach (var feed in privateRegistryBaseFeeds)
{
logger.LogDebug($"Using private registry base feed '{feed}'.");
yield return feed;
}
continue;
}

yield return url;
}
}

Expand DownExpand Up@@ -266,22 +312,6 @@ private ImmutableHashSet<string> CheckSpecifiedFeeds(ImmutableHashSet<string> fe
return reachable.Union(feeds.Where(feed => excludedFeeds.Contains(feed))).ToImmutableHashSet();
}

/// <summary>
/// Return true if the default NuGet feed is reachable, false otherwise.
/// If the reachability check is disabled, this method will always return true.
/// </summary>
/// <returns>True if the default NuGet feed is reachable, false otherwise.</returns>
public bool IsDefaultFeedReachable()
{
if (CheckNugetFeedResponsiveness)
{
var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false);
return feedManagerIo.IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount);
}

return true;
}

/// <summary>
/// Tests which of the feeds given by <paramref name="feedsToCheck"/> are reachable.
/// </summary>
Expand DownExpand Up@@ -315,8 +345,8 @@ private List<string> GetReachableFallbackNugetFeeds()
var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
if (fallbackFeeds.Count == 0)
{
fallbackFeeds.Add(PublicNugetOrgFeed);
logger.LogInfo($"No fallback NuGet feeds specified. Adding default feed: {PublicNugetOrgFeed}");
fallbackFeeds.UnionWith(DefaultFeeds);
logger.LogInfo($"No fallback NuGet feeds specified. Adding default feeds: {string.Join(", ", DefaultFeeds.OrderBy(f => f))}");
Comment thread
michaelnebel marked this conversation as resolved.
Comment thread
michaelnebel marked this conversation as resolved.

var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback);
logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}");
Expand Down
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Security.Cryptography.X509Certificates;

namespace Semmle.Extraction.CSharp.DependencyFetching
Expand All@@ -14,7 +14,12 @@ public interface IDependabotProxy : IDisposable
/// <summary>
/// The URLs of package registries that are configured for the proxy.
/// </summary>
HashSet<string> RegistryURLs { get; }
ImmutableHashSet<string> RegistryURLs { get; }

/// <summary>
/// The URLs of package registries that replace the base registry.
/// </summary>
ImmutableHashSet<string> RegistryBaseURLs { get; }

/// <summary>
/// The path to the temporary file where the certificate is stored.
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -460,7 +460,7 @@ private bool TryRestorePackageManually(string package, List<string> nugetSources
return true;
}

if (!feedManager.CheckNugetFeedResponsiveness && res.HasNugetPackageSourceError && nugetSources.Count > 0)
if (!feedManager.CheckNugetFeedResponsiveness && !feedManager.HasPrivateRegistryFeeds && res.HasNugetPackageSourceError && nugetSources.Count > 0)
{
logger.LogDebug($"Trying to restore '{package}' without explicitly providing NuGet sources.");
// Restore could not be completed because the listed source is unavailable. Try without an explicit restore source argument.
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -67,10 +67,6 @@ private class NugetExeWrapper : IPackagesConfigRestore

private bool IsWindows => SystemBuildActions.Instance.IsWindows();

private bool? isDefaultFeedReachable;
private bool IsDefaultFeedReachable =>
isDefaultFeedReachable ??= feedManager.IsDefaultFeedReachable();

/// <summary>
/// Create the package manager for a specified source tree.
/// </summary>
Expand DownExpand Up@@ -169,15 +165,18 @@ private bool TryRestoreNugetPackage(string packagesConfig)

List<string> sourcesArgument = [];
var feedsToUse = feedManager.FeedsToUse(packagesConfig).ToList();
var useDefaultFeed = feedsToUse.Count == 0 && IsDefaultFeedReachable;
var defaultFeeds = feedManager.CheckNugetFeedResponsiveness
? feedManager.ReachableDefaultFeeds
: feedManager.DefaultFeeds;
var useDefaultFeeds = feedsToUse.Count == 0 && defaultFeeds.Count > 0;

// Explicitly construct the sources to be used for the restore command when checking feed
// responsiveness, using private registries, or falling back to nuget.org.
if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds || useDefaultFeed)
// responsiveness, using private registries, or falling back to default feeds.
if (feedManager.CheckNugetFeedResponsiveness || feedManager.HasPrivateRegistryFeeds || useDefaultFeeds)
{
if (useDefaultFeed)
if (useDefaultFeeds)
{
feedsToUse.Add(FeedManager.PublicNugetOrgFeed);
feedsToUse.AddRange(defaultFeeds);
}
var restoreFeeds = feedManager.RestoreFeeds(feedsToUse);
sourcesArgument = restoreFeeds.SelectMany<string, string>(feed => ["-Source", feed]).ToList();
Expand Down
31 changes: 30 additions & 1 deletion csharp/extractor/Semmle.Extraction.Tests/DependabotProxy.cs
Original file line numberDiff line numberDiff line change
Expand Up@@ -135,7 +135,8 @@ public void TestDependabotRegistryUrls1()

// Verify
Assert.NotNull(proxy);
Assert.Equal([], proxy.RegistryURLs);
Assert.Empty(proxy.RegistryURLs);
Assert.Empty(proxy.RegistryBaseURLs);
}

[Fact]
Expand All@@ -158,6 +159,7 @@ public void TestDependabotRegistryUrls2()
Assert.Equal([
"https://nuget.pkg.github.com/org/index.json"
], proxy.RegistryURLs);
Assert.Empty(proxy.RegistryBaseURLs);
}

[Fact]
Expand All@@ -180,6 +182,33 @@ public void TestDependabotRegistryUrls3()
Assert.Equal([
"https://example.com/org/index.json"
], proxy.RegistryURLs);
Assert.Empty(proxy.RegistryBaseURLs);
}

[Fact]
public void TestDependabotReplacesBase1()
{
// Setup
var config = new DependabotConfigurationStub
{
Port = "8080",
Host = "localhost",
RegistryURLs = "[ { \"type\": \"nuget_feed\", \"url\": \"https://example.com/org/index.json\", \"replaces-base\": true }, { \"type\": \"nuget_feed\", \"url\": \"https://example2.com/org/index.json\", \"replaces-base\": false } ]"
};

// Execute
using var tempWorkingDirectory = MakeTemporaryDirectory();
using var proxy = DependabotProxy.Make(config, new LoggerStub(), new DiagnosticsWriterStub(), tempWorkingDirectory);

// Verify
Assert.NotNull(proxy);
Assert.Equal([
"https://example.com/org/index.json",
"https://example2.com/org/index.json"
], proxy.RegistryURLs);
Assert.Equal([
"https://example.com/org/index.json",
], proxy.RegistryBaseURLs);
}
}
}
Loading
Loading