-
Notifications
You must be signed in to change notification settings - Fork 2.1k
C#: Support replaces-base via the DependabotProxy.
#22494
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
74d2612
1cf5d23
f10882c
1b1a22a
d6ae24f
6e71be1
ae89e13
983dbbc
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,5 @@ | ||
| using System; | ||
| using System.Collections.Immutable; | ||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using System.Security.Cryptography.X509Certificates; | ||
|
|
@@ -14,13 +15,39 @@ public class DependabotProxy : IDependabotProxy | |
| /// <summary> | ||
| /// Represents configurations for package registries. | ||
| /// </summary> | ||
| /// <param name="Type">The type of package registry.</param> | ||
| /// <param name="URL">The URL of the package registry.</param> | ||
| public record class RegistryConfig(string Type, string URL); | ||
| public class RegistryConfig | ||
| { | ||
| /// <summary> | ||
| /// The type of the package registry. | ||
| /// </summary> | ||
| public string Type { get; init; } = ""; | ||
|
|
||
| /// <summary> | ||
| /// The URL of the package registry. | ||
| /// </summary> | ||
| public string URL { get; init; } = ""; | ||
|
|
||
| /// <summary> | ||
| /// A boolean indicating whether this registry replaces the base registry. | ||
| /// </summary> | ||
| [JsonProperty("replaces-base")] | ||
| public bool ReplacesBase { get; init; } = false; | ||
| }; | ||
|
|
||
| public string Address { get; } | ||
|
|
||
| public HashSet<string> RegistryURLs { get; } = []; | ||
| /// <summary> | ||
| /// A dictionary mapping registry URLs to a boolean indicating whether they replace the base registry. | ||
| /// </summary> | ||
| private readonly Dictionary<string, bool> registryMapping = []; | ||
|
|
||
| private ImmutableHashSet<string>? registryURLs; | ||
| public ImmutableHashSet<string> RegistryURLs => | ||
| registryURLs ??= registryMapping.Keys.ToImmutableHashSet(); | ||
|
|
||
| private ImmutableHashSet<string>? registryBaseURLs; | ||
| public ImmutableHashSet<string> RegistryBaseURLs => | ||
| registryBaseURLs ??= registryMapping.Where(kvp => kvp.Value).Select(kvp => kvp.Key).ToImmutableHashSet(); | ||
|
Comment on lines
+44
to
+50
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It might be good to add docs comments for |
||
|
|
||
| public string? CertificatePath { get; private set; } | ||
|
|
||
|
|
@@ -65,7 +92,7 @@ private DependabotProxy(IDependabotProxyConfiguration config, ILogger logger, Te | |
| } | ||
|
|
||
| logger.LogInfo($"Found private registry at '{registry.URL}'"); | ||
| RegistryURLs.Add(registry.URL); | ||
| registryMapping.AddOrUpdateToLatest(registry.URL, registry.ReplacesBase); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,13 +10,17 @@ namespace Semmle.Extraction.CSharp.DependencyFetching | |
| { | ||
| internal sealed partial class FeedManager : IDisposable | ||
| { | ||
| internal const string PublicNugetOrgFeed = "https://api.nuget.org/v3/index.json"; | ||
| private const string PublicNugetOrg = "nuget.org"; | ||
| private const string PublicDotNugetOrg = $".{PublicNugetOrg}"; | ||
| internal const string PublicApiNugetOrgFeed = $"https://api{PublicDotNugetOrg}/v3/index.json"; | ||
|
|
||
| private readonly ILogger logger; | ||
| private readonly IDotNet dotnet; | ||
| private readonly IFileProvider fileProvider; | ||
| private readonly DependencyDirectory emptyPackageDirectory; | ||
| private readonly ImmutableHashSet<string> privateRegistryFeeds; | ||
| private readonly bool hasPrivateRegistryBaseFeeds; | ||
| private readonly ImmutableHashSet<string> privateRegistryBaseFeeds; | ||
| private readonly IFeedManagerIO feedManagerIo; | ||
|
|
||
| /// <summary> | ||
|
|
@@ -72,14 +76,33 @@ internal sealed partial class FeedManager : IDisposable | |
| /// </summary> | ||
| public ImmutableHashSet<string> ReachableFallbackFeeds => lazyReachableFallbackFeeds.Value; | ||
|
|
||
| private readonly Lazy<ImmutableHashSet<string>> lazyReachableDefaultFeeds; | ||
|
|
||
| /// <summary> | ||
| /// Gets the list of default NuGet feeds that are configured in the environment. | ||
| /// This is either the public NuGet feed or a set of feeds specified by the environment. | ||
| /// </summary> | ||
| public ImmutableHashSet<string> DefaultFeeds { get; init; } | ||
|
|
||
| /// <summary> | ||
| /// Gets the list of reachable default NuGet feeds. | ||
| /// </summary> | ||
| public ImmutableHashSet<string> ReachableDefaultFeeds => lazyReachableDefaultFeeds.Value; | ||
|
|
||
| public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider, IFeedManagerIO feedManagerIo) | ||
| { | ||
| this.logger = logger; | ||
| this.dotnet = dotnet; | ||
| this.fileProvider = fileProvider; | ||
| this.feedManagerIo = feedManagerIo; | ||
| privateRegistryFeeds = dependabotProxy?.RegistryURLs.ToImmutableHashSet() ?? []; | ||
| privateRegistryFeeds = dependabotProxy?.RegistryURLs ?? []; | ||
| HasPrivateRegistryFeeds = privateRegistryFeeds.Count > 0; | ||
| privateRegistryBaseFeeds = dependabotProxy?.RegistryBaseURLs ?? []; | ||
| hasPrivateRegistryBaseFeeds = privateRegistryBaseFeeds.Count > 0; | ||
|
|
||
| DefaultFeeds = hasPrivateRegistryBaseFeeds | ||
| ? privateRegistryBaseFeeds | ||
| : [PublicApiNugetOrgFeed]; | ||
| emptyPackageDirectory = new DependencyDirectory("empty", "empty package", logger); | ||
|
|
||
| lazyExplicitFeeds = new Lazy<ImmutableHashSet<string>>(GetExplicitFeeds); | ||
|
|
@@ -96,13 +119,28 @@ public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotP | |
| var reachableFallbackFeeds = GetReachableFallbackNugetFeeds(); | ||
| return reachableFallbackFeeds.ToImmutableHashSet(); | ||
| }); | ||
| lazyReachableDefaultFeeds = new Lazy<ImmutableHashSet<string>>(() => CheckSpecifiedFeeds(DefaultFeeds)); | ||
| } | ||
|
|
||
| public FeedManager(ILogger logger, IDotNet dotnet, IDependabotProxy? dependabotProxy, IFileProvider fileProvider) | ||
| : this(logger, dotnet, dependabotProxy, fileProvider, new FeedManagerIO(logger, dependabotProxy)) | ||
| { | ||
| } | ||
|
|
||
| private bool IsNugetOrgFeed(string url) | ||
| { | ||
| try | ||
| { | ||
| var uri = new Uri(url); | ||
| return uri.Host.EndsWith(PublicDotNugetOrg, StringComparison.InvariantCultureIgnoreCase) || | ||
| string.Equals(uri.Host, PublicNugetOrg, StringComparison.InvariantCultureIgnoreCase); | ||
| } | ||
| catch (UriFormatException) | ||
| { | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
| private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds) | ||
| { | ||
| var results = getNugetFeeds(); | ||
|
|
@@ -124,10 +162,18 @@ private IEnumerable<string> GetFeeds(Func<IList<string>> getNugetFeeds) | |
| continue; | ||
| } | ||
|
|
||
| if (!string.IsNullOrWhiteSpace(url)) | ||
| if (hasPrivateRegistryBaseFeeds && IsNugetOrgFeed(url)) | ||
| { | ||
| yield return url; | ||
| // Use private registry base feeds. | ||
| foreach (var feed in privateRegistryBaseFeeds) | ||
| { | ||
| logger.LogDebug($"Using private registry base feed '{feed}'."); | ||
| yield return feed; | ||
| } | ||
| continue; | ||
| } | ||
|
|
||
| yield return url; | ||
| } | ||
| } | ||
|
|
||
|
|
@@ -266,22 +312,6 @@ private ImmutableHashSet<string> CheckSpecifiedFeeds(ImmutableHashSet<string> fe | |
| return reachable.Union(feeds.Where(feed => excludedFeeds.Contains(feed))).ToImmutableHashSet(); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Return true if the default NuGet feed is reachable, false otherwise. | ||
| /// If the reachability check is disabled, this method will always return true. | ||
| /// </summary> | ||
| /// <returns>True if the default NuGet feed is reachable, false otherwise.</returns> | ||
| public bool IsDefaultFeedReachable() | ||
| { | ||
| if (CheckNugetFeedResponsiveness) | ||
| { | ||
| var (initialTimeout, tryCount) = GetFeedRequestSettings(isFallback: false); | ||
| return feedManagerIo.IsFeedReachable(PublicNugetOrgFeed, initialTimeout, tryCount); | ||
| } | ||
|
|
||
| return true; | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Tests which of the feeds given by <paramref name="feedsToCheck"/> are reachable. | ||
| /// </summary> | ||
|
|
@@ -315,8 +345,8 @@ private List<string> GetReachableFallbackNugetFeeds() | |
| var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet(); | ||
| if (fallbackFeeds.Count == 0) | ||
|
Comment on lines
345
to
346
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I don't think it's possible to set the environment variable for the fallback feeds in Default Setup, but I would still treat both those and the ones from the private registry configurations as explicit, user-provided configuration. Therefore, it might make sense to compute |
||
| { | ||
| fallbackFeeds.Add(PublicNugetOrgFeed); | ||
| logger.LogInfo($"No fallback NuGet feeds specified. Adding default feed: {PublicNugetOrgFeed}"); | ||
| fallbackFeeds.UnionWith(DefaultFeeds); | ||
| logger.LogInfo($"No fallback NuGet feeds specified. Adding default feeds: {string.Join(", ", DefaultFeeds.OrderBy(f => f))}"); | ||
|
michaelnebel marked this conversation as resolved.
michaelnebel marked this conversation as resolved.
|
||
|
|
||
| var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback); | ||
| logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}"); | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You can treat these fields as required, so if it would help avoid unexpected results elsewhere, it might be better not to initialise them to the empty string.