From 55ce8356432c47d70901a6452c20a648a0c0b275 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Sharma?= Date: Sun, 31 Mar 2019 21:00:46 -0700 Subject: [PATCH] [Protocol] Mirror nuget.org's undocumented protocols (#249) NuGet.org uses RDF to generate its V3 API. This adds properties that aren't part of the official API, however, certain clients depend on these undocumented properties. --- .../RegistrationIndexController.cs | 5 +- .../RegistrationLeafController.cs | 3 +- .../Controllers/SearchController.cs | 8 ++- src/BaGet.Protocol/HttpClientExtensions.cs | 54 +++++++++++++++++++ src/BaGet.Protocol/HttpContentExtensions.cs | 29 ---------- .../PackageContent/PackageContentClient.cs | 8 ++- src/BaGet.Protocol/ProtocolException.cs | 27 ++++++++++ .../Registration/RegistrationClient.cs | 20 +++---- .../Registration/RegistrationIndex.cs | 19 ++++++- .../Registration/RegistrationLeaf.cs | 16 +++++- src/BaGet.Protocol/ResponseAndResult.cs | 49 +++++++++++++++++ .../Search/AutocompleteContext.cs | 15 ++++++ .../Search/AutocompleteResult.cs | 14 +++-- src/BaGet.Protocol/Search/SearchClient.cs | 14 ++--- src/BaGet.Protocol/Search/SearchContext.cs | 22 ++++++++ src/BaGet.Protocol/Search/SearchResponse.cs | 16 ++++-- .../ServiceIndex/ServiceIndexClient.cs | 8 ++- 17 files changed, 249 insertions(+), 78 deletions(-) create mode 100644 src/BaGet.Protocol/HttpClientExtensions.cs delete mode 100644 src/BaGet.Protocol/HttpContentExtensions.cs create mode 100644 src/BaGet.Protocol/ProtocolException.cs create mode 100644 src/BaGet.Protocol/ResponseAndResult.cs create mode 100644 src/BaGet.Protocol/Search/AutocompleteContext.cs create mode 100644 src/BaGet.Protocol/Search/SearchContext.cs diff --git a/src/BaGet.Core.Server/Controllers/Registration/RegistrationIndexController.cs b/src/BaGet.Core.Server/Controllers/Registration/RegistrationIndexController.cs index 2f79952d1..c39c936b8 100644 --- a/src/BaGet.Core.Server/Controllers/Registration/RegistrationIndexController.cs +++ b/src/BaGet.Core.Server/Controllers/Registration/RegistrationIndexController.cs @@ -51,6 +51,7 @@ public async Task Get(string id, CancellationToken cancellationTo // "Un-paged" example: https://api.nuget.org/v3/registration3/newtonsoft.json/index.json // Paged example: https://api.nuget.org/v3/registration3/fake/index.json return Json(new RegistrationIndex( + type: RegistrationIndex.DefaultType, count: packages.Count, totalDownloads: packages.Sum(p => p.Downloads), pages: new[] @@ -68,7 +69,7 @@ private RegistrationIndexPageItem ToRegistrationIndexPageItem(Package package) = new RegistrationIndexPageItem( leafUrl: Url.PackageRegistration(package.Id, package.Version), packageMetadata: new PackageMetadata( - catalogUri: $"https://api.nuget.org/v3/catalog0/data/2015.02.01.06.24.15/{package.Id}.{package.Version}.json", + catalogUri: Url.PackageRegistration(package.Id, package.Version), packageId: package.Id, version: package.Version, authors: string.Join(", ", package.Authors), @@ -115,4 +116,4 @@ private IReadOnlyList ToDependencyGroups(Package package) return groups; } } -} \ No newline at end of file +} diff --git a/src/BaGet.Core.Server/Controllers/Registration/RegistrationLeafController.cs b/src/BaGet.Core.Server/Controllers/Registration/RegistrationLeafController.cs index f1f42a6b5..53e8d87ca 100644 --- a/src/BaGet.Core.Server/Controllers/Registration/RegistrationLeafController.cs +++ b/src/BaGet.Core.Server/Controllers/Registration/RegistrationLeafController.cs @@ -44,6 +44,7 @@ public async Task Get(string id, string version, CancellationToke } var result = new RegistrationLeaf( + type: RegistrationLeaf.DefaultType, registrationUri: Url.PackageRegistration(id, nugetVersion), listed: package.Listed, downloads: package.Downloads, @@ -54,4 +55,4 @@ public async Task Get(string id, string version, CancellationToke return Json(result); } } -} \ No newline at end of file +} diff --git a/src/BaGet.Core.Server/Controllers/SearchController.cs b/src/BaGet.Core.Server/Controllers/SearchController.cs index 21521f87f..35b991250 100644 --- a/src/BaGet.Core.Server/Controllers/SearchController.cs +++ b/src/BaGet.Core.Server/Controllers/SearchController.cs @@ -43,14 +43,18 @@ public async Task> Get( return new SearchResponse( totalHits: results.Count, - data: results.Select(ToSearchResult).ToList()); + data: results.Select(ToSearchResult).ToList(), + context: SearchContext.Default(Url.RegistrationsBase())); } public async Task> Autocomplete([FromQuery(Name = "q")] string query = null) { var results = await _searchService.AutocompleteAsync(query); - return new AutocompleteResult(results.Count, results); + return new AutocompleteResult( + results.Count, + results, + AutocompleteContext.Default); } public async Task> Dependents([FromQuery(Name = "packageId")] string packageId) diff --git a/src/BaGet.Protocol/HttpClientExtensions.cs b/src/BaGet.Protocol/HttpClientExtensions.cs new file mode 100644 index 000000000..abb842ddd --- /dev/null +++ b/src/BaGet.Protocol/HttpClientExtensions.cs @@ -0,0 +1,54 @@ +using System.IO; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Newtonsoft.Json; + +namespace BaGet.Protocol +{ + internal static class HttpClientExtensions + { + private static readonly JsonSerializer Serializer = JsonSerializer.Create(Settings); + + private static readonly JsonSerializerSettings Settings = new JsonSerializerSettings + { + DateTimeZoneHandling = DateTimeZoneHandling.Utc, + DateParseHandling = DateParseHandling.DateTimeOffset, + NullValueHandling = NullValueHandling.Ignore, + }; + + public static async Task> DeserializeUrlAsync( + this HttpClient httpClient, + string documentUrl) + { + using (var response = await httpClient.GetAsync( + documentUrl, + HttpCompletionOption.ResponseHeadersRead)) + { + if (response.StatusCode != HttpStatusCode.OK) + { + return new ResponseAndResult( + HttpMethod.Get, + documentUrl, + response.StatusCode, + response.ReasonPhrase, + hasResult: false, + result: default); + } + + using (var stream = await response.Content.ReadAsStreamAsync()) + using (var textReader = new StreamReader(stream)) + using (var jsonReader = new JsonTextReader(textReader)) + { + return new ResponseAndResult( + HttpMethod.Get, + documentUrl, + response.StatusCode, + response.ReasonPhrase, + hasResult: true, + result: Serializer.Deserialize(jsonReader)); + } + } + } + } +} diff --git a/src/BaGet.Protocol/HttpContentExtensions.cs b/src/BaGet.Protocol/HttpContentExtensions.cs deleted file mode 100644 index 5ab6e9fbb..000000000 --- a/src/BaGet.Protocol/HttpContentExtensions.cs +++ /dev/null @@ -1,29 +0,0 @@ -using System.IO; -using System.Net.Http; -using System.Threading.Tasks; -using Newtonsoft.Json; - -namespace BaGet.Protocol -{ - internal static class HttpContentExtensions - { - public static JsonSerializer Serializer => JsonSerializer.Create(Settings); - - public static JsonSerializerSettings Settings => new JsonSerializerSettings - { - DateTimeZoneHandling = DateTimeZoneHandling.Utc, - DateParseHandling = DateParseHandling.DateTimeOffset, - NullValueHandling = NullValueHandling.Ignore, - }; - - public static async Task ReadAsAsync(this HttpContent content) - { - using (var stream = await content.ReadAsStreamAsync()) - using (var textReader = new StreamReader(stream)) - using (var jsonReader = new JsonTextReader(textReader)) - { - return Serializer.Deserialize(jsonReader); - } - } - } -} diff --git a/src/BaGet.Protocol/PackageContent/PackageContentClient.cs b/src/BaGet.Protocol/PackageContent/PackageContentClient.cs index cd5b461ee..e643cba9d 100644 --- a/src/BaGet.Protocol/PackageContent/PackageContentClient.cs +++ b/src/BaGet.Protocol/PackageContent/PackageContentClient.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.IO; using System.Net; using System.Net.Http; @@ -22,15 +22,13 @@ public PackageContentClient(HttpClient httpClient) /// The package's versions, or null if the package does not exist public async Task GetPackageVersionsOrNullAsync(string url) { - var response = await _httpClient.GetAsync(url); + var response = await _httpClient.DeserializeUrlAsync(url); if (response.StatusCode == HttpStatusCode.NotFound) { return null; } - response.EnsureSuccessStatusCode(); - - return await response.Content.ReadAsAsync(); + return response.GetResultOrThrow(); } public async Task GetPackageContentStreamAsync(string url) diff --git a/src/BaGet.Protocol/ProtocolException.cs b/src/BaGet.Protocol/ProtocolException.cs new file mode 100644 index 000000000..a894d5ff2 --- /dev/null +++ b/src/BaGet.Protocol/ProtocolException.cs @@ -0,0 +1,27 @@ +using System; +using System.Net; +using System.Net.Http; + +namespace BaGet.Protocol +{ + public class ProtocolException : Exception + { + public ProtocolException( + string message, + HttpMethod method, + string requestUri, + HttpStatusCode statusCode, + string reasonPhrase) : base(message) + { + Method = method ?? throw new ArgumentNullException(nameof(method)); + RequestUri = requestUri ?? throw new ArgumentNullException(nameof(requestUri)); + StatusCode = statusCode; + ReasonPhrase = reasonPhrase ?? throw new ArgumentNullException(nameof(reasonPhrase)); + } + + public HttpMethod Method { get; } + public string RequestUri { get; } + public HttpStatusCode StatusCode { get; } + public string ReasonPhrase { get; } + } +} diff --git a/src/BaGet.Protocol/Registration/RegistrationClient.cs b/src/BaGet.Protocol/Registration/RegistrationClient.cs index 4b98fdc35..d03ec6d0c 100644 --- a/src/BaGet.Protocol/Registration/RegistrationClient.cs +++ b/src/BaGet.Protocol/Registration/RegistrationClient.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Net; using System.Net.Http; using System.Threading.Tasks; @@ -16,33 +16,27 @@ public RegistrationClient(HttpClient httpClient) public async Task GetRegistrationIndexOrNullAsync(string indexUrl) { - var response = await _httpClient.GetAsync(indexUrl); + var response = await _httpClient.DeserializeUrlAsync(indexUrl); if (response.StatusCode == HttpStatusCode.NotFound) { return null; } - response.EnsureSuccessStatusCode(); - - return await response.Content.ReadAsAsync(); + return response.GetResultOrThrow(); } public async Task GetRegistrationIndexPageAsync(string pageUrl) { - var response = await _httpClient.GetAsync(pageUrl); - - response.EnsureSuccessStatusCode(); + var response = await _httpClient.DeserializeUrlAsync(pageUrl); - return await response.Content.ReadAsAsync(); + return response.GetResultOrThrow(); } public async Task GetRegistrationLeafAsync(string leafUrl) { - var response = await _httpClient.GetAsync(leafUrl); - - response.EnsureSuccessStatusCode(); + var response = await _httpClient.DeserializeUrlAsync(leafUrl); - return await response.Content.ReadAsAsync(); + return response.GetResultOrThrow(); } } } diff --git a/src/BaGet.Protocol/Registration/RegistrationIndex.cs b/src/BaGet.Protocol/Registration/RegistrationIndex.cs index 58ac613ee..23bcd3a6d 100644 --- a/src/BaGet.Protocol/Registration/RegistrationIndex.cs +++ b/src/BaGet.Protocol/Registration/RegistrationIndex.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using Newtonsoft.Json; @@ -9,12 +9,27 @@ namespace BaGet.Protocol /// public class RegistrationIndex { - public RegistrationIndex(int count, long totalDownloads, IReadOnlyList pages) + public static readonly IReadOnlyList DefaultType = new List + { + "catalog:CatalogRoot", + "PackageRegistration", + "catalog:Permalink" + }; + + public RegistrationIndex( + int count, + long totalDownloads, + IReadOnlyList pages, + IReadOnlyList type = null) { Count = count; Pages = pages ?? throw new ArgumentNullException(nameof(pages)); + Type = type; } + [JsonProperty(PropertyName = "@type")] + public IReadOnlyList Type { get; } + /// /// The number of registration pages. See . /// diff --git a/src/BaGet.Protocol/Registration/RegistrationLeaf.cs b/src/BaGet.Protocol/Registration/RegistrationLeaf.cs index c063a21c3..d21eeb64f 100644 --- a/src/BaGet.Protocol/Registration/RegistrationLeaf.cs +++ b/src/BaGet.Protocol/Registration/RegistrationLeaf.cs @@ -1,4 +1,5 @@ -using System; +using System; +using System.Collections.Generic; using Newtonsoft.Json; namespace BaGet.Protocol @@ -9,13 +10,20 @@ namespace BaGet.Protocol /// public class RegistrationLeaf { + public static readonly IReadOnlyList DefaultType = new List + { + "Package", + "http://schema.nuget.org/catalog#Permalink" + }; + public RegistrationLeaf( string registrationUri, bool listed, long downloads, string packageContentUrl, DateTimeOffset published, - string registrationIndexUrl) + string registrationIndexUrl, + IReadOnlyList type = null) { RegistrationUri = registrationUri ?? throw new ArgumentNullException(nameof(registrationIndexUrl)); Listed = listed; @@ -23,11 +31,15 @@ public RegistrationLeaf( Downloads = downloads; PackageContentUrl = packageContentUrl ?? throw new ArgumentNullException(nameof(packageContentUrl)); RegistrationIndexUrl = registrationIndexUrl ?? throw new ArgumentNullException(nameof(registrationIndexUrl)); + Type = type; } [JsonProperty(PropertyName = "@id")] public string RegistrationUri { get; } + [JsonProperty(PropertyName = "@type")] + public IReadOnlyList Type { get; } + public bool Listed { get; } public long Downloads { get; } diff --git a/src/BaGet.Protocol/ResponseAndResult.cs b/src/BaGet.Protocol/ResponseAndResult.cs new file mode 100644 index 000000000..cfa0dad6d --- /dev/null +++ b/src/BaGet.Protocol/ResponseAndResult.cs @@ -0,0 +1,49 @@ +using System; +using System.Net; +using System.Net.Http; + +namespace BaGet.Protocol +{ + internal class ResponseAndResult + { + public ResponseAndResult( + HttpMethod method, + string requestUri, + HttpStatusCode statusCode, + string reasonPhrase, + bool hasResult, + T result) + { + Method = method ?? throw new ArgumentNullException(nameof(method)); + RequestUri = requestUri ?? throw new ArgumentNullException(nameof(requestUri)); + StatusCode = statusCode; + ReasonPhrase = reasonPhrase ?? throw new ArgumentNullException(nameof(reasonPhrase)); + HasResult = hasResult; + Result = result; + } + + public HttpMethod Method { get; } + public string RequestUri { get; } + public HttpStatusCode StatusCode { get; } + public string ReasonPhrase { get; } + public bool HasResult { get; } + public T Result { get; } + + public T GetResultOrThrow() + { + if (!HasResult) + { + throw new ProtocolException( + $"The HTTP request failed.{Environment.NewLine}" + + $"Request: {Method} {RequestUri}{Environment.NewLine}" + + $"Response: {(int)StatusCode} {ReasonPhrase}", + Method, + RequestUri, + StatusCode, + ReasonPhrase); + } + + return Result; + } + } +} diff --git a/src/BaGet.Protocol/Search/AutocompleteContext.cs b/src/BaGet.Protocol/Search/AutocompleteContext.cs new file mode 100644 index 000000000..589aa10ba --- /dev/null +++ b/src/BaGet.Protocol/Search/AutocompleteContext.cs @@ -0,0 +1,15 @@ +using Newtonsoft.Json; + +namespace BaGet.Protocol +{ + public class AutocompleteContext + { + public static readonly AutocompleteContext Default = new AutocompleteContext + { + Vocab = "http://schema.nuget.org/schema#" + }; + + [JsonProperty("@vocab")] + public string Vocab { get; set; } + } +} diff --git a/src/BaGet.Protocol/Search/AutocompleteResult.cs b/src/BaGet.Protocol/Search/AutocompleteResult.cs index 775b02cd7..4c5278150 100644 --- a/src/BaGet.Protocol/Search/AutocompleteResult.cs +++ b/src/BaGet.Protocol/Search/AutocompleteResult.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; namespace BaGet.Protocol @@ -9,20 +9,26 @@ namespace BaGet.Protocol /// public class AutocompleteResult { - public AutocompleteResult(int totalHits, IReadOnlyList data) + public AutocompleteResult( + int totalHits, + IReadOnlyList data, + AutocompleteContext context = null) { TotalHits = totalHits; Data = data ?? throw new ArgumentNullException(nameof(data)); + Context = context; } + public AutocompleteContext Context { get; } + /// /// The total number of matches, disregarding skip and take. /// - public int TotalHits; + public int TotalHits { get; } /// /// The package IDs matched by the autocomplete query. /// - public IReadOnlyList Data; + public IReadOnlyList Data { get; } } } diff --git a/src/BaGet.Protocol/Search/SearchClient.cs b/src/BaGet.Protocol/Search/SearchClient.cs index 88de4ed48..7b56f27c2 100644 --- a/src/BaGet.Protocol/Search/SearchClient.cs +++ b/src/BaGet.Protocol/Search/SearchClient.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Net.Http; using System.Threading.Tasks; @@ -15,20 +15,16 @@ public SearchClient(HttpClient httpClient) public async Task GetSearchResultsAsync(string searchUrl) { - var response = await _httpClient.GetAsync(searchUrl); + var response = await _httpClient.DeserializeUrlAsync(searchUrl); - response.EnsureSuccessStatusCode(); - - return await response.Content.ReadAsAsync(); + return response.GetResultOrThrow(); } public async Task GetAutocompleteResultsAsync(string searchUrl) { - var response = await _httpClient.GetAsync(searchUrl); - - response.EnsureSuccessStatusCode(); + var response = await _httpClient.DeserializeUrlAsync(searchUrl); - return await response.Content.ReadAsAsync(); + return response.GetResultOrThrow(); } } } diff --git a/src/BaGet.Protocol/Search/SearchContext.cs b/src/BaGet.Protocol/Search/SearchContext.cs new file mode 100644 index 000000000..022663791 --- /dev/null +++ b/src/BaGet.Protocol/Search/SearchContext.cs @@ -0,0 +1,22 @@ +using Newtonsoft.Json; + +namespace BaGet.Protocol +{ + public class SearchContext + { + public static SearchContext Default(string registrationBaseUrl) + { + return new SearchContext + { + Vocab = "http://schema.nuget.org/schema#", + Base = registrationBaseUrl + }; + } + + [JsonProperty("@vocab")] + public string Vocab { get; set; } + + [JsonProperty("@base")] + public string Base { get; set; } + } +} diff --git a/src/BaGet.Protocol/Search/SearchResponse.cs b/src/BaGet.Protocol/Search/SearchResponse.cs index bf6efbbb4..3c6222130 100644 --- a/src/BaGet.Protocol/Search/SearchResponse.cs +++ b/src/BaGet.Protocol/Search/SearchResponse.cs @@ -1,5 +1,6 @@ -using System; +using System; using System.Collections.Generic; +using Newtonsoft.Json; namespace BaGet.Protocol { @@ -9,20 +10,27 @@ namespace BaGet.Protocol /// public class SearchResponse { - public SearchResponse(int totalHits, IReadOnlyList data) + public SearchResponse( + int totalHits, + IReadOnlyList data, + SearchContext context = null) { TotalHits = totalHits; Data = data ?? throw new ArgumentNullException(nameof(data)); + Context = context; } + [JsonProperty("@context")] + public SearchContext Context { get; } + /// /// The total number of matches, disregarding skip and take. /// - public int TotalHits; + public int TotalHits { get; } /// /// The packages that matched the search query. /// - public IReadOnlyList Data; + public IReadOnlyList Data { get; } } } diff --git a/src/BaGet.Protocol/ServiceIndex/ServiceIndexClient.cs b/src/BaGet.Protocol/ServiceIndex/ServiceIndexClient.cs index 57db884ff..e1b822e88 100644 --- a/src/BaGet.Protocol/ServiceIndex/ServiceIndexClient.cs +++ b/src/BaGet.Protocol/ServiceIndex/ServiceIndexClient.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Net.Http; using System.Threading.Tasks; @@ -18,11 +18,9 @@ public ServiceIndexClient(HttpClient httpClient) public async Task GetServiceIndexAsync(string indexUrl) { - var response = await _httpClient.GetAsync(indexUrl); + var response = await _httpClient.DeserializeUrlAsync(indexUrl); - response.EnsureSuccessStatusCode(); - - return await response.Content.ReadAsAsync(); + return response.GetResultOrThrow(); } } }