From f03ad4d6a2af2d02d4a957dd61b2c7113cfa770a Mon Sep 17 00:00:00 2001 From: Harel M Date: Wed, 13 Nov 2024 23:46:15 +0200 Subject: [PATCH] Add first class support to Wikidata in the backend (#2075) * Main commit which adds support to wikidata POIs. * Added TODO * Add support for iNature reference merge * Remove update mechanism * Add support for wikidata in website links * Improve text for merge algorithm * Add tests for wikidata and iNature merger * Improve coverage, remove todo comments. --- .../Controllers/PointsOfInterestController.cs | 17 +-- .../Executors/FeaturesMergeExecutor.cs | 88 ++++++++++- IsraelHiking.API/RegisterApi.cs | 1 + .../Services/Osm/DatabasesUpdaterService.cs | 23 +-- .../Services/Poi/IPointsOfInterestProvider.cs | 8 - .../Services/Poi/PointsOfInterestProvider.cs | 61 ++++---- .../Poi/WikidataPointsOfInterestAdapter.cs | 54 +++++++ IsraelHiking.Common/Strings.cs | 4 +- .../ElasticSearchGateway.cs | 12 -- .../OsmLatestFileGateway.cs | 2 +- IsraelHiking.DataAccess/RegisterDataAccess.cs | 1 + IsraelHiking.DataAccess/WikidataGateway.cs | 138 ++++++++++++++++++ .../IWikidataGateway.cs | 11 ++ .../IPointsOfInterestRepository.cs | 1 - .../PointsOfInterestControllerTests.cs | 16 +- .../Executors/FeaturesMergeExecutorTests.cs | 58 ++++++++ .../IsraelHiking.API.Tests.csproj | 2 +- .../Osm/DatabasesUpdaterServiceTests.cs | 13 +- .../Poi/PointsOfInterestProviderTests.cs | 20 +-- .../WikidataPointsOfInterestAdapterTests.cs | 57 ++++++++ .../WikipediaPointsOfInterestAdapterTests.cs | 5 +- .../ElasticSearchGatewayTests.cs | 9 -- .../WikidataGatewayTests.cs | 32 ++++ 23 files changed, 480 insertions(+), 153 deletions(-) create mode 100644 IsraelHiking.API/Services/Poi/WikidataPointsOfInterestAdapter.cs create mode 100644 IsraelHiking.DataAccess/WikidataGateway.cs create mode 100644 IsraelHiking.DataAccessInterfaces/IWikidataGateway.cs create mode 100644 Tests/IsraelHiking.API.Tests/Services/Poi/WikidataPointsOfInterestAdapterTests.cs create mode 100644 Tests/IsraelHiking.DataAccess.Tests/WikidataGatewayTests.cs diff --git a/IsraelHiking.API/Controllers/PointsOfInterestController.cs b/IsraelHiking.API/Controllers/PointsOfInterestController.cs index c405ba721..bd8b1b837 100644 --- a/IsraelHiking.API/Controllers/PointsOfInterestController.cs +++ b/IsraelHiking.API/Controllers/PointsOfInterestController.cs @@ -240,19 +240,14 @@ public Task GetClosestPoint(string location, string source, string lan [Route("updates/{lastModified}/")] [Route("updates/{lastModified}/{modifiedUntil}")] [HttpGet] - public async Task GetPointOfInterestUpdates(DateTime lastModified, DateTime? modifiedUntil) + [Obsolete("Remove by 5.2025")] + public UpdatesResponse GetPointOfInterestUpdates(DateTime lastModified, DateTime? modifiedUntil) { - var response = await _pointsOfInterestProvider.GetUpdates(lastModified, modifiedUntil ?? DateTime.Now); - var imageUrls = new List(); - foreach (var feature in response.Features) + return new UpdatesResponse { - var currentImageUrls = feature.Attributes.GetNames() - .Where(a => a.StartsWith(FeatureAttributes.IMAGE_URL)) - .Select(k => feature.Attributes[k].ToString()); - imageUrls.AddRange(currentImageUrls.ToList()); - } - response.Images = await _imageUrlStoreExecutor.GetAllImagesForUrls(imageUrls.ToArray()); - return response; + Features = Array.Empty(), + Images = Array.Empty() + }; } /// diff --git a/IsraelHiking.API/Executors/FeaturesMergeExecutor.cs b/IsraelHiking.API/Executors/FeaturesMergeExecutor.cs index d541e3db9..f251fda32 100644 --- a/IsraelHiking.API/Executors/FeaturesMergeExecutor.cs +++ b/IsraelHiking.API/Executors/FeaturesMergeExecutor.cs @@ -75,10 +75,13 @@ public FeaturesMergeExecutor(IOptions options, public List Merge(List osmFeatures, List externalFeatures) { AddAlternativeTitleToNatureReserves(osmFeatures); + externalFeatures = MergeWikipediaAndWikidataIntoWikidata(externalFeatures); externalFeatures = MergeWikipediaToOsmByWikipediaTags(osmFeatures, externalFeatures); - _logger.LogInformation($"Starting to sort features by importance: {osmFeatures.Count}"); + externalFeatures = MergeWikidataToOsmByWikidataTags(osmFeatures, externalFeatures); + externalFeatures = MergeINatureToOsmByINatureTags(osmFeatures, externalFeatures); + _logger.LogInformation($"Starting to sort OSM features by importance: {osmFeatures.Count}"); osmFeatures = osmFeatures.OrderBy(f => f, new FeatureComparer()).ToList(); - _logger.LogInformation($"Finished sorting features by importance: {osmFeatures.Count}"); + _logger.LogInformation($"Finished sorting OSM features by importance: {osmFeatures.Count}"); osmFeatures = MergePlaceNodes(osmFeatures); var namesAttributes = new List {FeatureAttributes.NAME, FeatureAttributes.MTB_NAME}; namesAttributes.AddRange(Languages.Array.Select(language => FeatureAttributes.NAME + ":" + language)); @@ -95,7 +98,7 @@ public List Merge(List osmFeatures, List externalF private List MergeOsmElementsByName(List orderedOsmFeatures, string nameAttribute) { - _logger.LogInformation($"Starting OSM merging by {nameAttribute}."); + _logger.LogInformation($"Starting OSM merging by {nameAttribute}, current items count: {orderedOsmFeatures.Count}"); var featureIdsToRemove = new ConcurrentBag(); var groupedByName = orderedOsmFeatures.Where(f => f.Attributes.Exists(nameAttribute)) .GroupBy(f => f.Attributes[nameAttribute].ToString()).ToList(); @@ -130,17 +133,16 @@ private List MergeOsmElementsByName(List orderedOsmFeatures, features.RemoveAt(0); } }); - _logger.LogInformation($"Finished processing geometries, removing items."); var list = featureIdsToRemove.ToHashSet(); orderedOsmFeatures = orderedOsmFeatures.Where(f => list.Contains(f.GetId()) == false).ToList(); - _logger.LogInformation($"Finished OSM merging by name: {orderedOsmFeatures.Count}"); + _logger.LogInformation($"Finished OSM merging by name, removed {list.Count} items, remaining OSM items: {orderedOsmFeatures.Count}"); return orderedOsmFeatures; } private List MergeExternalFeaturesToOsm(List osmFeatures, List externalFeatures) { var featureIdsToRemove = new HashSet(); - _logger.LogInformation("Starting external features merging by title into OSM."); + _logger.LogInformation($"Starting external features merging by title into OSM. Current OSM items: {osmFeatures.Count}, external features: {externalFeatures.Count}"); var titlesDictionary = new Dictionary>(); foreach (var osmFeature in osmFeatures) { @@ -172,7 +174,7 @@ private List MergeExternalFeaturesToOsm(List osmFeatures, Li } } externalFeatures = externalFeatures.Where(f => featureIdsToRemove.Contains(f.GetId()) == false).ToList(); - _logger.LogInformation("Finished external features merging by title into OSM. " + externalFeatures.Count); + _logger.LogInformation("Finished external features merging by title into OSM. Remaining external features: " + externalFeatures.Count); return externalFeatures; } @@ -196,7 +198,7 @@ private List MergePlaceNodes(List osmFeatures) } }); var list = featureIdsToRemove.ToList(); - WriteToBothLoggers($"Finished places merging. Merged places: {list.Count}"); + WriteToBothLoggers($"Finished places merging. Removed places entities: {list.Count}"); return osmFeatures.Where(f => list.Contains(f.GetId()) == false).ToList(); } @@ -522,6 +524,28 @@ private bool IsFeaturesTagsMismatched(IFeature target, IFeature source, string t source.Attributes.GetNames().Contains(tagName)); } + private List MergeWikipediaAndWikidataIntoWikidata(List externalFeatures) + { + WriteToBothLoggers("Starting joining Wikipedia and wikidata."); + var wikidataFeatures = externalFeatures.Where(f => f.Attributes[FeatureAttributes.POI_SOURCE].Equals(Sources.WIKIDATA)).ToList(); + var wikipediaFeatures = externalFeatures + .Where(f => f.Attributes[FeatureAttributes.POI_SOURCE].Equals(Sources.WIKIPEDIA)) + .ToDictionary(f => f.Attributes[FeatureAttributes.NAME], f => f); + var featureIdsToRemove = new HashSet(); + foreach (var wikidataFeature in wikidataFeatures) + { + var names = wikidataFeature.Attributes.GetNames().Where(n => n.StartsWith(FeatureAttributes.NAME)) + .Select(n => wikidataFeature.Attributes[n].ToString()); + foreach (var name in names.Where(n => wikipediaFeatures.ContainsKey(n))) + { + featureIdsToRemove.Add(wikipediaFeatures[name].GetId()); + MergeFeatures(wikidataFeature, wikipediaFeatures[name]); + } + } + WriteToBothLoggers($"Finished joining Wikipedia and wikidata. Merged features: {featureIdsToRemove.Count}"); + return externalFeatures.Where(f => featureIdsToRemove.Contains(f.GetId()) == false).ToList(); + } + private List MergeWikipediaToOsmByWikipediaTags(List osmFeatures, List externalFeatures) { WriteToBothLoggers("Starting joining Wikipedia markers."); @@ -549,6 +573,54 @@ private List MergeWikipediaToOsmByWikipediaTags(List osmFeat WriteToBothLoggers($"Finished joining Wikipedia markers. Merged features: {featureIdsToRemove.Count}"); return externalFeatures.Where(f => featureIdsToRemove.Contains(f.GetId()) == false).ToList(); } + + private List MergeWikidataToOsmByWikidataTags(List osmFeatures, List externalFeatures) + { + WriteToBothLoggers("Starting joining Wikidata markers."); + var featureIdsToRemove = new HashSet(); + var wikidataFeatures = externalFeatures.Where(f => f.Attributes[FeatureAttributes.POI_SOURCE].Equals(Sources.WIKIDATA)).ToList(); + var osmWikiFeatures = osmFeatures.Where(f => + f.Attributes.GetNames().Any(n => n == FeatureAttributes.WIKIDATA) && + f.Attributes[FeatureAttributes.POI_SOURCE].Equals(Sources.OSM)) + .ToList(); + foreach (var osmWikiFeature in osmWikiFeatures) + { + var wikidataId = osmWikiFeature.Attributes[FeatureAttributes.WIKIDATA].ToString(); + var wikiFeatureToRemove = wikidataFeatures.FirstOrDefault(f => f.Attributes[FeatureAttributes.ID].ToString() == wikidataId); + if (wikiFeatureToRemove == null) + { + continue; + } + featureIdsToRemove.Add(wikiFeatureToRemove.GetId()); + MergeFeatures(osmWikiFeature, wikiFeatureToRemove); + } + WriteToBothLoggers($"Finished joining Wikidata markers. Merged features: {featureIdsToRemove.Count}"); + return externalFeatures.Where(f => featureIdsToRemove.Contains(f.GetId()) == false).ToList(); + } + + private List MergeINatureToOsmByINatureTags(List osmFeatures, List externalFeatures) + { + WriteToBothLoggers("Starting joining iNature markers."); + var featureIdsToRemove = new HashSet(); + var iNatureFeatures = externalFeatures.Where(f => f.Attributes[FeatureAttributes.POI_SOURCE].Equals(Sources.INATURE)).ToList(); + var osmINatureFeatures = osmFeatures.Where(f => + f.Attributes.GetNames().Any(n => n == FeatureAttributes.INATURE_REF) && + f.Attributes[FeatureAttributes.POI_SOURCE].Equals(Sources.OSM)) + .ToList(); + foreach (var osmINatureFeature in osmINatureFeatures) + { + var iNaturePage = osmINatureFeature.Attributes[FeatureAttributes.INATURE_REF].ToString(); + var iNatureFeatureToRemove = iNatureFeatures.FirstOrDefault(f => f.Attributes[FeatureAttributes.NAME].ToString() == iNaturePage); + if (iNatureFeatureToRemove == null) + { + continue; + } + featureIdsToRemove.Add(iNatureFeatureToRemove.GetId()); + MergeFeatures(osmINatureFeature, iNatureFeatureToRemove); + } + WriteToBothLoggers($"Finished joining iNature markers. Merged features: {featureIdsToRemove.Count}"); + return externalFeatures.Where(f => featureIdsToRemove.Contains(f.GetId()) == false).ToList(); + } private void AddAlternativeTitleToNatureReserves(List features) { diff --git a/IsraelHiking.API/RegisterApi.cs b/IsraelHiking.API/RegisterApi.cs index 919f8b3f9..8ab7d249c 100644 --- a/IsraelHiking.API/RegisterApi.cs +++ b/IsraelHiking.API/RegisterApi.cs @@ -47,6 +47,7 @@ public static void AddIHMApi(this IServiceCollection services) services.AddTransient(); services.AddTransient(); services.AddTransient(); + services.AddTransient(); services.AddTransient(); services.AddSingleton(); // last one is the least important diff --git a/IsraelHiking.API/Services/Osm/DatabasesUpdaterService.cs b/IsraelHiking.API/Services/Osm/DatabasesUpdaterService.cs index 6be72ebdc..c30e59332 100644 --- a/IsraelHiking.API/Services/Osm/DatabasesUpdaterService.cs +++ b/IsraelHiking.API/Services/Osm/DatabasesUpdaterService.cs @@ -145,27 +145,8 @@ private async Task RebuildPointsOfInterest(RebuildContext rebuildContext) var externalFeatures = sources.Select(s => _externalSourcesRepository.GetExternalPoisBySource(s)).SelectMany(t => t.Result).ToList(); var features = _featuresMergeExecutor.Merge(osmFeaturesTask.Result, externalFeatures); _unauthorizedImageUrlsRemover.RemoveImages(features); - var exitingFeatures = await _pointsOfInterestRepository.GetAllPointsOfInterest(true); - _logger.LogInformation($"Adding deleted features to new ones, total merged features: {features.Count} total existing features including deleted: {exitingFeatures.Count} of them: {exitingFeatures.Count(f => f.Attributes.Exists(FeatureAttributes.POI_DELETED))}"); - var newFeaturesDictionary = features.ToDictionary(f => f.GetId(), f => f); - var deletedFeatures = exitingFeatures.Where(f => f.GetLastModified() <= rebuildContext.StartTime && !newFeaturesDictionary.ContainsKey(f.GetId())).ToArray(); - foreach (var deletedFeatureToMark in deletedFeatures) - { - if (!deletedFeatureToMark.Attributes.Exists(FeatureAttributes.POI_DELETED)) - { - deletedFeatureToMark.Attributes.Add(FeatureAttributes.POI_DELETED, true); - deletedFeatureToMark.SetLastModified(DateTime.Now); - _logger.LogInformation("Removed feature id: " + deletedFeatureToMark.GetId()); - } - } - var featuresToStore = features.Concat(deletedFeatures).ToList(); - _logger.LogInformation($"Added deleted features to new ones: {deletedFeatures.Length} total features to store: {featuresToStore.Count}"); - await _pointsOfInterestRepository.StorePointsOfInterestDataToSecondaryIndex(featuresToStore); - _logger.LogInformation("Getting all features added since rebuild started: " + rebuildContext.StartTime.ToLongTimeString()); - var addedFeaturesAfterRebuildStart = await _pointsOfInterestRepository.GetPointsOfInterestUpdates(rebuildContext.StartTime, DateTime.Now); - _logger.LogInformation("Got all features added since rebuild started: " + addedFeaturesAfterRebuildStart.Count); - await _pointsOfInterestRepository.StorePointsOfInterestDataToSecondaryIndex(addedFeaturesAfterRebuildStart); - _logger.LogInformation("Finished storing all features"); + await _pointsOfInterestRepository.StorePointsOfInterestDataToSecondaryIndex(features); + _logger.LogInformation("Finished storing all features " + features.Count); await _pointsOfInterestRepository.SwitchPointsOfInterestIndices(); _logger.LogInformation("Finished rebuilding POIs database."); } diff --git a/IsraelHiking.API/Services/Poi/IPointsOfInterestProvider.cs b/IsraelHiking.API/Services/Poi/IPointsOfInterestProvider.cs index 11ac83b70..7971a046e 100644 --- a/IsraelHiking.API/Services/Poi/IPointsOfInterestProvider.cs +++ b/IsraelHiking.API/Services/Poi/IPointsOfInterestProvider.cs @@ -58,14 +58,6 @@ public interface IPointsOfInterestProvider /// public Task GetClosestPoint(Coordinate location, string source, string language = ""); - /// - /// Get the all the points that were undated since the given date, and up until a given data - /// - /// The last modidifaction date that the client has - /// The end time of the updates to reduce response size - /// - public Task GetUpdates(DateTime lastModifiedDate, DateTime modifiedUntil); - /// /// Get all points from the OSM repository /// diff --git a/IsraelHiking.API/Services/Poi/PointsOfInterestProvider.cs b/IsraelHiking.API/Services/Poi/PointsOfInterestProvider.cs index e13f5a240..c964db468 100644 --- a/IsraelHiking.API/Services/Poi/PointsOfInterestProvider.cs +++ b/IsraelHiking.API/Services/Poi/PointsOfInterestProvider.cs @@ -2,7 +2,6 @@ using IsraelHiking.API.Executors; using IsraelHiking.API.Services.Osm; using IsraelHiking.Common; -using IsraelHiking.Common.Api; using IsraelHiking.Common.Configuration; using IsraelHiking.Common.Extensions; using IsraelHiking.DataAccessInterfaces; @@ -222,28 +221,36 @@ private void RemoveEmptyTagsAndWhiteSpaces(TagsCollectionBase tags) } } - private void SetWebsiteAndWikipediaTags(TagsCollectionBase tags, List urls) + private void SetWebsiteAndWikiTags(TagsCollectionBase tags, List urls) { - var regexp = new Regex(@"((https?://)|^)([a-z]+)(\.m)?\.wikipedia.org/wiki/(.*)"); + var wikipediaRegexp = new Regex(@"((https?://)|^)([a-z]+)(\.m)?\.wikipedia.org/wiki/(.*)"); + var wikidataRegexp = new Regex(@"((https?://)|^)([a-z]+)(\.m)?\.wikidata.org/wiki/(.*)"); + var nonWikipediaUrls = new List(); foreach (var url in urls) { - var match = regexp.Match(url ?? string.Empty); - if (!match.Success) + var matchWikipedia = wikipediaRegexp.Match(url ?? string.Empty); + if (matchWikipedia.Success) { - nonWikipediaUrls.Add(url); + var language = matchWikipedia.Groups[3].Value; + var pageTitle = Uri.UnescapeDataString(matchWikipedia.Groups[5].Value.Replace("_", " ")); + var key = FeatureAttributes.WIKIPEDIA + ":" + language; + tags.AddOrReplace(key, pageTitle); + key = FeatureAttributes.WIKIPEDIA; + pageTitle = language + ":" + pageTitle; + if (tags.ContainsKey(key) == false) + { + tags.Add(key, pageTitle); + } continue; } - var language = match.Groups[3].Value; - var pageTitle = Uri.UnescapeDataString(match.Groups[5].Value.Replace("_", " ")); - var key = FeatureAttributes.WIKIPEDIA + ":" + language; - tags.AddOrReplace(key, pageTitle); - key = FeatureAttributes.WIKIPEDIA; - pageTitle = language + ":" + pageTitle; - if (tags.ContainsKey(key) == false) + var matchWikidata = wikidataRegexp.Match(url ?? string.Empty); + if (matchWikidata.Success) { - tags.Add(key, pageTitle); + tags.AddOrReplace(FeatureAttributes.WIKIDATA, matchWikidata.Groups[5].Value); + continue; } + nonWikipediaUrls.Add(url); } SetMultipleValuesForTag(tags, FeatureAttributes.WEBSITE, nonWikipediaUrls.ToArray()); } @@ -276,22 +283,6 @@ public async Task GetClosestPoint(Coordinate location, string source, .FirstOrDefault(); } - /// - public async Task GetUpdates(DateTime lastModifiedDate, DateTime modifiedUntil) - { - var results = (lastModifiedDate.Year < 2010) - ? throw new ArgumentException("Last modified date must be higher than 2010", nameof(lastModifiedDate)) - : await _pointsOfInterestRepository.GetPointsOfInterestUpdates(lastModifiedDate, modifiedUntil); - var lastModified = await _pointsOfInterestRepository.GetLastSuccessfulRebuildTime(); - _elevationSetterExecutor.GeometryTo3D(results); - return new UpdatesResponse - { - Features = results.ToArray(), - LastModified = lastModified - }; - - } - /// public async Task GetFeatureById(string source, string id) { @@ -337,7 +328,7 @@ public async Task AddFeature(IFeature feature, IAuthClient osmGateway, Longitude = location.X, Tags = new TagsCollection() }; - SetWebsiteAndWikipediaTags(node.Tags, feature.Attributes.GetNames() + SetWebsiteAndWikiTags(node.Tags, feature.Attributes.GetNames() .Where(n => n.StartsWith(FeatureAttributes.WEBSITE)) .Select(p => feature.Attributes[p].ToString()) .ToList()); @@ -390,7 +381,7 @@ public async Task UpdateFeature(IFeature partialFeature, IAuthClient o locationWasUpdated = UpdateLocationIfNeeded(completeOsmGeo, location); } - await UpdateLists(partialFeature, completeOsmGeo, osmGateway, language); + await UpdateWebsitesAndImages(partialFeature, completeOsmGeo, osmGateway, language); RemoveEmptyTagsAndWhiteSpaces(completeOsmGeo.Tags); if (oldTags.SequenceEqual(completeOsmGeo.Tags.ToArray()) && @@ -420,7 +411,7 @@ await osmGateway.UploadToOsmWithRetries( /// The gateway to get the user details from /// The language to use for the tags /// - private async Task UpdateLists(IFeature partialFeature, ICompleteOsmGeo completeOsmGeo, IAuthClient osmGateway, string language) + private async Task UpdateWebsitesAndImages(IFeature partialFeature, ICompleteOsmGeo completeOsmGeo, IAuthClient osmGateway, string language) { var featureAfterTagsUpdates = ConvertOsmToFeature(completeOsmGeo); var existingUrls = featureAfterTagsUpdates.Attributes.GetNames() @@ -440,7 +431,7 @@ private async Task UpdateLists(IFeature partialFeature, ICompleteOsmGeo complete .Select(u => u.ToString()) .ToList(); var wikipediaTagsToRemove = new TagsCollection(); - SetWebsiteAndWikipediaTags(wikipediaTagsToRemove, urlsToRemove); + SetWebsiteAndWikiTags(wikipediaTagsToRemove, urlsToRemove); foreach(var urlToRemove in urlsToRemove) { existingUrls.Remove(urlToRemove); @@ -454,7 +445,7 @@ private async Task UpdateLists(IFeature partialFeature, ICompleteOsmGeo complete } } } - SetWebsiteAndWikipediaTags(completeOsmGeo.Tags, existingUrls.Distinct().ToList()); + SetWebsiteAndWikiTags(completeOsmGeo.Tags, existingUrls.Distinct().ToList()); var existingImages = featureAfterTagsUpdates.Attributes.GetNames() .Where(n => n.StartsWith(FeatureAttributes.IMAGE_URL)) diff --git a/IsraelHiking.API/Services/Poi/WikidataPointsOfInterestAdapter.cs b/IsraelHiking.API/Services/Poi/WikidataPointsOfInterestAdapter.cs new file mode 100644 index 000000000..0541fb5bb --- /dev/null +++ b/IsraelHiking.API/Services/Poi/WikidataPointsOfInterestAdapter.cs @@ -0,0 +1,54 @@ +using IsraelHiking.Common; +using IsraelHiking.Common.Extensions; +using IsraelHiking.DataAccessInterfaces; +using Microsoft.Extensions.Logging; +using NetTopologySuite.Features; +using NetTopologySuite.Geometries; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace IsraelHiking.API.Services.Poi; + +/// +/// Points of interest adapter for Wikidata data +/// +public class WikidataPointsOfInterestAdapter : IPointsOfInterestAdapter +{ + private readonly IWikidataGateway _wikidataGateway; + private readonly ILogger _logger; + + /// + /// Class constructor + /// + /// + /// + public WikidataPointsOfInterestAdapter(IWikidataGateway wikidataGateway, + ILogger logger) + { + _logger = logger; + _wikidataGateway = wikidataGateway; + } + + /// + public string Source => Sources.WIKIDATA; + + /// + public async Task> GetAll() + { + _logger.LogInformation("Starting getting Wikidata items for indexing."); + var startCoordinate = new Coordinate(34, 29); + var endCoordinate = new Coordinate(36, 34); + var allFeatures = await _wikidataGateway.GetByBoundingBox(startCoordinate, endCoordinate); + _logger.LogInformation($"Finished getting Wikidata items for indexing, got {allFeatures.Count} items."); + return allFeatures; + } + + /// + public async Task> GetUpdates(DateTime lastModifiedDate) + { + var features = await GetAll(); + return features.Where(f => f.GetLastModified() > lastModifiedDate).ToList(); + } +} \ No newline at end of file diff --git a/IsraelHiking.Common/Strings.cs b/IsraelHiking.Common/Strings.cs index 2f3d97363..9832c234a 100644 --- a/IsraelHiking.Common/Strings.cs +++ b/IsraelHiking.Common/Strings.cs @@ -49,6 +49,8 @@ public static class FeatureAttributes public const string IMAGE_URL = "image"; public const string WEBSITE = "website"; public const string WIKIPEDIA = "wikipedia"; + public const string WIKIDATA = "wikidata"; + public const string INATURE_REF = "ref:IL:inature"; public const string POI_PREFIX = "poi"; public const string POI_ID = POI_PREFIX + "Id"; public const string POI_SOURCE = POI_PREFIX + "Source"; @@ -94,8 +96,8 @@ public static class Sources public const string OSM = "OSM"; public const string NAKEB = "Nakeb"; public const string WIKIPEDIA = "Wikipedia"; + public const string WIKIDATA = "Wikidata"; public const string INATURE = "iNature"; - public const string COORDINATES = "Coordinates"; } public static class Languages diff --git a/IsraelHiking.DataAccess/ElasticSearchGateway.cs b/IsraelHiking.DataAccess/ElasticSearchGateway.cs index ec0cf7ae3..6f194d180 100644 --- a/IsraelHiking.DataAccess/ElasticSearchGateway.cs +++ b/IsraelHiking.DataAccess/ElasticSearchGateway.cs @@ -649,18 +649,6 @@ private GeoBoundingBoxQueryDescriptor ConvertToGeoBoundingBox(GeoBound ).Field($"{PROPERTIES}.{FeatureAttributes.POI_GEOLOCATION}"); } - public async Task> GetPointsOfInterestUpdates(DateTime lastModifiedDate, DateTime modifiedUntil) - { - var categories = Categories.Points.Concat(Categories.Routes).Select(c => c.ToLower()).ToArray(); - var response = await _elasticClient.SearchAsync(s => s.Index(OSM_POIS_ALIAS) - .Size(10000) - .Scroll("10s") - .Query(q => q.DateRange(t => t.Field($"{PROPERTIES}.{FeatureAttributes.POI_LAST_MODIFIED}").GreaterThan(lastModifiedDate).LessThanOrEquals(modifiedUntil)) - && q.Terms(t => t.Field($"{PROPERTIES}.{FeatureAttributes.POI_CATEGORY}").Terms(categories)) - )); - return GetAllItemsByScrolling(response); - } - public async Task GetPointOfInterestById(string id, string source) { var fullId = GeoJsonExtensions.GetId(source, id); diff --git a/IsraelHiking.DataAccess/OsmLatestFileGateway.cs b/IsraelHiking.DataAccess/OsmLatestFileGateway.cs index fd906b6ec..a4cbe91ec 100644 --- a/IsraelHiking.DataAccess/OsmLatestFileGateway.cs +++ b/IsraelHiking.DataAccess/OsmLatestFileGateway.cs @@ -35,7 +35,7 @@ public OsmLatestFileGateway(IOptions options, /// public async Task Get() { - // return await Task.FromResult(new MemoryStream(File.ReadAllBytes("/Users/harel/Downloads/israel-and-palestine-latest.osm.pbf")) as Stream); + // return await Task.FromResult(new MemoryStream(File.ReadAllBytes("/Users/harelmazor/Downloads/israel-and-palestine-latest.osm.pbf")) as Stream); var client = _httpClientFactory.CreateClient(); _logger.LogInformation($"Starting to fetch OSM file from {_options.OsmFileAddress}"); client.Timeout = TimeSpan.FromMinutes(20); diff --git a/IsraelHiking.DataAccess/RegisterDataAccess.cs b/IsraelHiking.DataAccess/RegisterDataAccess.cs index ac1e23e1e..aa874b06e 100644 --- a/IsraelHiking.DataAccess/RegisterDataAccess.cs +++ b/IsraelHiking.DataAccess/RegisterDataAccess.cs @@ -26,6 +26,7 @@ public static IServiceCollection AddIHMDataAccess(this IServiceCollection servic services.AddTransient(); services.AddTransient(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddTransient(); services.AddSingleton(); diff --git a/IsraelHiking.DataAccess/WikidataGateway.cs b/IsraelHiking.DataAccess/WikidataGateway.cs new file mode 100644 index 000000000..0c9b30654 --- /dev/null +++ b/IsraelHiking.DataAccess/WikidataGateway.cs @@ -0,0 +1,138 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading.Tasks; +using IsraelHiking.Common; +using IsraelHiking.Common.Extensions; +using IsraelHiking.DataAccessInterfaces; +using Microsoft.Extensions.Logging; +using NetTopologySuite.Features; +using NetTopologySuite.Geometries; +using NetTopologySuite.IO; + +namespace IsraelHiking.DataAccess; + +internal class WikidataLiteral +{ + [JsonPropertyName("type")] + public string Type { get; set; } + [JsonPropertyName("value")] + public string Value { get; set; } +} + +internal class WikidataBinding +{ + [JsonPropertyName("place")] + public WikidataLiteral Place{ get; set; } + [JsonPropertyName("location")] + public WikidataLiteral Location { get; set; } + [JsonPropertyName("links")] + public WikidataLiteral WikipediaLinks { get; set; } +} + +internal class WikidataResult +{ + [JsonPropertyName("bindings")] + public WikidataBinding[] Bindings { get; set; } +} + +internal class WikidataResults +{ + [JsonPropertyName("results")] + public WikidataResult Results { get; set; } +} +public class WikidataGateway : IWikidataGateway +{ + private const string WIKIDATA_LOGO = "https://upload.wikimedia.org/wikipedia/commons/thumb/6/66/Wikidata-logo-en.svg/128px-Wikidata-logo-en.svg.png"; + private const string QUERY_API = "https://query.wikidata.org/sparql?query="; + + private readonly WKTReader _wktReader; + private readonly IHttpClientFactory _httpClientFactory; + private readonly ILogger _logger; + + public WikidataGateway(IHttpClientFactory httpClientFactory, ILogger logger) + { + _httpClientFactory = httpClientFactory; + _logger = logger; + _wktReader = new WKTReader(); + } + + public async Task> GetByBoundingBox(Coordinate southWest, Coordinate northEast) + { + _logger.LogInformation($"Starting getting Wikidata items for coordinates: ({southWest.X}, {southWest.Y}), ({northEast.X}, {northEast.Y})"); + var query = "SELECT ?place ?location ?links WHERE {\n" + + " SERVICE wikibase:box {\n" + + " ?place wdt:P625 ?location.\n" + + $" bd:serviceParam wikibase:cornerWest \"Point({southWest.X} {southWest.Y})\"^^geo:wktLiteral.\n" + + $" bd:serviceParam wikibase:cornerEast \"Point({northEast.X} {northEast.Y})\"^^geo:wktLiteral.\n" + + " }\n"; + foreach (var language in Languages.Array) + { + query += " OPTIONAL {\n" + + " SERVICE wikibase:label {\n" + + $" bd:serviceParam wikibase:language \"{language}\".\n" + + " }\n" + + $" ?webRaw{language} schema:about ?place; schema:inLanguage \"{language}\"; schema:isPartOf ;\n" + + $" BIND(wikibase:decodeUri(STR(?webRaw{language})) AS ?{language}).\n" + + " }\n\n"; + } + var languagesCoalesce = Languages.Array.Select(l => "COALESCE(?" + l + ", \"\")"); + query += " BIND(CONCAT(" + string.Join(",\";\",", languagesCoalesce) + ") AS ?links).\n}"; + var client = _httpClientFactory.CreateClient(); + client.DefaultRequestHeaders.Add("User-Agent", Branding.USER_AGENT); + client.DefaultRequestHeaders.Add("Accept", "application/sparql-results+json"); + var response = await client.GetAsync(QUERY_API + Uri.EscapeDataString(query)); + var content = await response.Content.ReadAsStringAsync(); + var results = JsonSerializer.Deserialize(content); + var features = results.Results.Bindings.Select(b => + { + var point = _wktReader.Read(b.Location.Value); + if (string.IsNullOrEmpty(b.WikipediaLinks?.Value)) + { + return null; + } + var links = b.WikipediaLinks.Value.Split(";").Where(s => !string.IsNullOrEmpty(s)).ToArray(); + if (links.Length == 0) + { + return null; + } + var languagesTitlesAndLinks = links.Select(l => + ( + Language: l.Replace("https://", "").Split(".").First(), + Title: l.Split("/").Last().Replace("_", " "), + Link: l + )).ToArray(); + var feature = new Feature(point, new AttributesTable + { + {FeatureAttributes.ID, b.Place.Value.Split("/").Last()}, + {FeatureAttributes.NAME, languagesTitlesAndLinks.First().Title}, + {FeatureAttributes.POI_SOURCE, Sources.WIKIDATA}, + {FeatureAttributes.POI_CATEGORY, Categories.WIKIPEDIA}, + {FeatureAttributes.POI_LANGUAGE, languagesTitlesAndLinks.First().Language}, + {FeatureAttributes.POI_ICON, "icon-wikipedia-w"}, + {FeatureAttributes.POI_ICON_COLOR, "black"}, + {FeatureAttributes.POI_SEARCH_FACTOR, 1.0}, + {FeatureAttributes.POI_SOURCE_IMAGE_URL, WIKIDATA_LOGO} + }) as IFeature; + for (var index = 0; index < languagesTitlesAndLinks.Length; index++) + { + feature.Attributes.AddOrUpdate(FeatureAttributes.NAME + ":" + languagesTitlesAndLinks[index].Language, + languagesTitlesAndLinks[index].Title); + var posix = index > 0 ? index.ToString() : string.Empty; + // HM TODO: website is with "_" instead of space. + feature.Attributes.AddOrUpdate(FeatureAttributes.WEBSITE + posix, languagesTitlesAndLinks[index].Link); + feature.Attributes.AddOrUpdate(FeatureAttributes.POI_SOURCE_IMAGE_URL + posix, WIKIDATA_LOGO); + } + feature.SetLocation(point.Coordinate); + feature.SetTitles(); + feature.SetId(); + return feature; + }).Where(f => f != null).ToList(); + + _logger.LogInformation($"Finished getting Wikidata items, got {features.Count} items"); + return features; + } +} \ No newline at end of file diff --git a/IsraelHiking.DataAccessInterfaces/IWikidataGateway.cs b/IsraelHiking.DataAccessInterfaces/IWikidataGateway.cs new file mode 100644 index 000000000..70b7315e2 --- /dev/null +++ b/IsraelHiking.DataAccessInterfaces/IWikidataGateway.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; +using System.Threading.Tasks; +using NetTopologySuite.Features; +using NetTopologySuite.Geometries; + +namespace IsraelHiking.DataAccessInterfaces; + +public interface IWikidataGateway +{ + Task> GetByBoundingBox(Coordinate southWest, Coordinate northEast); +} \ No newline at end of file diff --git a/IsraelHiking.DataAccessInterfaces/Repositories/IPointsOfInterestRepository.cs b/IsraelHiking.DataAccessInterfaces/Repositories/IPointsOfInterestRepository.cs index 11bd2f90d..c9a5bb036 100644 --- a/IsraelHiking.DataAccessInterfaces/Repositories/IPointsOfInterestRepository.cs +++ b/IsraelHiking.DataAccessInterfaces/Repositories/IPointsOfInterestRepository.cs @@ -14,7 +14,6 @@ public interface IPointsOfInterestRepository Task UpdatePointsOfInterestData(List features); Task> GetPointsOfInterest(Coordinate northEast, Coordinate southWest, string[] categories, string language); Task> GetAllPointsOfInterest(bool withDeleted); - Task> GetPointsOfInterestUpdates(DateTime lastModifiedDate, DateTime modifiedUntil); Task GetPointOfInterestById(string id, string source); Task DeletePointOfInterestById(string id, string source); Task StoreRebuildContext(RebuildContext context); diff --git a/Tests/IsraelHiking.API.Tests/Controllers/PointsOfInterestControllerTests.cs b/Tests/IsraelHiking.API.Tests/Controllers/PointsOfInterestControllerTests.cs index 559ca4037..919bdac63 100644 --- a/Tests/IsraelHiking.API.Tests/Controllers/PointsOfInterestControllerTests.cs +++ b/Tests/IsraelHiking.API.Tests/Controllers/PointsOfInterestControllerTests.cs @@ -291,20 +291,10 @@ public void GetClosestPoint_ShouldGetTheClosesOsmPoint() [TestMethod] public void GetPointOfInterestUpdates_ShouldGetThem() { - _pointsOfInterestProvider.GetUpdates(Arg.Any(), Arg.Any()).Returns(new UpdatesResponse - { - Features = new[] {new Feature(null, new AttributesTable - { - {FeatureAttributes.IMAGE_URL, "imageUrl"} - })}, - Images = Array.Empty(), - LastModified = DateTime.Now - }); - - var results = _controller.GetPointOfInterestUpdates(DateTime.MinValue, DateTime.Now).Result; + var results = _controller.GetPointOfInterestUpdates(DateTime.MinValue, DateTime.Now); - _imagesUrlsStorageExecutor.Received(1).GetAllImagesForUrls(Arg.Any()); - Assert.AreEqual(1, results.Features.Length); + Assert.AreEqual(0, results.Features.Length); + Assert.AreEqual(0, results.Images.Length); } [TestMethod] diff --git a/Tests/IsraelHiking.API.Tests/Executors/FeaturesMergeExecutorTests.cs b/Tests/IsraelHiking.API.Tests/Executors/FeaturesMergeExecutorTests.cs index 82c5fb3df..3c5be95b1 100644 --- a/Tests/IsraelHiking.API.Tests/Executors/FeaturesMergeExecutorTests.cs +++ b/Tests/IsraelHiking.API.Tests/Executors/FeaturesMergeExecutorTests.cs @@ -300,6 +300,64 @@ public void MergeFeatures_OsmWithWikipediaTags_ShouldMerge() Assert.AreEqual(1, results.Count); } + [TestMethod] + public void MergeFeatures_OsmWithWikidataTags_ShouldMerge() + { + var feature1 = CreateFeature("1", 0, 0); + feature1.Attributes.AddOrUpdate(FeatureAttributes.NAME, "1"); + feature1.Attributes.AddOrUpdate(FeatureAttributes.WIKIDATA, "Q1234"); + feature1.SetTitles(); + feature1.SetLocation(feature1.Geometry.Coordinate); + var feature2 = CreateFeature("Q1234", 0, 0); + feature2.Attributes.AddOrUpdate(FeatureAttributes.NAME, "2"); + feature2.Attributes.AddOrUpdate(FeatureAttributes.POI_SOURCE, Sources.WIKIDATA); + feature2.SetTitles(); + feature2.SetLocation(feature1.Geometry.Coordinate); + var results = _executor.Merge(new List { feature1 }, new List { feature2 }); + + Assert.AreEqual(1, results.Count); + } + + [TestMethod] + public void MergeFeatures_WikidataAndWikipediaTags_ShouldMerge() + { + var feature1 = CreateFeature("1", 0, 0); + feature1.Attributes.AddOrUpdate(FeatureAttributes.NAME, "1"); + feature1.Attributes.AddOrUpdate(FeatureAttributes.WIKIPEDIA, "page"); + feature1.Attributes.AddOrUpdate(FeatureAttributes.POI_SOURCE, Sources.WIKIDATA); + feature1.SetTitles(); + feature1.SetLocation(feature1.Geometry.Coordinate); + var feature2 = CreateFeature("2", 0, 0); + feature2.Attributes.AddOrUpdate(FeatureAttributes.NAME, "1"); + feature2.Attributes.AddOrUpdate(FeatureAttributes.WIKIDATA, "page"); + feature2.Attributes.AddOrUpdate(FeatureAttributes.POI_SOURCE, Sources.WIKIPEDIA); + feature2.SetTitles(); + feature2.SetLocation(feature1.Geometry.Coordinate); + + var results = _executor.Merge(new List(), new List { feature1, feature2 }); + + Assert.AreEqual(1, results.Count); + } + + [TestMethod] + public void MergeFeatures_OsmWithINatureTags_ShouldMerge() + { + var feature1 = CreateFeature("1", 0, 0); + feature1.Attributes.AddOrUpdate(FeatureAttributes.NAME, "1"); + feature1.Attributes.AddOrUpdate(FeatureAttributes.INATURE_REF, "page"); + feature1.SetTitles(); + feature1.SetLocation(feature1.Geometry.Coordinate); + var feature2 = CreateFeature("2", 0, 0); + feature2.Attributes.AddOrUpdate(FeatureAttributes.NAME, "page"); + feature2.Attributes.AddOrUpdate(FeatureAttributes.POI_SOURCE, Sources.INATURE); + feature2.SetTitles(); + feature2.SetLocation(feature2.Geometry.Coordinate); + + var results = _executor.Merge(new List { feature1 }, new List { feature2 }); + + Assert.AreEqual(1, results.Count); + } + [TestMethod] public void MergeFeatures_TwoPolygonsAndPoint_ShouldMerge() { diff --git a/Tests/IsraelHiking.API.Tests/IsraelHiking.API.Tests.csproj b/Tests/IsraelHiking.API.Tests/IsraelHiking.API.Tests.csproj index 0d5170a1f..7c8c9cb5b 100644 --- a/Tests/IsraelHiking.API.Tests/IsraelHiking.API.Tests.csproj +++ b/Tests/IsraelHiking.API.Tests/IsraelHiking.API.Tests.csproj @@ -16,7 +16,7 @@ - + diff --git a/Tests/IsraelHiking.API.Tests/Services/Osm/DatabasesUpdaterServiceTests.cs b/Tests/IsraelHiking.API.Tests/Services/Osm/DatabasesUpdaterServiceTests.cs index 77742c6e6..b66842ab4 100644 --- a/Tests/IsraelHiking.API.Tests/Services/Osm/DatabasesUpdaterServiceTests.cs +++ b/Tests/IsraelHiking.API.Tests/Services/Osm/DatabasesUpdaterServiceTests.cs @@ -100,20 +100,12 @@ public void TestRebuild_Highways_ShouldRebuildHighwaysAndPoints() _pointsOfInterestRepository.StoreRebuildContext(Arg.Is(c => c.Succeeded == true)); } - [TestMethod] public void TestRebuild_Points_ShouldRebuildPointsWhileMarkingOneAsDeleted() + [TestMethod] public void TestRebuild_Points_ShouldRebuildPoints() { var adapter = Substitute.For(); adapter.GetAll().Returns(new List()); _pointsOfInterestAdapterFactory.GetAll().Returns(new[] {adapter}); _externalSourcesRepository.GetExternalPoisBySource(Arg.Any()).Returns(new List()); - var feature = new Feature(new Point(0, 0), new AttributesTable - { - {FeatureAttributes.NAME, "feature in database that needs to be deleted"}, - {FeatureAttributes.POI_ID, "42"} - }); - feature.SetLastModified(new DateTime(0)); - _pointsOfInterestRepository.GetAllPointsOfInterest(Arg.Any()).Returns(new List {feature}); - _pointsOfInterestRepository.GetPointsOfInterestUpdates(Arg.Any(), Arg.Any()).Returns(new List()); _featuresMergeExecutor.Merge(Arg.Any>(), Arg.Any>()).Returns(new List { new Feature(new Point(0,0), new AttributesTable { {FeatureAttributes.POI_ID, "1"}}) @@ -122,8 +114,7 @@ [TestMethod] public void TestRebuild_Points_ShouldRebuildPointsWhileMarkingOneAs _service.Rebuild(new UpdateRequest {PointsOfInterest = true}).Wait(); - _pointsOfInterestRepository.Received(2).StorePointsOfInterestDataToSecondaryIndex(Arg.Any>()); - _pointsOfInterestRepository.Received(1).StorePointsOfInterestDataToSecondaryIndex(Arg.Is>(l => l.Any(f => f.Attributes.Exists(FeatureAttributes.POI_DELETED)))); + _pointsOfInterestRepository.Received(1).StorePointsOfInterestDataToSecondaryIndex(Arg.Any>()); _pointsOfInterestRepository.Received(1).SwitchPointsOfInterestIndices(); _pointsOfInterestRepository.StoreRebuildContext(Arg.Is(c => c.Succeeded == true)); } diff --git a/Tests/IsraelHiking.API.Tests/Services/Poi/PointsOfInterestProviderTests.cs b/Tests/IsraelHiking.API.Tests/Services/Poi/PointsOfInterestProviderTests.cs index 76d2bf761..b1fb507fd 100644 --- a/Tests/IsraelHiking.API.Tests/Services/Poi/PointsOfInterestProviderTests.cs +++ b/Tests/IsraelHiking.API.Tests/Services/Poi/PointsOfInterestProviderTests.cs @@ -257,6 +257,7 @@ public void AddFeature_ShouldUpdateOsmAndElasticSearch() "8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU5ErkJggg=="); feature.Attributes.AddOrUpdate(FeatureAttributes.POI_ICON, _tagsHelper.GetCategoriesByGroup(Categories.POINTS_OF_INTEREST).First().Icon); feature.Attributes.AddOrUpdate(FeatureAttributes.WEBSITE, "he.wikipedia.org/wiki/%D7%AA%D7%9C_%D7%A9%D7%9C%D7%9D"); + feature.Attributes.AddOrUpdate(FeatureAttributes.WEBSITE + "1", "www.wikidata.org/wiki/Q19401334"); _imagesUrlsStorageExecutor.GetImageUrlIfExists(Arg.Any(), Arg.Any()).Returns((string)null); _pointsOfInterestRepository.GetPointOfInterestById(Arg.Any(), Arg.Any()).Returns(null as IFeature); @@ -265,6 +266,7 @@ public void AddFeature_ShouldUpdateOsmAndElasticSearch() Assert.IsNotNull(results); _pointsOfInterestRepository.Received(1).UpdatePointsOfInterestData(Arg.Any>()); gateway.Received().CreateElement(Arg.Any(), Arg.Is(x => x.Tags[FeatureAttributes.WIKIPEDIA + ":" + language].Contains("תל שלם"))); + gateway.Received().CreateElement(Arg.Any(), Arg.Is(x => x.Tags[FeatureAttributes.WIKIDATA].Equals("Q19401334"))); gateway.Received().CreateChangeset(Arg.Any()); gateway.Received().CloseChangeset(Arg.Any()); } @@ -494,24 +496,6 @@ public void GetClosestPoint_ShouldGetTheClosesOsmPoint() Assert.AreEqual(list.Last(), results); } - [TestMethod] - public void GetUpdates_TooOld_ShouldThrow() - { - Assert.ThrowsException(() => _adapter.GetUpdates(DateTime.MinValue, DateTime.Now).Result); - } - - [TestMethod] - public void GetUpdates_ShouldReturnThem() - { - _pointsOfInterestRepository.GetPointsOfInterestUpdates(Arg.Any(), Arg.Any()) - .Returns(new List()); - _pointsOfInterestRepository.GetLastSuccessfulRebuildTime().Returns(DateTime.Now); - - var results = _adapter.GetUpdates(DateTime.Now, DateTime.Now).Result; - - Assert.AreEqual(0, results.Features.Length); - } - [TestMethod] public void UpdateFeature_WithImageIdExists_ShouldUpdate() { diff --git a/Tests/IsraelHiking.API.Tests/Services/Poi/WikidataPointsOfInterestAdapterTests.cs b/Tests/IsraelHiking.API.Tests/Services/Poi/WikidataPointsOfInterestAdapterTests.cs new file mode 100644 index 000000000..3aff2127e --- /dev/null +++ b/Tests/IsraelHiking.API.Tests/Services/Poi/WikidataPointsOfInterestAdapterTests.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using IsraelHiking.API.Services.Poi; +using IsraelHiking.Common; +using IsraelHiking.Common.Extensions; +using IsraelHiking.DataAccessInterfaces; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using NetTopologySuite.Features; +using NetTopologySuite.Geometries; +using NSubstitute; + +namespace IsraelHiking.API.Tests.Services.Poi; + +[TestClass] +public class WikidataPointsOfInterestAdapterTests : BasePointsOfInterestAdapterTestsHelper +{ + private WikidataPointsOfInterestAdapter _adapter; + private IWikidataGateway _wikidataGateway; + + [TestInitialize] + public void TestInitialize() + { + InitializeSubstitutes(); + _wikidataGateway = Substitute.For(); + _adapter = new WikidataPointsOfInterestAdapter(_wikidataGateway, Substitute.For()); + } + + [TestMethod] + public void GetAll_ShouldGetAllPointsFromGateway() + { + var feature = GetValidFeature("1", Sources.WIKIDATA); + feature.SetId(); + var list = new List { feature }; + _wikidataGateway.GetByBoundingBox(Arg.Any(), Arg.Any()).Returns(list); + + var points = _adapter.GetAll().Result; + + _wikidataGateway.Received(1).GetByBoundingBox(Arg.Any(), Arg.Any()); + Assert.AreEqual(1, points.Count); // distinct by number of languages + } + + [TestMethod] + public void GetUpdates_ShouldGetAllPointsFromGateway() + { + var feature = GetValidFeature("1", Sources.WIKIDATA); + feature.SetId(); + feature.SetLastModified(DateTime.Now.AddDays(-1)); + var list = new List { feature }; + _wikidataGateway.GetByBoundingBox(Arg.Any(), Arg.Any()).Returns(list); + + var points = _adapter.GetUpdates(DateTime.Now).Result; + + _wikidataGateway.Received(1).GetByBoundingBox(Arg.Any(), Arg.Any()); + Assert.AreEqual(0, points.Count); // distinct by number of languages + } +} \ No newline at end of file diff --git a/Tests/IsraelHiking.API.Tests/Services/Poi/WikipediaPointsOfInterestAdapterTests.cs b/Tests/IsraelHiking.API.Tests/Services/Poi/WikipediaPointsOfInterestAdapterTests.cs index d149c688d..9a83d94d7 100644 --- a/Tests/IsraelHiking.API.Tests/Services/Poi/WikipediaPointsOfInterestAdapterTests.cs +++ b/Tests/IsraelHiking.API.Tests/Services/Poi/WikipediaPointsOfInterestAdapterTests.cs @@ -19,7 +19,7 @@ public class WikipediaPointsOfInterestAdapterTests : BasePointsOfInterestAdapter private IOverpassTurboGateway _overpassTurboGateway; [TestInitialize] - public void TestInialize() + public void TestInitialize() { InitializeSubstitutes(); _wikipediaGateway = Substitute.For(); @@ -40,8 +40,7 @@ public void GetAll_ShouldGetAllPointsFromGateway() var points = _adapter.GetAll().Result; _wikipediaGateway.Received(952).GetByBoundingBox(Arg.Any(), Arg.Any(), Arg.Any()); - Assert.AreEqual(Languages.Array.Length, points.Count); // distinct by number of lanugages + Assert.AreEqual(Languages.Array.Length, points.Count); // distinct by number of languages } - } } diff --git a/Tests/IsraelHiking.DataAccess.Tests/ElasticSearch/ElasticSearchGatewayTests.cs b/Tests/IsraelHiking.DataAccess.Tests/ElasticSearch/ElasticSearchGatewayTests.cs index 7d0931c14..79aedf96f 100644 --- a/Tests/IsraelHiking.DataAccess.Tests/ElasticSearch/ElasticSearchGatewayTests.cs +++ b/Tests/IsraelHiking.DataAccess.Tests/ElasticSearch/ElasticSearchGatewayTests.cs @@ -154,15 +154,6 @@ public void UpdatePointOfInterest_ShouldBeAbleToGetRightAfterAdding() Assert.IsNotNull(results); } - [TestMethod] - [Ignore] - public void GetPointsOfInterestUpdates_ShouldGetSome() - { - var results = _gateway.GetPointsOfInterestUpdates(DateTime.Now.AddDays(-50), DateTime.Now.AddDays(-45)).Result; - - Assert.IsNotNull(results); - } - [TestMethod] [Ignore] public void GetAllPointsOfInterest_ShouldGetThem() diff --git a/Tests/IsraelHiking.DataAccess.Tests/WikidataGatewayTests.cs b/Tests/IsraelHiking.DataAccess.Tests/WikidataGatewayTests.cs new file mode 100644 index 000000000..22ae227e8 --- /dev/null +++ b/Tests/IsraelHiking.DataAccess.Tests/WikidataGatewayTests.cs @@ -0,0 +1,32 @@ +using IsraelHiking.Common; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using NetTopologySuite.Geometries; +using System.Linq; +using System.Net.Http; +using NSubstitute; + +namespace IsraelHiking.DataAccess.Tests +{ + [TestClass] + public class WikidataGatewayTests + { + private WikidataGateway _gateway; + + [TestInitialize] + public void TestInitialize() + { + var factory = Substitute.For(); + factory.CreateClient().Returns(new HttpClient()); + _gateway = new WikidataGateway(factory, new TraceLogger()); + } + + [TestMethod] + [Ignore] + public void GetWikiPageByBoundingBox() + { + var delta = 0.01; + var results = _gateway.GetByBoundingBox(new Coordinate(35.057, 32.596), new Coordinate(35.057 + delta, 32.596 + delta)).Result; + Assert.IsTrue(results.Count > 0); + } + } +}