diff --git a/src/Orleans.Core.Abstractions/Manifest/GrainProperties.cs b/src/Orleans.Core.Abstractions/Manifest/GrainProperties.cs index 41181c248d..4f994e01f2 100644 --- a/src/Orleans.Core.Abstractions/Manifest/GrainProperties.cs +++ b/src/Orleans.Core.Abstractions/Manifest/GrainProperties.cs @@ -68,6 +68,11 @@ public static class WellKnownGrainTypeProperties /// public const string PlacementStrategy = "placement-strategy"; + /// + /// The name of the placement strategy for grains of this type. + /// + public const string PlacementFilter = "placement-filter"; + /// /// The directory policy for grains of this type. /// diff --git a/src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs b/src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs new file mode 100644 index 0000000000..0203d6439b --- /dev/null +++ b/src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using Orleans.Metadata; +using Orleans.Runtime; + +#nullable enable +namespace Orleans.Placement; + +/// +/// Base for all placement filter marker attributes. +/// +[AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] +public abstract class PlacementFilterAttribute : Attribute, IGrainPropertiesProviderAttribute +{ + public PlacementFilterStrategy PlacementFilterStrategy { get; private set; } + + protected PlacementFilterAttribute(PlacementFilterStrategy placement) + { + ArgumentNullException.ThrowIfNull(placement); + PlacementFilterStrategy = placement; + } + + /// + public virtual void Populate(IServiceProvider services, Type grainClass, GrainType grainType, Dictionary properties) + => PlacementFilterStrategy?.PopulateGrainProperties(services, grainClass, grainType, properties); +} \ No newline at end of file diff --git a/src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs b/src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs new file mode 100644 index 0000000000..eede6af61b --- /dev/null +++ b/src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using Orleans.Metadata; +using Orleans.Runtime; + +#nullable enable +namespace Orleans.Placement; + +public abstract class PlacementFilterStrategy +{ + public int Order { get; private set; } + + protected PlacementFilterStrategy(int order) + { + Order = order; + } + + /// + /// Initializes an instance of this type using the provided grain properties. + /// + /// + /// The grain properties. + /// + public void Initialize(GrainProperties properties) + { + var orderProperty = GetPlacementFilterGrainProperty("order", properties); + if (!int.TryParse(orderProperty, out var parsedOrder)) + { + throw new ArgumentException("Invalid order property value."); + } + + Order = parsedOrder; + + AdditionalInitialize(properties); + } + + public virtual void AdditionalInitialize(GrainProperties properties) + { + + } + + /// + /// Populates grain properties to specify the preferred placement strategy. + /// + /// The service provider. + /// The grain class. + /// The grain type. + /// The grain properties which will be populated by this method call. + public void PopulateGrainProperties(IServiceProvider services, Type grainClass, GrainType grainType, Dictionary properties) + { + var typeName = GetType().Name; + if (properties.TryGetValue(WellKnownGrainTypeProperties.PlacementFilter, out var existingValue)) + { + properties[WellKnownGrainTypeProperties.PlacementFilter] = $"{existingValue},{typeName}"; + } + else + { + properties[WellKnownGrainTypeProperties.PlacementFilter] = typeName; + } + + properties[$"{WellKnownGrainTypeProperties.PlacementFilter}.{typeName}.order"] = Order.ToString(); + + foreach (var additionalGrainProperty in GetAdditionalGrainProperties(services, grainClass, grainType, properties)) + { + properties[$"{WellKnownGrainTypeProperties.PlacementFilter}.{typeName}.{additionalGrainProperty.Key}"] = additionalGrainProperty.Value; + } + } + + protected string? GetPlacementFilterGrainProperty(string key, GrainProperties properties) + { + var typeName = GetType().Name; + return properties.Properties.TryGetValue($"{WellKnownGrainTypeProperties.PlacementFilter}.{typeName}.{key}", out var value) ? value : null; + } + + protected virtual IEnumerable> GetAdditionalGrainProperties(IServiceProvider services, Type grainClass, GrainType grainType, IReadOnlyDictionary existingProperties) + => Array.Empty>(); +} diff --git a/src/Orleans.Core/Placement/IPlacementFilterDirector.cs b/src/Orleans.Core/Placement/IPlacementFilterDirector.cs new file mode 100644 index 0000000000..fe9a225a0e --- /dev/null +++ b/src/Orleans.Core/Placement/IPlacementFilterDirector.cs @@ -0,0 +1,12 @@ +using System.Collections.Generic; +using Orleans.Runtime; +using Orleans.Runtime.Placement; + +#nullable enable +namespace Orleans.Placement; + +public interface IPlacementFilterDirector +{ + IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, + IEnumerable silos); +} \ No newline at end of file diff --git a/src/Orleans.Core/Placement/PlacementFilterExtensions.cs b/src/Orleans.Core/Placement/PlacementFilterExtensions.cs new file mode 100644 index 0000000000..7592656607 --- /dev/null +++ b/src/Orleans.Core/Placement/PlacementFilterExtensions.cs @@ -0,0 +1,26 @@ +using Microsoft.Extensions.DependencyInjection; + +#nullable enable +namespace Orleans.Placement; + +public static class PlacementFilterExtensions +{ + /// + /// Configures a for filtering candidate grain placements. + /// + /// The placement filter. + /// The placement filter director. + /// The service collection. + /// The lifetime of the placement strategy. + /// The service collection. + public static IServiceCollection AddPlacementFilter(this IServiceCollection services, ServiceLifetime strategyLifetime) + where TFilter : PlacementFilterStrategy + where TDirector : class, IPlacementFilterDirector + { + services.Add(ServiceDescriptor.DescribeKeyed(typeof(PlacementFilterStrategy), typeof(TFilter).Name, typeof(TFilter), strategyLifetime)); + services.AddKeyedSingleton(typeof(TFilter)); + + return services; + } + +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs b/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs index 67f98907cd..bba9fb3276 100644 --- a/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs +++ b/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs @@ -43,8 +43,7 @@ using Orleans.Serialization.Internal; using Orleans.Core; using Orleans.Placement.Repartitioning; -using Orleans.GrainDirectory; -using Orleans.Runtime.Hosting; +using Orleans.Runtime.Placement.Filtering; namespace Orleans.Hosting { @@ -206,6 +205,10 @@ internal static void AddDefaultServices(ISiloBuilder builder) // Configure the default placement strategy. services.TryAddSingleton(); + // Placement filters + services.AddSingleton(); + services.AddSingleton(); + // Placement directors services.AddPlacementDirector(); services.AddPlacementDirector(); diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs new file mode 100644 index 0000000000..96f5d5bbfb --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs @@ -0,0 +1,7 @@ +#nullable enable +namespace Orleans.Runtime.MembershipService.SiloMetadata; + +public interface ISiloMetadataCache +{ + SiloMetadata GetMetadata(SiloAddress siloAddress); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs new file mode 100644 index 0000000000..e2ab060bea --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs @@ -0,0 +1,10 @@ +using System.Threading.Tasks; +using Orleans.Services; + +#nullable enable +namespace Orleans.Runtime.MembershipService.SiloMetadata; + +public interface ISiloMetadataClient : IGrainServiceClient +{ + Task GetSiloMetadata(SiloAddress siloAddress); +} diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs new file mode 100644 index 0000000000..06121ae40c --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs @@ -0,0 +1,12 @@ +using System.Threading.Tasks; +using Orleans.Services; + +#nullable enable +namespace Orleans.Runtime.MembershipService.SiloMetadata; + +[Alias("Orleans.Runtime.MembershipService.SiloMetadata.ISiloMetadataGrainService")] +public interface ISiloMetadataGrainService : IGrainService +{ + [Alias("GetSiloMetadata")] + Task GetSiloMetadata(); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs new file mode 100644 index 0000000000..8483c570a1 --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs @@ -0,0 +1,103 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Orleans.Configuration; +using Orleans.Internal; + +#nullable enable +namespace Orleans.Runtime.MembershipService.SiloMetadata; + +internal class SiloMetadataCache( + ISiloMetadataClient siloMetadataClient, + MembershipTableManager membershipTableManager, + ILogger logger) + : ISiloMetadataCache, ILifecycleParticipant, IDisposable +{ + private readonly ConcurrentDictionary _metadata = new(); + private readonly CancellationTokenSource _cts = new(); + + void ILifecycleParticipant.Participate(ISiloLifecycle lifecycle) + { + var tasks = new List(1); + var cancellation = new CancellationTokenSource(); + Task OnRuntimeInitializeStart(CancellationToken _) + { + tasks.Add(Task.Run(() => this.ProcessMembershipUpdates(cancellation.Token))); + return Task.CompletedTask; + } + + async Task OnRuntimeInitializeStop(CancellationToken ct) + { + cancellation.Cancel(throwOnFirstException: false); + var shutdownGracePeriod = Task.WhenAll(Task.Delay(ClusterMembershipOptions.ClusteringShutdownGracePeriod), ct.WhenCancelled()); + await Task.WhenAny(shutdownGracePeriod, Task.WhenAll(tasks)); + } + + lifecycle.Subscribe( + nameof(ClusterMembershipService), + ServiceLifecycleStage.RuntimeInitialize, + OnRuntimeInitializeStart, + OnRuntimeInitializeStop); + } + + + private async Task ProcessMembershipUpdates(CancellationToken ct) + { + try + { + if (logger.IsEnabled(LogLevel.Debug)) logger.LogDebug("Starting to process membership updates"); + await foreach (var update in membershipTableManager.MembershipTableUpdates.WithCancellation(ct)) + { + // Add entries for members that aren't already in the cache + foreach (var membershipEntry in update.Entries.Where(e => e.Value.Status != SiloStatus.Dead)) + { + if (!_metadata.ContainsKey(membershipEntry.Key)) + { + try + { + var metadata = await siloMetadataClient.GetSiloMetadata(membershipEntry.Key); + _metadata.TryAdd(membershipEntry.Key, metadata); + } + catch(Exception exception) + { + logger.LogError(exception, "Error fetching metadata for silo {Silo}", membershipEntry.Key); + } + } + } + + // Remove entries for members that are now dead + foreach (var membershipEntry in update.Entries.Where(e => e.Value.Status == SiloStatus.Dead)) + { + _metadata.TryRemove(membershipEntry.Key, out _); + } + + // Remove entries for members that are no longer in the table + foreach (var silo in _metadata.Keys.ToList()) + { + if (!update.Entries.ContainsKey(silo)) + { + _metadata.TryRemove(silo, out _); + } + } + } + } + catch (Exception exception) + { + logger.LogError(exception, "Error processing membership updates"); + } + finally + { + if (logger.IsEnabled(LogLevel.Debug)) logger.LogDebug("Stopping membership update processor"); + } + } + + public SiloMetadata GetMetadata(SiloAddress siloAddress) => _metadata.GetValueOrDefault(siloAddress) ?? SiloMetadata.Empty; + + public void SetMetadata(SiloAddress siloAddress, SiloMetadata metadata) => _metadata.TryAdd(siloAddress, metadata); + + public void Dispose() => _cts.Cancel(); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs new file mode 100644 index 0000000000..278a7aa76a --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; +using System.Collections.Immutable; + +#nullable enable +namespace Orleans.Runtime.MembershipService.SiloMetadata; + +[GenerateSerializer] +[Alias("Orleans.Runtime.MembershipService.SiloMetadata.SiloMetadata")] +public record SiloMetadata +{ + public static SiloMetadata Empty { get; } = new SiloMetadata(); + + [Id(0)] + public ImmutableDictionary Metadata { get; private set; } = ImmutableDictionary.Empty; + + internal void AddMetadata(IEnumerable> metadata) => Metadata = Metadata.SetItems(metadata); + internal void AddMetadata(string key, string value) => Metadata = Metadata.SetItem(key, value); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs new file mode 100644 index 0000000000..5d507bde72 --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs @@ -0,0 +1,17 @@ +using System; +using System.Threading.Tasks; +using Orleans.Runtime.Services; + +#nullable enable +namespace Orleans.Runtime.MembershipService.SiloMetadata; + +public class SiloMetadataClient(IServiceProvider serviceProvider) + : GrainServiceClient(serviceProvider), ISiloMetadataClient +{ + public async Task GetSiloMetadata(SiloAddress siloAddress) + { + var grainService = GetGrainService(siloAddress); + var metadata = await grainService.GetSiloMetadata(); + return metadata; + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs new file mode 100644 index 0000000000..8e60bff08d --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs @@ -0,0 +1,23 @@ +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +#nullable enable +namespace Orleans.Runtime.MembershipService.SiloMetadata; + +public class SiloMetadataGrainService : GrainService, ISiloMetadataGrainService +{ + private readonly SiloMetadata _siloMetadata; + + public SiloMetadataGrainService(IOptions siloMetadata) : base() + { + _siloMetadata = siloMetadata.Value; + } + + public SiloMetadataGrainService(IOptions siloMetadata, GrainId grainId, Silo silo, ILoggerFactory loggerFactory) : base(grainId, silo, loggerFactory) + { + _siloMetadata = siloMetadata.Value; + } + + public Task GetSiloMetadata() => Task.FromResult(_siloMetadata); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs new file mode 100644 index 0000000000..a2adf33f55 --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs @@ -0,0 +1,92 @@ +using System.Collections.Generic; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Configuration.Internal; +using Orleans.Hosting; +using Orleans.Placement; +using Orleans.Runtime.Placement.Filtering; + +#nullable enable +namespace Orleans.Runtime.MembershipService.SiloMetadata; + +public static class SiloMetadataHostingExtensions +{ + + /// + /// Configure silo metadata from the builder configuration. + /// + /// Silo builder + /// + /// Get the ORLEANS__METADATA section from config + /// Key/value pairs in configuration as a will look like this as environment variables: + /// ORLEANS__METADATA__key1=value1 + /// + /// + public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder) => builder.UseSiloMetadata(builder.Configuration); + + /// + /// Configure silo metadata from configuration. + /// + /// Silo builder + /// Configuration to pull from + /// + /// Get the ORLEANS__METADATA section from config + /// Key/value pairs in configuration as a will look like this as environment variables: + /// ORLEANS__METADATA__key1=value1 + /// + /// + public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, IConfiguration configuration) + { + + var metadataConfigSection = configuration.GetSection("Orleans").GetSection("Metadata"); + + return builder.UseSiloMetadata(metadataConfigSection); + } + + /// + /// Configure silo metadata from configuration section. + /// + /// Silo builder + /// Configuration section to pull from + /// + /// Get the ORLEANS__METADATA section from config section + /// Key/value pairs in configuration as a will look like this as environment variables: + /// ORLEANS__METADATA__key1=value1 + /// + /// + public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, IConfigurationSection configurationSection) + { + var dictionary = configurationSection.Get>(); + + return builder.UseSiloMetadata(dictionary ?? new Dictionary()); + } + + /// + /// Configure silo metadata from configuration section. + /// + /// Silo builder + /// Metadata to add + /// + public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, Dictionary metadata) + { + builder.ConfigureServices(services => + { + services + .AddOptionsWithValidateOnStart() + .Configure(m => + { + m.AddMetadata(metadata); + }); + + services.AddGrainService(); + services.AddSingleton(); + services.AddFromExisting(); + services.AddFromExisting, SiloMetadataCache>(); + services.AddSingleton(); + // Placement filters + services.AddPlacementFilter(ServiceLifetime.Transient); + services.AddPlacementFilter(ServiceLifetime.Transient); + }); + return builder; + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs new file mode 100644 index 0000000000..f6ef6cba0a --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs @@ -0,0 +1,14 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Placement; + +#nullable enable +namespace Orleans.Runtime.Placement.Filtering; + +/// +/// Responsible for resolving an for a . +/// +public sealed class PlacementFilterDirectorResolver(IServiceProvider services) +{ + public IPlacementFilterDirector GetFilterDirector(PlacementFilterStrategy placementFilterStrategy) => services.GetRequiredKeyedService(placementFilterStrategy.GetType()); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs new file mode 100644 index 0000000000..41524681cf --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Metadata; +using Orleans.Placement; + +#nullable enable +namespace Orleans.Runtime.Placement.Filtering; + +/// +/// Responsible for resolving an for a . +/// +public sealed class PlacementFilterStrategyResolver +{ + private readonly ConcurrentDictionary _resolvedFilters = new(); + private readonly Func _getFiltersInternal; + private readonly GrainPropertiesResolver _grainPropertiesResolver; + private readonly IServiceProvider _services; + + /// + /// Create a instance. + /// + public PlacementFilterStrategyResolver( + IServiceProvider services, + GrainPropertiesResolver grainPropertiesResolver) + { + _services = services; + _getFiltersInternal = GetPlacementFilterStrategyInternal; + _grainPropertiesResolver = grainPropertiesResolver; + } + + /// + /// Gets the placement filter strategy associated with the provided grain type. + /// + public PlacementFilterStrategy[] GetPlacementFilterStrategies(GrainType grainType) => _resolvedFilters.GetOrAdd(grainType, _getFiltersInternal); + + private PlacementFilterStrategy[] GetPlacementFilterStrategyInternal(GrainType grainType) + { + _grainPropertiesResolver.TryGetGrainProperties(grainType, out var properties); + + if (properties is not null + && properties.Properties.TryGetValue(WellKnownGrainTypeProperties.PlacementFilter, out var placementFilterIds) + && !string.IsNullOrWhiteSpace(placementFilterIds)) + { + var filterList = new List(); + foreach (var filterId in placementFilterIds.Split(",")) + { + var filter = _services.GetKeyedService(filterId); + if (filter is not null) + { + filter.Initialize(properties); + filterList.Add(filter); + } + else + { + throw new KeyNotFoundException($"Could not resolve placement filter strategy {filterId} for grain type {grainType}. Ensure that dependencies for that filter have been configured in the Container. This is often through a .Use* extension method provided by the implementation."); + } + } + + var orderedFilters = filterList.OrderBy(f => f.Order).ToArray(); + // check that the order is unique + if (orderedFilters.Select(f => f.Order).Distinct().Count() != orderedFilters.Length) + { + throw new InvalidOperationException($"Placement filters for grain type {grainType} have duplicate order values. Order values must be specified if more than one filter is applied and must be unique."); + } + return orderedFilters; + } + + return []; + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs new file mode 100644 index 0000000000..bde94878c9 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs @@ -0,0 +1,17 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using Orleans.Placement; + +#nullable enable +namespace Orleans.Runtime.Placement.Filtering; + +/// +/// Attribute to specify the preferred match silo metadata placement filter that preferentially filters down to silos where the metadata matches the local (calling) silo metadata. +/// +/// Ordered set of metadata keys to try to match. The earlier entries are considered less important and will be dropped to find a less-specific match if sufficient more-specific matches do not have enough results. +/// The minimum desired candidates to filter. This is to balance meeting the metadata preferences with not overloading a single or small set of silos with activations. Set this to 1 if you only want the best matches, even if there's only one silo that is currently the best match. +/// Example: If keys ["first","second"] are specified, then it will attempt to return only silos where both keys match the local silo's metadata values. If there are not sufficient silos matching both, then it will also include silos matching only the second key. Finally, if there are still fewer than minCandidates results then it will include all silos. +[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] +[Experimental("ORLEANSEXP004")] +public class PreferredMatchSiloMetadataPlacementFilterAttribute(string[] orderedMetadataKeys, int minCandidates = 2, int order = 0) + : PlacementFilterAttribute(new PreferredMatchSiloMetadataPlacementFilterStrategy(orderedMetadataKeys, minCandidates, order)); \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs new file mode 100644 index 0000000000..7b92a41c0e --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Orleans.Placement; +using Orleans.Runtime.MembershipService.SiloMetadata; + +#nullable enable +namespace Orleans.Runtime.Placement.Filtering; + +internal class PreferredMatchSiloMetadataPlacementFilterDirector( + ILocalSiloDetails localSiloDetails, + ISiloMetadataCache siloMetadataCache) + : IPlacementFilterDirector +{ + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + { + var preferredMatchSiloMetadataPlacementFilterStrategy = (filterStrategy as PreferredMatchSiloMetadataPlacementFilterStrategy); + var minCandidates = preferredMatchSiloMetadataPlacementFilterStrategy?.MinCandidates ?? 1; + var orderedMetadataKeys = preferredMatchSiloMetadataPlacementFilterStrategy?.OrderedMetadataKeys ?? []; + + var localSiloMetadata = siloMetadataCache.GetMetadata(localSiloDetails.SiloAddress).Metadata; + + if (localSiloMetadata.Count == 0) + { + return silos; + } + + var siloList = silos.ToList(); + if (siloList.Count <= minCandidates) + { + return siloList; + } + + // return the list of silos that match the most metadata keys. The first key in the list is the least important. + // This means that the last key in the list is the most important. + // If no silos match any metadata keys, return the original list of silos. + var maxScore = 0; + var siloScores = new int[siloList.Count]; + var scoreCounts = new int[orderedMetadataKeys.Length+1]; + for (var i = 0; i < siloList.Count; i++) + { + var siloMetadata = siloMetadataCache.GetMetadata(siloList[i]).Metadata; + var siloScore = 0; + for (var j = orderedMetadataKeys.Length - 1; j >= 0; --j) + { + if (siloMetadata.TryGetValue(orderedMetadataKeys[j], out var siloMetadataValue) && + localSiloMetadata.TryGetValue(orderedMetadataKeys[j], out var localSiloMetadataValue) && + siloMetadataValue == localSiloMetadataValue) + { + siloScore = ++siloScores[i]; + maxScore = Math.Max(maxScore, siloScore); + } + else + { + break; + } + } + scoreCounts[siloScore]++; + } + + if (maxScore == 0) + { + return siloList; + } + + var candidateCount = 0; + var scoreCutOff = orderedMetadataKeys.Length; + for (var i = scoreCounts.Length-1; i >= 0; i--) + { + candidateCount += scoreCounts[i]; + if (candidateCount >= minCandidates) + { + scoreCutOff = i; + break; + } + } + + return siloList.Where((_, i) => siloScores[i] >= scoreCutOff); + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs new file mode 100644 index 0000000000..e6a160adef --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using Orleans.Metadata; +using Orleans.Placement; + +#nullable enable +namespace Orleans.Runtime.Placement.Filtering; + +public class PreferredMatchSiloMetadataPlacementFilterStrategy(string[] orderedMetadataKeys, int minCandidates, int order) + : PlacementFilterStrategy(order) +{ + public string[] OrderedMetadataKeys { get; set; } = orderedMetadataKeys; + public int MinCandidates { get; set; } = minCandidates; + + public PreferredMatchSiloMetadataPlacementFilterStrategy() : this([], 2, 0) + { + } + + public override void AdditionalInitialize(GrainProperties properties) + { + var placementFilterGrainProperty = GetPlacementFilterGrainProperty("ordered-metadata-keys", properties); + if (placementFilterGrainProperty is null) + { + throw new ArgumentException("Invalid ordered-metadata-keys property value."); + } + OrderedMetadataKeys = placementFilterGrainProperty.Split(","); + var minCandidatesProperty = GetPlacementFilterGrainProperty("min-candidates", properties); + if (!int.TryParse(minCandidatesProperty, out var parsedMinCandidates)) + { + throw new ArgumentException("Invalid min-candidates property value."); + } + + MinCandidates = parsedMinCandidates; + } + + protected override IEnumerable> GetAdditionalGrainProperties(IServiceProvider services, Type grainClass, GrainType grainType, + IReadOnlyDictionary existingProperties) + { + yield return new KeyValuePair("ordered-metadata-keys", string.Join(",", OrderedMetadataKeys)); + yield return new KeyValuePair("min-candidates", MinCandidates.ToString()); + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs new file mode 100644 index 0000000000..ec317ad2c6 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs @@ -0,0 +1,15 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using Orleans.Placement; + +#nullable enable +namespace Orleans.Runtime.Placement.Filtering; + +/// +/// Attribute to specify that a silo must have a specific metadata key-value pair matching the local (calling) silo to be considered for placement. +/// +/// +[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] +[Experimental("ORLEANSEXP004")] +public class RequiredMatchSiloMetadataPlacementFilterAttribute(string[] metadataKeys, int order = 0) + : PlacementFilterAttribute(new RequiredMatchSiloMetadataPlacementFilterStrategy(metadataKeys, order)); \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs new file mode 100644 index 0000000000..91ed8cbac7 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs @@ -0,0 +1,53 @@ +using System.Collections.Generic; +using System.Linq; +using Orleans.Placement; +using Orleans.Runtime.MembershipService.SiloMetadata; + +#nullable enable +namespace Orleans.Runtime.Placement.Filtering; + +internal class RequiredMatchSiloMetadataPlacementFilterDirector(ILocalSiloDetails localSiloDetails, ISiloMetadataCache siloMetadataCache) + : IPlacementFilterDirector +{ + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + { + var metadataKeys = (filterStrategy as RequiredMatchSiloMetadataPlacementFilterStrategy)?.MetadataKeys ?? []; + + // yield return all silos if no silos match any metadata keys + if (metadataKeys.Length == 0) + { + return silos; + } + + var localMetadata = siloMetadataCache.GetMetadata(localSiloDetails.SiloAddress); + var localRequiredMetadata = GetMetadata(localMetadata, metadataKeys); + + return silos.Where(silo => + { + var remoteMetadata = siloMetadataCache.GetMetadata(silo); + return DoesMetadataMatch(localRequiredMetadata, remoteMetadata, metadataKeys); + }); + } + + private static bool DoesMetadataMatch(string?[] localMetadata, SiloMetadata siloMetadata, string[] metadataKeys) + { + for (var i = 0; i < metadataKeys.Length; i++) + { + if(localMetadata[i] != siloMetadata.Metadata?.GetValueOrDefault(metadataKeys[i])) + { + return false; + } + } + + return true; + } + private static string?[] GetMetadata(SiloMetadata siloMetadata, string[] metadataKeys) + { + var result = new string?[metadataKeys.Length]; + for (var i = 0; i < metadataKeys.Length; i++) + { + result[i] = siloMetadata.Metadata?.GetValueOrDefault(metadataKeys[i]); + } + return result; + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs new file mode 100644 index 0000000000..c2c19a995b --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; +using Orleans.Metadata; +using Orleans.Placement; + +#nullable enable +namespace Orleans.Runtime.Placement.Filtering; + +public class RequiredMatchSiloMetadataPlacementFilterStrategy(string[] metadataKeys, int order) + : PlacementFilterStrategy(order) +{ + public string[] MetadataKeys { get; private set; } = metadataKeys; + + public RequiredMatchSiloMetadataPlacementFilterStrategy() : this([], 0) + { + } + + public override void AdditionalInitialize(GrainProperties properties) + { + var placementFilterGrainProperty = GetPlacementFilterGrainProperty("metadata-keys", properties); + if (placementFilterGrainProperty is null) + { + throw new ArgumentException("Invalid metadata-keys property value."); + } + MetadataKeys = placementFilterGrainProperty.Split(","); + } + + protected override IEnumerable> GetAdditionalGrainProperties(IServiceProvider services, Type grainClass, GrainType grainType, + IReadOnlyDictionary existingProperties) + { + yield return new KeyValuePair("metadata-keys", String.Join(",", MetadataKeys)); + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/PlacementService.cs b/src/Orleans.Runtime/Placement/PlacementService.cs index cd17709a05..4688ccafeb 100644 --- a/src/Orleans.Runtime/Placement/PlacementService.cs +++ b/src/Orleans.Runtime/Placement/PlacementService.cs @@ -9,6 +9,7 @@ using Orleans.Configuration; using Orleans.Runtime.GrainDirectory; using Orleans.Runtime.Internal; +using Orleans.Runtime.Placement.Filtering; using Orleans.Runtime.Versions; namespace Orleans.Runtime.Placement @@ -28,6 +29,8 @@ internal class PlacementService : IPlacementContext private readonly ISiloStatusOracle _siloStatusOracle; private readonly bool _assumeHomogeneousSilosForTesting; private readonly PlacementWorker[] _workers; + private readonly PlacementFilterStrategyResolver _filterStrategyResolver; + private readonly PlacementFilterDirectorResolver _placementFilterDirectoryResolver; /// /// Create a instance. @@ -41,11 +44,15 @@ public PlacementService( GrainVersionManifest grainInterfaceVersions, CachedVersionSelectorManager versionSelectorManager, PlacementDirectorResolver directorResolver, - PlacementStrategyResolver strategyResolver) + PlacementStrategyResolver strategyResolver, + PlacementFilterStrategyResolver filterStrategyResolver, + PlacementFilterDirectorResolver placementFilterDirectoryResolver) { LocalSilo = localSiloDetails.SiloAddress; _strategyResolver = strategyResolver; _directorResolver = directorResolver; + _filterStrategyResolver = filterStrategyResolver; + _placementFilterDirectoryResolver = placementFilterDirectoryResolver; _logger = logger; _grainLocator = grainLocator; _grainInterfaceVersions = grainInterfaceVersions; @@ -117,6 +124,22 @@ public SiloAddress[] GetCompatibleSilos(PlacementTarget target) : _grainInterfaceVersions.GetSupportedSilos(grainType).Result; var compatibleSilos = silos.Intersect(AllActiveSilos).ToArray(); + + + var filters = _filterStrategyResolver.GetPlacementFilterStrategies(grainType); + if (filters.Length > 0) + { + IEnumerable filteredSilos = compatibleSilos; + foreach (var placementFilter in filters) + { + var director = _placementFilterDirectoryResolver.GetFilterDirector(placementFilter); + filteredSilos = director.Filter(placementFilter, target, filteredSilos); + } + + compatibleSilos = filteredSilos.ToArray(); + } + + if (compatibleSilos.Length == 0) { var allWithType = _grainInterfaceVersions.GetSupportedSilos(grainType).Result; diff --git a/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs b/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs new file mode 100644 index 0000000000..ce80d6ae5b --- /dev/null +++ b/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs @@ -0,0 +1,266 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Placement; +using Orleans.Runtime.Placement; +using Orleans.TestingHost; +using TestExtensions; +using Xunit; + +namespace UnitTests.PlacementFilterTests; + +[TestCategory("Placement"), TestCategory("Filters")] +public class GrainPlacementFilterTests : TestClusterPerTest +{ + public static Dictionary> FilterScratchpad = new(); + private static Random random = new(); + + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.AddSiloBuilderConfigurator(); + } + + private class SiloConfigurator : ISiloConfigurator + { + public void Configure(ISiloBuilder hostBuilder) + { + hostBuilder.ConfigureServices(services => + { + services.AddPlacementFilter(ServiceLifetime.Singleton); + services.AddPlacementFilter(ServiceLifetime.Singleton); + services.AddPlacementFilter(ServiceLifetime.Singleton); + }); + } + } + + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_GrainWithoutFilterCanBeCalled() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var managementGrain = Client.GetGrain(0); + var silos = await managementGrain.GetHosts(true); + Assert.NotNull(silos); + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_FilterIsTriggered() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var triggered = false; + var task = Task.Run(async () => + { + triggered = await TestPlacementFilterDirector.Triggered.WaitAsync(TimeSpan.FromSeconds(1)); + }); + var localOnlyGrain = Client.GetGrain(0); + await localOnlyGrain.Ping(); + await task; + Assert.True(triggered); + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_OrderAB12() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + + var primaryKey = random.Next(); + var testGrain = Client.GetGrain(primaryKey); + await testGrain.Ping(); + var list = FilterScratchpad.GetValueOrAddNew(testGrain.GetGrainId().ToString()); + Assert.Equal(2, list.Count); + Assert.Equal("A", list[0]); + Assert.Equal("B", list[1]); + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_OrderAB21() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + + var primaryKey = random.Next(); + var testGrain = Client.GetGrain(primaryKey); + await testGrain.Ping(); + var list = FilterScratchpad.GetValueOrAddNew(testGrain.GetGrainId().ToString()); + Assert.Equal(2, list.Count); + Assert.Equal("B", list[0]); + Assert.Equal("A", list[1]); + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_OrderBA12() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + + var primaryKey = random.Next(); + var testGrain = Client.GetGrain(primaryKey); + await testGrain.Ping(); + var list = FilterScratchpad.GetValueOrAddNew(testGrain.GetGrainId().ToString()); + Assert.Equal(2, list.Count); + Assert.Equal("B", list[0]); + Assert.Equal("A", list[1]); + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_OrderBA21() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + + var primaryKey = random.Next(); + var testGrain = Client.GetGrain(primaryKey); + await testGrain.Ping(); + + var list = FilterScratchpad.GetValueOrAddNew(testGrain.GetGrainId().ToString()); + Assert.Equal(2, list.Count); + Assert.Equal("A", list[0]); + Assert.Equal("B", list[1]); + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_DuplicateOrder() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + + var primaryKey = random.Next(); + var testGrain = Client.GetGrain(primaryKey); + await Assert.ThrowsAsync(async () => + { + await testGrain.Ping(); + }); + } +} + +[TestPlacementFilter(order: 1)] +public class TestFilteredGrain : Grain, ITestFilteredGrain +{ + public Task Ping() => Task.CompletedTask; +} + +public interface ITestFilteredGrain : IGrainWithIntegerKey +{ + Task Ping(); +} + +public class TestPlacementFilterAttribute(int order) : PlacementFilterAttribute(new TestPlacementFilterStrategy(order)); + +public class TestPlacementFilterStrategy(int order) : PlacementFilterStrategy(order) +{ + public TestPlacementFilterStrategy() : this(0) + { + } +} + +public class TestPlacementFilterDirector() : IPlacementFilterDirector +{ + public static SemaphoreSlim Triggered { get; } = new(0); + + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + { + Triggered.Release(1); + return silos; + } +} + + + +public class OrderAPlacementFilterAttribute(int order) : PlacementFilterAttribute(new OrderAPlacementFilterStrategy(order)); + +public class OrderAPlacementFilterStrategy(int order) : PlacementFilterStrategy(order) +{ + public OrderAPlacementFilterStrategy() : this(0) + { + } +} + +public class OrderAPlacementFilterDirector : IPlacementFilterDirector +{ + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + { + var dict = GrainPlacementFilterTests.FilterScratchpad; + var list = dict.GetValueOrAddNew(target.GrainIdentity.ToString()); + list.Add("A"); + return silos; + } +} + + +public class OrderBPlacementFilterAttribute(int order) : PlacementFilterAttribute(new OrderBPlacementFilterStrategy(order)); + +public class OrderBPlacementFilterStrategy(int order) : PlacementFilterStrategy(order) +{ + + public OrderBPlacementFilterStrategy() : this(0) + { + } +} + +public class OrderBPlacementFilterDirector() : IPlacementFilterDirector +{ + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + { + var dict = GrainPlacementFilterTests.FilterScratchpad; + var list = dict.GetValueOrAddNew(target.GrainIdentity.ToString()); + list.Add("B"); + return silos; + } +} + +[OrderAPlacementFilter(order: 1)] +[OrderBPlacementFilter(order: 2)] +public class TestAB12FilteredGrain : Grain, ITestAB12FilteredGrain +{ + public Task Ping() => Task.CompletedTask; +} + +public interface ITestAB12FilteredGrain : IGrainWithIntegerKey +{ + Task Ping(); +} + +[OrderAPlacementFilter(order: 2)] +[OrderBPlacementFilter(order: 1)] +public class TestAB21FilteredGrain : Grain, ITestAB21FilteredGrain +{ + public Task Ping() => Task.CompletedTask; +} + +public interface ITestAB21FilteredGrain : IGrainWithIntegerKey +{ + Task Ping(); +} + +[OrderBPlacementFilter(order: 1)] +[OrderAPlacementFilter(order: 2)] +public class TestBA12FilteredGrain : Grain, ITestBA12FilteredGrain +{ + public Task Ping() => Task.CompletedTask; +} + +public interface ITestBA12FilteredGrain : IGrainWithIntegerKey +{ + Task Ping(); +} + + +[OrderBPlacementFilter(order: 2)] +[OrderAPlacementFilter(order: 1)] +public class TestBA121FilteredGrain : Grain, ITestBA21FilteredGrain +{ + public Task Ping() => Task.CompletedTask; +} + +public interface ITestBA21FilteredGrain : IGrainWithIntegerKey +{ + Task Ping(); +} + + + +[OrderBPlacementFilter(order: 2)] +[OrderAPlacementFilter(order: 2)] +public class TestDuplicateOrderFilteredGrain : Grain, ITestDuplicateOrderFilteredGrain +{ + public Task Ping() => Task.CompletedTask; +} + +public interface ITestDuplicateOrderFilteredGrain : IGrainWithIntegerKey +{ + Task Ping(); +} \ No newline at end of file diff --git a/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs b/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs new file mode 100644 index 0000000000..369829fb7b --- /dev/null +++ b/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs @@ -0,0 +1,176 @@ +using System.Net; +using Orleans.Metadata; +using Orleans.Runtime.MembershipService.SiloMetadata; +using Orleans.Runtime.Placement; +using Orleans.Runtime.Placement.Filtering; +using Xunit; + +namespace UnitTests.PlacementFilterTests; + +[TestCategory("Placement"), TestCategory("Filters"), TestCategory("SiloMetadata")] +public class PreferredMatchSiloMetadataPlacementFilterDirectorTests +{ + [Fact, TestCategory("Functional")] + public void CanBeCreated() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var director = new PreferredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + { testLocalSiloAddress, SiloMetadata.Empty } + })); + Assert.NotNull(director); + } + + [Fact, TestCategory("Functional")] + public void CanBeCalled() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var director = new PreferredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testLocalSiloAddress, SiloMetadata.Empty} + })); + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(), new PlacementTarget(), + new List() { testLocalSiloAddress } + ).ToList(); + Assert.NotNull(result); + Assert.NotEmpty(result); + } + + [Fact, TestCategory("Functional")] + public void FiltersToAllWhenNoEntry() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var siloMetadata = new SiloMetadata(); + siloMetadata.AddMetadata("metadata.key", "something"); + var director = new PreferredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress, SiloMetadata.Empty}, + {testLocalSiloAddress, siloMetadata}, + })); + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 1, 0), new PlacementTarget(), + new List() { testOtherSiloAddress }).ToList(); + Assert.NotEmpty(result); + } + + + [Theory, TestCategory("Functional")] + [InlineData(1, 3, "no.match")] + [InlineData(2, 3, "no.match")] + [InlineData( 1, 1, "one.match")] + [InlineData( 2, 3, "one.match")] + [InlineData( 1, 2, "two.match")] + [InlineData( 2, 2, "two.match")] + [InlineData( 3, 3, "two.match")] + [InlineData( 1, 3, "all.match")] + [InlineData( 2, 3, "all.match")] + + public void FiltersOnSingleMetadata(int minCandidates, int expectedCount, string key) + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress1 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var testOtherSiloAddress2 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1002, 1); + var testOtherSiloAddress3 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1003, 1); + var localSiloMetadata = new SiloMetadata(); + localSiloMetadata.AddMetadata("all.match", "match"); + localSiloMetadata.AddMetadata("one.match", "match"); + localSiloMetadata.AddMetadata("two.match", "match"); + localSiloMetadata.AddMetadata("no.match", "match"); + var otherSiloMetadata1 = new SiloMetadata(); + otherSiloMetadata1.AddMetadata("all.match", "match"); + otherSiloMetadata1.AddMetadata("one.match", "match"); + otherSiloMetadata1.AddMetadata("two.match", "match"); + otherSiloMetadata1.AddMetadata("no.match", "nomatch"); + var otherSiloMetadata2 = new SiloMetadata(); + otherSiloMetadata2.AddMetadata("all.match", "match"); + otherSiloMetadata2.AddMetadata("one.match", "nomatch"); + otherSiloMetadata2.AddMetadata("two.match", "match"); + otherSiloMetadata2.AddMetadata("no.match", "nomatch"); + var otherSiloMetadata3 = new SiloMetadata(); + otherSiloMetadata3.AddMetadata("all.match", "match"); + otherSiloMetadata3.AddMetadata("one.match", "nomatch"); + otherSiloMetadata3.AddMetadata("two.match", "nomatch"); + otherSiloMetadata3.AddMetadata("no.match", "nomatch"); + var director = new PreferredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress1, otherSiloMetadata1}, + {testOtherSiloAddress2, otherSiloMetadata2}, + {testOtherSiloAddress3, otherSiloMetadata3}, + {testLocalSiloAddress, localSiloMetadata}, + })); + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy([key], minCandidates, 0), new PlacementTarget(), + new List() { testOtherSiloAddress1, testOtherSiloAddress2, testOtherSiloAddress3 }).ToList(); + Assert.NotEmpty(result); + Assert.Equal(expectedCount, result.Count); + } + + [Theory, TestCategory("Functional")] + [InlineData(1, 3, "no.match", "all.match")] + [InlineData(1, 1, "no.match", "one.match", "two.match")] + [InlineData(2, 2, "no.match", "one.match", "two.match")] + [InlineData(3, 3, "no.match", "one.match", "two.match")] + [InlineData(1, 3, "all.match", "no.match")] + [InlineData(1, 1, "one.match", "all.match")] + [InlineData(2, 3, "one.match", "all.match")] + [InlineData(1, 1, "no.match", "one.match", "all.match")] + [InlineData(2, 3, "no.match", "one.match", "all.match")] + + public void FiltersOnMultipleMetadata(int minCandidates, int expectedCount, params string[] keys) + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress1 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var testOtherSiloAddress2 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1002, 1); + var testOtherSiloAddress3 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1003, 1); + var localSiloMetadata = new SiloMetadata(); + localSiloMetadata.AddMetadata("all.match", "match"); + localSiloMetadata.AddMetadata("one.match", "match"); + localSiloMetadata.AddMetadata("two.match", "match"); + localSiloMetadata.AddMetadata("no.match", "match"); + var otherSiloMetadata1 = new SiloMetadata(); + otherSiloMetadata1.AddMetadata("all.match", "match"); + otherSiloMetadata1.AddMetadata("one.match", "match"); + otherSiloMetadata1.AddMetadata("two.match", "match"); + otherSiloMetadata1.AddMetadata("no.match", "not.match"); + var otherSiloMetadata2 = new SiloMetadata(); + otherSiloMetadata2.AddMetadata("all.match", "match"); + otherSiloMetadata2.AddMetadata("one.match", "nomatch"); + otherSiloMetadata2.AddMetadata("two.match", "match"); + otherSiloMetadata2.AddMetadata("no.match", "not.match"); + var otherSiloMetadata3 = new SiloMetadata(); + otherSiloMetadata3.AddMetadata("all.match", "match"); + otherSiloMetadata3.AddMetadata("one.match", "not.match"); + otherSiloMetadata3.AddMetadata("two.match", "not.match"); + otherSiloMetadata3.AddMetadata("no.match", "not.match"); + var director = new PreferredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress1, otherSiloMetadata1}, + {testOtherSiloAddress2, otherSiloMetadata2}, + {testOtherSiloAddress3, otherSiloMetadata3}, + {testLocalSiloAddress, localSiloMetadata}, + })); + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(keys, minCandidates, 0), new PlacementTarget(), + new List() { testOtherSiloAddress1, testOtherSiloAddress2, testOtherSiloAddress3 }).ToList(); + Assert.NotEmpty(result); + Assert.Equal(expectedCount, result.Count); + } +} \ No newline at end of file diff --git a/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs b/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs new file mode 100644 index 0000000000..19d03626c9 --- /dev/null +++ b/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs @@ -0,0 +1,141 @@ +using System.Net; +using Orleans.Runtime.MembershipService.SiloMetadata; +using Orleans.Runtime.Placement; +using Orleans.Runtime.Placement.Filtering; +using Xunit; + +namespace UnitTests.PlacementFilterTests; + +[TestCategory("Placement"), TestCategory("Filters"), TestCategory("SiloMetadata")] +public class RequiredMatchSiloMetadataPlacementFilterDirectorTests +{ + [Fact, TestCategory("Functional")] + public void RequiredMatchSiloMetadataPlacementFilterDirector_CanBeCreated() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var director = new RequiredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testLocalSiloAddress, SiloMetadata.Empty} + })); + Assert.NotNull(director); + } + + [Fact, TestCategory("Functional")] + public void RequiredMatchSiloMetadataPlacementFilterDirector_CanBeCalled() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var director = new RequiredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testLocalSiloAddress, SiloMetadata.Empty} + })); + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(), new PlacementTarget(), + new List() { testLocalSiloAddress } + ).ToList(); + Assert.NotNull(result); + Assert.NotEmpty(result); + } + + [Fact, TestCategory("Functional")] + public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToNothingWhenNoEntry() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var siloMetadata = new SiloMetadata(); + siloMetadata.AddMetadata("metadata.key", "something"); + var director = new RequiredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress, SiloMetadata.Empty}, + {testLocalSiloAddress, siloMetadata}, + })); + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 0), new PlacementTarget(), + new List() { testOtherSiloAddress }).ToList(); + Assert.Empty(result); + } + + [Fact, TestCategory("Functional")] + public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToNothingWhenDifferentValue() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var localSiloMetadata = new SiloMetadata(); + localSiloMetadata.AddMetadata("metadata.key", "local"); + var otherSiloMetadata = new SiloMetadata(); + otherSiloMetadata.AddMetadata("metadata.key", "other"); + var director = new RequiredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress, otherSiloMetadata}, + {testLocalSiloAddress, localSiloMetadata}, + })); + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 0), new PlacementTarget(), + new List() { testOtherSiloAddress }).ToList(); + Assert.Empty(result); + } + + [Fact, TestCategory("Functional")] + public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToSiloWhenMatching() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var localSiloMetadata = new SiloMetadata(); + localSiloMetadata.AddMetadata("metadata.key", "same"); + var otherSiloMetadata = new SiloMetadata(); + otherSiloMetadata.AddMetadata("metadata.key", "same"); + var director = new RequiredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress, otherSiloMetadata}, + {testLocalSiloAddress, localSiloMetadata}, + })); + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 0), new PlacementTarget(), + new List() { testOtherSiloAddress }).ToList(); + Assert.NotEmpty(result); + } + + [Fact, TestCategory("Functional")] + public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToMultipleSilosWhenMatching() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress1 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var testOtherSiloAddress2 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1002, 1); + var localSiloMetadata = new SiloMetadata(); + localSiloMetadata.AddMetadata("metadata.key", "same"); + var otherSiloMetadata1 = new SiloMetadata(); + otherSiloMetadata1.AddMetadata("metadata.key", "same"); + var otherSiloMetadata2 = new SiloMetadata(); + otherSiloMetadata2.AddMetadata("metadata.key", "same"); + var director = new RequiredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress1, otherSiloMetadata1}, + {testOtherSiloAddress2, otherSiloMetadata2}, + {testLocalSiloAddress, localSiloMetadata}, + })); + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 0), new PlacementTarget(), + new List() { testOtherSiloAddress1, testOtherSiloAddress2 }).ToList(); + Assert.NotEmpty(result); + Assert.Equal(2, result.Count); + } + +} \ No newline at end of file diff --git a/test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs b/test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs new file mode 100644 index 0000000000..e0fca917bb --- /dev/null +++ b/test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs @@ -0,0 +1,293 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime.MembershipService.SiloMetadata; +using Orleans.Runtime.Placement.Filtering; +using Orleans.TestingHost; +using TestExtensions; +using Xunit; + +namespace UnitTests.PlacementFilterTests; + +[TestCategory("Placement"), TestCategory("Filters"), TestCategory("SiloMetadata")] +public class SiloMetadataPlacementFilterTests : TestClusterPerTest +{ + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.AddSiloBuilderConfigurator(); + } + + private class SiloConfigurator : ISiloConfigurator + { + public void Configure(ISiloBuilder hostBuilder) + { + hostBuilder.UseSiloMetadata(new Dictionary + { + {"first", "1"}, + {"second", "2"}, + {"third", "3"}, + {"unique", Guid.NewGuid().ToString()} + }); + } + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_GrainWithoutFilterCanBeCalled() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var managementGrain = Client.GetGrain(0); + var silos = await managementGrain.GetHosts(true); + Assert.NotNull(silos); + } + + /// + /// Unique silo metadata is set up to be different for each silo, so this will require that placement happens on the calling silo. + /// + /// + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_RequiredFilterCanBeCalled() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var id = 0; + foreach (var hostedClusterSilo in HostedCluster.Silos) + { + for (var i = 0; i < 50; i++) + { + ++id; + var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var managementGrain = firstSiloMetadataCache.GetGrain(id); + var hostingSilo = await managementGrain.GetHostingSilo(); + Assert.NotNull(hostingSilo); + Assert.Equal(hostedClusterSilo.SiloAddress, hostingSilo); + } + } + } + + /// + /// Unique silo metadata is set up to be different for each silo, so this will require that placement happens on the calling silo because it is the only one that matches. + /// + /// + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_PreferredFilterCanBeCalled() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var id = 0; + foreach (var hostedClusterSilo in HostedCluster.Silos) + { + for (var i = 0; i < 50; i++) + { + ++id; + var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var managementGrain = firstSiloMetadataCache.GetGrain(id); + var hostingSilo = await managementGrain.GetHostingSilo(); + Assert.NotNull(hostingSilo); + Assert.Equal(hostedClusterSilo.SiloAddress, hostingSilo); + } + } + } + + /// + /// Unique silo metadata is set up to be different for each silo, so this will still place on any of the two silos since just the matching silos (just the one) is not enough to make the minimum desired candidates. + /// + /// + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_PreferredMin2FilterCanBeCalled() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var id = 0; + foreach (var hostedClusterSilo in HostedCluster.Silos) + { + var dict = new Dictionary(); + foreach (var clusterSilo in HostedCluster.Silos) + { + dict[clusterSilo.SiloAddress] = 0; + } + for (var i = 0; i < 50; i++) + { + ++id; + var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var managementGrain = firstSiloMetadataCache.GetGrain(id); + var hostingSilo = await managementGrain.GetHostingSilo(); + Assert.NotNull(hostingSilo); + dict[hostingSilo] = dict.TryGetValue(hostingSilo, out var count) ? count + 1 : 1; + } + + foreach (var kv in dict) + { + Assert.True(kv.Value >= 1, $"Silo {kv.Key} did not host at least 1 grain"); + } + } + } + + /// + /// Unique silo metadata is set up to be different for each silo, so this will still place on any of the two silos since just the matching silos (just the one) is not enough to make the minimum desired candidates. + /// + /// + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_PreferredMultipleFilterCanBeCalled() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var id = 0; + foreach (var hostedClusterSilo in HostedCluster.Silos) + { + var dict = new Dictionary(); + foreach (var clusterSilo in HostedCluster.Silos) + { + dict[clusterSilo.SiloAddress] = 0; + } + for (var i = 0; i < 50; i++) + { + ++id; + var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var managementGrain = firstSiloMetadataCache.GetGrain(id); + var hostingSilo = await managementGrain.GetHostingSilo(); + Assert.NotNull(hostingSilo); + dict[hostingSilo] = dict.TryGetValue(hostingSilo, out var count) ? count + 1 : 1; + } + + foreach (var kv in dict) + { + Assert.True(kv.Value >= 1, $"Silo {kv.Key} did not host at least 1 grain"); + } + } + } + + /// + /// Unique silo metadata is set up to be different for each silo, so this will still place on any of the two silos since just the matching silos (just the one) is not enough to make the minimum desired candidates. + /// + /// + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_PreferredMin2FilterCanBeCalledWithLargerCluster() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.StartAdditionalSiloAsync(); + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var id = 0; + foreach (var hostedClusterSilo in HostedCluster.Silos) + { + var dict = new Dictionary(); + foreach (var clusterSilo in HostedCluster.Silos) + { + dict[clusterSilo.SiloAddress] = 0; + } + for (var i = 0; i < 50; i++) + { + ++id; + var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var managementGrain = firstSiloMetadataCache.GetGrain(id); + var hostingSilo = await managementGrain.GetHostingSilo(); + Assert.NotNull(hostingSilo); + dict[hostingSilo] = dict.TryGetValue(hostingSilo, out var count) ? count + 1 : 1; + } + + foreach (var kv in dict) + { + Assert.True(kv.Value >= 1, $"Silo {kv.Key} did not host at least 1 grain"); + } + } + } + + /// + /// If no metadata key is defined then it should fall back to matching all silos + /// + /// + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_PreferredNoMetadataFilterCanBeCalled() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.StartAdditionalSiloAsync(); + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var id = 0; + foreach (var hostedClusterSilo in HostedCluster.Silos) + { + var dict = new Dictionary(); + foreach (var clusterSilo in HostedCluster.Silos) + { + dict[clusterSilo.SiloAddress] = 0; + } + for (var i = 0; i < 50; i++) + { + ++id; + var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var managementGrain = firstSiloMetadataCache.GetGrain(id); + var hostingSilo = await managementGrain.GetHostingSilo(); + Assert.NotNull(hostingSilo); + dict[hostingSilo] = dict.TryGetValue(hostingSilo, out var count) ? count + 1 : 1; + } + + foreach (var kv in dict) + { + Assert.True(kv.Value >= 1, $"Silo {kv.Key} did not host at least 1 grain"); + } + } + } +} + +public interface IUniqueRequiredMatchFilteredGrain : IGrainWithIntegerKey +{ + Task GetHostingSilo(); +} + +#pragma warning disable ORLEANSEXP004 +[RequiredMatchSiloMetadataPlacementFilter(["unique"])] +#pragma warning restore ORLEANSEXP004 +public class UniqueRequiredMatchFilteredGrain(ILocalSiloDetails localSiloDetails) : Grain, IUniqueRequiredMatchFilteredGrain +{ + public Task GetHostingSilo() => Task.FromResult(localSiloDetails.SiloAddress); +} +public interface IPreferredMatchFilteredGrain : IGrainWithIntegerKey +{ + Task GetHostingSilo(); +} + +#pragma warning disable ORLEANSEXP004 +[PreferredMatchSiloMetadataPlacementFilter(["unique"], 1)] +#pragma warning restore ORLEANSEXP004 +public class PreferredMatchFilteredGrain(ILocalSiloDetails localSiloDetails) : Grain, IPreferredMatchFilteredGrain +{ + public Task GetHostingSilo() => Task.FromResult(localSiloDetails.SiloAddress); +} + + +public interface IPreferredMatchMin2FilteredGrain : IGrainWithIntegerKey +{ + Task GetHostingSilo(); +} + +#pragma warning disable ORLEANSEXP004 +[PreferredMatchSiloMetadataPlacementFilter(["unique"])] +#pragma warning restore ORLEANSEXP004 +public class PreferredMatchMinTwoFilteredGrain(ILocalSiloDetails localSiloDetails) : Grain, IPreferredMatchMin2FilteredGrain +{ + public Task GetHostingSilo() => Task.FromResult(localSiloDetails.SiloAddress); +} + +public interface IPreferredMatchMultipleFilteredGrain : IGrainWithIntegerKey +{ + Task GetHostingSilo(); +} + +#pragma warning disable ORLEANSEXP004 +[PreferredMatchSiloMetadataPlacementFilter(["unique", "other"], 2)] +#pragma warning restore ORLEANSEXP004 +public class PreferredMatchMultipleFilteredGrain(ILocalSiloDetails localSiloDetails) : Grain, IPreferredMatchMultipleFilteredGrain +{ + public Task GetHostingSilo() => Task.FromResult(localSiloDetails.SiloAddress); +} + +#pragma warning disable ORLEANSEXP004 +[PreferredMatchSiloMetadataPlacementFilter(["not.there"])] +#pragma warning restore ORLEANSEXP004 +public class PreferredMatchNoMetadataFilteredGrain(ILocalSiloDetails localSiloDetails) : Grain, IPreferredMatchNoMetadataFilteredGrain +{ + public Task GetHostingSilo() => Task.FromResult(localSiloDetails.SiloAddress); +} + +public interface IPreferredMatchNoMetadataFilteredGrain : IGrainWithIntegerKey +{ + Task GetHostingSilo(); +} \ No newline at end of file diff --git a/test/TesterInternal/PlacementFilterTests/TestLocalSiloDetails.cs b/test/TesterInternal/PlacementFilterTests/TestLocalSiloDetails.cs new file mode 100644 index 0000000000..849480f0e9 --- /dev/null +++ b/test/TesterInternal/PlacementFilterTests/TestLocalSiloDetails.cs @@ -0,0 +1,19 @@ +namespace UnitTests.PlacementFilterTests; + +internal class TestLocalSiloDetails : ILocalSiloDetails +{ + public TestLocalSiloDetails(string name, string clusterId, string dnsHostName, SiloAddress siloAddress, SiloAddress gatewayAddress) + { + Name = name; + ClusterId = clusterId; + DnsHostName = dnsHostName; + SiloAddress = siloAddress; + GatewayAddress = gatewayAddress; + } + + public string Name { get; } + public string ClusterId { get; } + public string DnsHostName { get; } + public SiloAddress SiloAddress { get; } + public SiloAddress GatewayAddress { get; } +} \ No newline at end of file diff --git a/test/TesterInternal/PlacementFilterTests/TestSiloMetadataCache.cs b/test/TesterInternal/PlacementFilterTests/TestSiloMetadataCache.cs new file mode 100644 index 0000000000..a6ab78dda4 --- /dev/null +++ b/test/TesterInternal/PlacementFilterTests/TestSiloMetadataCache.cs @@ -0,0 +1,15 @@ +using Orleans.Runtime.MembershipService.SiloMetadata; + +namespace UnitTests.PlacementFilterTests; + +internal class TestSiloMetadataCache : ISiloMetadataCache +{ + private readonly Dictionary _metadata; + + public TestSiloMetadataCache(Dictionary metadata) + { + _metadata = metadata; + } + + public SiloMetadata GetMetadata(SiloAddress siloAddress) => _metadata.GetValueOrDefault(siloAddress) ?? new SiloMetadata(); +} \ No newline at end of file diff --git a/test/TesterInternal/SiloMetadataTests/SiloMetadataTests.cs b/test/TesterInternal/SiloMetadataTests/SiloMetadataTests.cs new file mode 100644 index 0000000000..30cfc94ec3 --- /dev/null +++ b/test/TesterInternal/SiloMetadataTests/SiloMetadataTests.cs @@ -0,0 +1,181 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime.MembershipService.SiloMetadata; +using Orleans.TestingHost; +using TestExtensions; +using Xunit; + +namespace UnitTests.SiloMetadataTests; + + +[TestCategory("SiloMetadata")] +public class SiloMetadataConfigTests : TestClusterPerTest +{ + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.AddSiloBuilderConfigurator(); + } + + private class SiloConfigurator : ISiloConfigurator + { + public static readonly List> Metadata = + [ + new("Orleans:Metadata:first", "1"), + new("Orleans:Metadata:second", "2"), + new("Orleans:Metadata:third", "3") + ]; + + public void Configure(ISiloBuilder hostBuilder) + { + var config = new ConfigurationBuilder() + .AddInMemoryCollection(Metadata) + .Build(); + hostBuilder.UseSiloMetadata(config); + } + } + + [Fact, TestCategory("Functional")] + public async Task SiloMetadata_CanBeSetAndRead() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(SiloConfigurator.Metadata.Select(kv => kv.Key.Split(':').Last()).ToArray()); + } + + [Fact, TestCategory("Functional")] + public async Task SiloMetadata_HasConfiguredValues() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + + var first = HostedCluster.Silos.First(); + var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var metadata = firstSiloMetadataCache.GetMetadata(first.SiloAddress); + Assert.NotNull(metadata); + Assert.NotNull(metadata.Metadata); + Assert.Equal(SiloConfigurator.Metadata.Count, metadata.Metadata.Count); + foreach (var kv in SiloConfigurator.Metadata) + { + Assert.Equal(kv.Value, metadata.Metadata[kv.Key.Split(':').Last()]); + } + } +} + +[TestCategory("SiloMetadata")] +public class SiloMetadataTests : TestClusterPerTest +{ + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.AddSiloBuilderConfigurator(); + } + + private class SiloConfigurator : ISiloConfigurator + { + public void Configure(ISiloBuilder hostBuilder) + { + hostBuilder.UseSiloMetadata(new Dictionary + { + {"host.id", Guid.NewGuid().ToString()} + }); + } + } + + [Fact, TestCategory("Functional")] + public async Task SiloMetadata_CanBeSetAndRead() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); + } + + [Fact, TestCategory("Functional")] + public async Task SiloMetadata_NewSilosHaveMetadata() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.StartAdditionalSiloAsync(); + HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); + } + + [Fact, TestCategory("Functional")] + public async Task SiloMetadata_RemovedSiloHasNoMetadata() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); + var first = HostedCluster.Silos.First(); + var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + + var second = HostedCluster.Silos.Skip(1).First(); + var metadata = firstSiloMetadataCache.GetMetadata(second.SiloAddress); + Assert.NotNull(metadata); + Assert.NotEmpty(metadata.Metadata); + + await HostedCluster.StopSiloAsync(second); + metadata = firstSiloMetadataCache.GetMetadata(second.SiloAddress); + Assert.NotNull(metadata); + Assert.Empty(metadata.Metadata); + } + + [Fact, TestCategory("Functional")] + public async Task SiloMetadata_BadSiloAddressHasNoMetadata() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var first = HostedCluster.Silos.First(); + var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var metadata = firstSiloMetadataCache.GetMetadata(SiloAddress.Zero); + Assert.NotNull(metadata); + Assert.Empty(metadata.Metadata); + } +} + +public static class SiloMetadataTestExtensions +{ + public static void AssertAllSiloMetadataMatchesOnAllSilos(this TestCluster hostedCluster, string[] expectedKeys) + { + var exampleSiloMetadata = new Dictionary(); + var first = hostedCluster.Silos.First(); + var firstSp = hostedCluster.GetSiloServiceProvider(first.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + foreach (var otherSilo in hostedCluster.Silos) + { + var metadata = firstSiloMetadataCache.GetMetadata(otherSilo.SiloAddress); + Assert.NotNull(metadata); + Assert.NotNull(metadata.Metadata); + foreach (var expectedKey in expectedKeys) + { + Assert.True(metadata.Metadata.ContainsKey(expectedKey)); + } + exampleSiloMetadata.Add(otherSilo.SiloAddress, metadata); + } + + foreach (var hostedClusterSilo in hostedCluster.Silos.Skip(1)) + { + var sp = hostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var siloMetadataCache = sp.GetRequiredService(); + var remoteMetadata = new Dictionary(); + foreach (var otherSilo in hostedCluster.Silos) + { + var metadata = siloMetadataCache.GetMetadata(otherSilo.SiloAddress); + Assert.NotNull(metadata); + Assert.NotNull(metadata.Metadata); + foreach (var expectedKey in expectedKeys) + { + Assert.True(metadata.Metadata.ContainsKey(expectedKey)); + } + remoteMetadata.Add(otherSilo.SiloAddress, metadata); + } + + //Assert that the two dictionaries have the same keys and the values for those keys are the same + Assert.Equal(exampleSiloMetadata.Count, remoteMetadata.Count); + foreach (var kvp in exampleSiloMetadata) + { + Assert.Equal(kvp.Value.Metadata.Count, remoteMetadata[kvp.Key].Metadata.Count); + foreach (var kvp2 in kvp.Value.Metadata) + { + Assert.True(remoteMetadata[kvp.Key].Metadata.TryGetValue(kvp2.Key, out var value), + $"Key '{kvp2.Key}' not found in actual dictionary."); + Assert.Equal(kvp2.Value, value); + } + } + } + } +} \ No newline at end of file diff --git a/test/TesterInternal/TesterInternal.csproj b/test/TesterInternal/TesterInternal.csproj index ceefb25f91..0adc91be07 100644 --- a/test/TesterInternal/TesterInternal.csproj +++ b/test/TesterInternal/TesterInternal.csproj @@ -1,4 +1,4 @@ - + UnitTests true