From ed20ef34183bb35439af9c8009d6d54567df2d4c Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Wed, 18 Dec 2024 17:10:35 -0800 Subject: [PATCH 01/16] Initial implementation Still requires tests --- .../Manifest/GrainProperties.cs | 5 + .../Hosting/DefaultSiloServices.cs | 7 +- .../SiloMetadata/ISiloMetadataCache.cs | 6 ++ .../SiloMetadata/ISiloMetadataClient.cs | 9 ++ .../SiloMetadata/ISiloMetadataGrainService.cs | 11 +++ .../SiloMetadata/SiloMetadaCache.cs | 96 +++++++++++++++++++ .../SiloMetadata/SiloMetadata.cs | 17 ++++ .../SiloMetadata/SiloMetadataClient.cs | 16 ++++ .../SiloMetadata/SiloMetadataGrainService.cs | 22 +++++ .../SiloMetadataHostingExtensions.cs | 90 +++++++++++++++++ .../Filtering/IPlacementFilterDirector.cs | 9 ++ .../Filtering/PlacementFilterAttribute.cs | 24 +++++ .../PlacementFilterDirectorResolver.cs | 12 +++ .../Filtering/PlacementFilterExtensions.cs | 23 +++++ .../Filtering/PlacementFilterStrategy.cs | 52 ++++++++++ .../PlacementFilterStrategyResolver.cs | 63 ++++++++++++ ...redSiloMetadataPlacementFilterAttribute.cs | 7 ++ ...rredSiloMetadataPlacementFilterDirector.cs | 73 ++++++++++++++ ...rredSiloMetadataPlacementFilterStrategy.cs | 26 +++++ .../RequiredSiloMetadataFilterDirector.cs | 59 ++++++++++++ ...redSiloMetadataPlacementFilterAttribute.cs | 7 ++ ...iredSiloMetadataPlacementFilterStrategy.cs | 26 +++++ .../Placement/PlacementService.cs | 25 ++++- 23 files changed, 682 insertions(+), 3 deletions(-) create mode 100644 src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs create mode 100644 src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs create mode 100644 src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs create mode 100644 src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs create mode 100644 src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs create mode 100644 src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs create mode 100644 src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs create mode 100644 src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/IPlacementFilterDirector.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/PlacementFilterAttribute.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/PlacementFilterExtensions.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterAttribute.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterDirector.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterStrategy.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataFilterDirector.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterAttribute.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterStrategy.cs diff --git a/src/Orleans.Core.Abstractions/Manifest/GrainProperties.cs b/src/Orleans.Core.Abstractions/Manifest/GrainProperties.cs index 41181c248d..4f994e01f2 100644 --- a/src/Orleans.Core.Abstractions/Manifest/GrainProperties.cs +++ b/src/Orleans.Core.Abstractions/Manifest/GrainProperties.cs @@ -68,6 +68,11 @@ public static class WellKnownGrainTypeProperties /// public const string PlacementStrategy = "placement-strategy"; + /// + /// The name of the placement strategy for grains of this type. + /// + public const string PlacementFilter = "placement-filter"; + /// /// The directory policy for grains of this type. /// diff --git a/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs b/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs index 67f98907cd..bba9fb3276 100644 --- a/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs +++ b/src/Orleans.Runtime/Hosting/DefaultSiloServices.cs @@ -43,8 +43,7 @@ using Orleans.Serialization.Internal; using Orleans.Core; using Orleans.Placement.Repartitioning; -using Orleans.GrainDirectory; -using Orleans.Runtime.Hosting; +using Orleans.Runtime.Placement.Filtering; namespace Orleans.Hosting { @@ -206,6 +205,10 @@ internal static void AddDefaultServices(ISiloBuilder builder) // Configure the default placement strategy. services.TryAddSingleton(); + // Placement filters + services.AddSingleton(); + services.AddSingleton(); + // Placement directors services.AddPlacementDirector(); services.AddPlacementDirector(); diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs new file mode 100644 index 0000000000..a15cb67a36 --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs @@ -0,0 +1,6 @@ +namespace Orleans.Runtime.MembershipService.SiloMetadata; +#nullable enable +public interface ISiloMetadataCache +{ + SiloMetadata GetMetadata(SiloAddress siloAddress); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs new file mode 100644 index 0000000000..eb622eaaa4 --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; +using Orleans.Services; + +namespace Orleans.Runtime.MembershipService.SiloMetadata; + +public interface ISiloMetadataClient : IGrainServiceClient +{ + Task GetSiloMetadata(SiloAddress siloAddress); +} diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs new file mode 100644 index 0000000000..97e3c7e600 --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs @@ -0,0 +1,11 @@ +using System.Threading.Tasks; +using Orleans.Services; + +namespace Orleans.Runtime.MembershipService.SiloMetadata; + +[Alias("Orleans.Runtime.MembershipService.SiloMetadata.ISiloMetadataGrainService")] +public interface ISiloMetadataGrainService : IGrainService +{ + [Alias("GetSiloMetadata")] + Task GetSiloMetadata(); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs new file mode 100644 index 0000000000..afdcb39c95 --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Orleans.Configuration; +using Orleans.Internal; + +namespace Orleans.Runtime.MembershipService.SiloMetadata; +#nullable enable +internal class SiloMetadataCache( + ISiloMetadataClient siloMetadataClient, + MembershipTableManager membershipTableManager, + ILogger logger) + : ISiloMetadataCache, ILifecycleParticipant, IDisposable +{ + private readonly ConcurrentDictionary _metadata = new(); + private readonly CancellationTokenSource _cts = new(); + + void ILifecycleParticipant.Participate(ISiloLifecycle lifecycle) + { + var tasks = new List(1); + var cancellation = new CancellationTokenSource(); + Task OnRuntimeInitializeStart(CancellationToken _) + { + tasks.Add(Task.Run(() => this.ProcessMembershipUpdates(cancellation.Token))); + return Task.CompletedTask; + } + + async Task OnRuntimeInitializeStop(CancellationToken ct) + { + cancellation.Cancel(throwOnFirstException: false); + var shutdownGracePeriod = Task.WhenAll(Task.Delay(ClusterMembershipOptions.ClusteringShutdownGracePeriod), ct.WhenCancelled()); + await Task.WhenAny(shutdownGracePeriod, Task.WhenAll(tasks)); + } + + lifecycle.Subscribe( + nameof(ClusterMembershipService), + ServiceLifecycleStage.RuntimeInitialize, + OnRuntimeInitializeStart, + OnRuntimeInitializeStop); + } + + + private async Task ProcessMembershipUpdates(CancellationToken ct) + { + try + { + if (logger.IsEnabled(LogLevel.Debug)) logger.LogDebug("Starting to process membership updates"); + await foreach (var update in membershipTableManager.MembershipTableUpdates.WithCancellation(ct)) + { + // Add entries for members that aren't already in the cache + foreach (var membershipEntry in update.Entries) + { + if (!_metadata.ContainsKey(membershipEntry.Key)) + { + try + { + var metadata = await siloMetadataClient.GetSiloMetadata(membershipEntry.Key); + _metadata.TryAdd(membershipEntry.Key, metadata); + } + catch(Exception exception) + { + logger.LogError(exception, "Error fetching metadata for silo {Silo}", membershipEntry.Key); + } + } + } + + // Remove entries for members that are no longer in the table + foreach (var silo in _metadata.Keys.ToList()) + { + if (!update.Entries.ContainsKey(silo)) + { + _metadata.TryRemove(silo, out _); + } + } + } + } + catch (Exception exception) + { + logger.LogError(exception, "Error processing membership updates"); + } + finally + { + if (logger.IsEnabled(LogLevel.Debug)) logger.LogDebug("Stopping membership update processor"); + } + } + + public SiloMetadata GetMetadata(SiloAddress siloAddress) => _metadata.GetValueOrDefault(siloAddress) ?? SiloMetadata.Empty; + + public void SetMetadata(SiloAddress siloAddress, SiloMetadata metadata) => _metadata.TryAdd(siloAddress, metadata); + + public void Dispose() => _cts.Cancel(); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs new file mode 100644 index 0000000000..a8adf8397c --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs @@ -0,0 +1,17 @@ +using System.Collections.Generic; +using System.Collections.Immutable; + +namespace Orleans.Runtime.MembershipService.SiloMetadata; + +[GenerateSerializer] +[Alias("Orleans.Runtime.MembershipService.SiloMetadata.SiloMetadata")] +public record SiloMetadata +{ + public static SiloMetadata Empty { get; } = new SiloMetadata(); + + [Id(0)] + public ImmutableDictionary Metadata { get; private set; } = ImmutableDictionary.Empty; + + internal void AddMetadata(IEnumerable> metadata) => Metadata = Metadata.AddRange(metadata); + internal void AddMetadata(string key, string value) => Metadata = Metadata.Add(key, value); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs new file mode 100644 index 0000000000..377bf30288 --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs @@ -0,0 +1,16 @@ +using System; +using System.Threading.Tasks; +using Orleans.Runtime.Services; + +namespace Orleans.Runtime.MembershipService.SiloMetadata; + +public class SiloMetadataClient(IServiceProvider serviceProvider) + : GrainServiceClient(serviceProvider), ISiloMetadataClient +{ + public async Task GetSiloMetadata(SiloAddress siloAddress) + { + var grainService = GetGrainService(siloAddress); + var metadata = await grainService.GetSiloMetadata(); + return metadata; + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs new file mode 100644 index 0000000000..336b527458 --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs @@ -0,0 +1,22 @@ +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Orleans.Runtime.MembershipService.SiloMetadata; + +public class SiloMetadataGrainService : GrainService, ISiloMetadataGrainService +{ + private readonly SiloMetadata _siloMetadata; + + public SiloMetadataGrainService(IOptions siloMetadata) : base() + { + _siloMetadata = siloMetadata.Value; + } + + public SiloMetadataGrainService(IOptions siloMetadata, GrainId grainId, Silo silo, ILoggerFactory loggerFactory) : base(grainId, silo, loggerFactory) + { + _siloMetadata = siloMetadata.Value; + } + + public Task GetSiloMetadata() => Task.FromResult(_siloMetadata); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs new file mode 100644 index 0000000000..3457bd0984 --- /dev/null +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs @@ -0,0 +1,90 @@ +using System.Collections.Generic; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Configuration.Internal; +using Orleans.Hosting; +using Orleans.Runtime.Placement.Filtering; + +namespace Orleans.Runtime.MembershipService.SiloMetadata; + +public static class SiloMetadataHostingExtensions +{ + + /// + /// Configure silo metadata from the builder configuration. + /// + /// Silo builder + /// + /// Get the ORLEANS__METADATA section from config + /// Key/value pairs in configuration as a will look like this as environment variables: + /// ORLEANS__METADATA__key1=value1 + /// + /// + public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder) => builder.UseSiloMetadata(builder.Configuration); + + /// + /// Configure silo metadata from configuration. + /// + /// Silo builder + /// Configuration to pull from + /// + /// Get the ORLEANS__METADATA section from config + /// Key/value pairs in configuration as a will look like this as environment variables: + /// ORLEANS__METADATA__key1=value1 + /// + /// + public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, IConfiguration configuration) + { + + var metadataConfigSection = builder.Configuration.GetSection("ORLEANS").GetSection("METADATA"); + + return builder.UseSiloMetadata(metadataConfigSection); + } + + /// + /// Configure silo metadata from configuration section. + /// + /// Silo builder + /// Configuration section to pull from + /// + /// Get the ORLEANS__METADATA section from config section + /// Key/value pairs in configuration as a will look like this as environment variables: + /// ORLEANS__METADATA__key1=value1 + /// + /// + public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, IConfigurationSection configurationSection) + { + var dictionary = configurationSection.Get>(); + + return builder.UseSiloMetadata(dictionary ?? new Dictionary()); + } + + /// + /// Configure silo metadata from configuration section. + /// + /// Silo builder + /// Metadata to add + /// + public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, Dictionary metadata) + { + builder.ConfigureServices(services => + { + services + .AddOptionsWithValidateOnStart() + .Configure(m => + { + m.AddMetadata(metadata); + }); + + services.AddGrainService(); + services.AddSingleton(); + services.AddFromExisting(); + services.AddFromExisting, SiloMetadataCache>(); + services.AddSingleton(); + // Placement filters + services.AddPlacementFilter(ServiceLifetime.Transient); + services.AddPlacementFilter(ServiceLifetime.Transient); + }); + return builder; + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/IPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/IPlacementFilterDirector.cs new file mode 100644 index 0000000000..23b2437771 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/IPlacementFilterDirector.cs @@ -0,0 +1,9 @@ +using System.Collections.Generic; + +namespace Orleans.Runtime.Placement.Filtering; + +public interface IPlacementFilterDirector +{ + IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, + IEnumerable silos); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterAttribute.cs new file mode 100644 index 0000000000..b67e23fd10 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterAttribute.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using Orleans.Metadata; + +namespace Orleans.Runtime.Placement.Filtering; + +/// +/// Base for all placement filter marker attributes. +/// +[AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] +public abstract class PlacementFilterAttribute : Attribute, IGrainPropertiesProviderAttribute +{ + public PlacementFilterStrategy PlacementFilterStrategy { get; private set; } + + protected PlacementFilterAttribute(PlacementFilterStrategy placement) + { + ArgumentNullException.ThrowIfNull(placement); + PlacementFilterStrategy = placement; + } + + /// + public virtual void Populate(IServiceProvider services, Type grainClass, GrainType grainType, Dictionary properties) + => PlacementFilterStrategy?.PopulateGrainProperties(services, grainClass, grainType, properties); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs new file mode 100644 index 0000000000..8401f5eac7 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs @@ -0,0 +1,12 @@ +using System; +using Microsoft.Extensions.DependencyInjection; + +namespace Orleans.Runtime.Placement.Filtering; + +/// +/// Responsible for resolving an for a . +/// +public sealed class PlacementFilterDirectorResolver(IServiceProvider services) +{ + public IPlacementFilterDirector GetFilterDirector(PlacementFilterStrategy placementFilterStrategy) => services.GetRequiredKeyedService(placementFilterStrategy.GetType()); +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterExtensions.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterExtensions.cs new file mode 100644 index 0000000000..faf9367f27 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterExtensions.cs @@ -0,0 +1,23 @@ +using Microsoft.Extensions.DependencyInjection; + +namespace Orleans.Runtime.Placement.Filtering; + +public static class PlacementFilterExtensions +{ + /// + /// Configures a for filtering candidate grain placements. + /// + /// The placement filter. + /// The placement filter director. + /// The service collection. + /// The lifetime of the placement strategy. + /// The service collection. + public static void AddPlacementFilter(this IServiceCollection services, ServiceLifetime strategyLifetime) + where TFilter : PlacementFilterStrategy + where TDirector : class, IPlacementFilterDirector + { + services.Add(ServiceDescriptor.DescribeKeyed(typeof(PlacementFilterStrategy), typeof(TFilter).Name, typeof(TFilter), strategyLifetime)); + services.AddKeyedSingleton(typeof(TFilter)); + } + +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs new file mode 100644 index 0000000000..614b42ec99 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using Orleans.Metadata; + +namespace Orleans.Runtime.Placement.Filtering; + +public abstract class PlacementFilterStrategy +{ + /// + /// Initializes an instance of this type using the provided grain properties. + /// + /// + /// The grain properties. + /// + public virtual void Initialize(GrainProperties properties) + { + } + + /// + /// Populates grain properties to specify the preferred placement strategy. + /// + /// The service provider. + /// The grain class. + /// The grain type. + /// The grain properties which will be populated by this method call. + public void PopulateGrainProperties(IServiceProvider services, Type grainClass, GrainType grainType, Dictionary properties) + { + var typeName = GetType().Name; + if (properties.TryGetValue(WellKnownGrainTypeProperties.PlacementFilter, out var existingValue)) + { + properties[WellKnownGrainTypeProperties.PlacementFilter] = $"{existingValue},{typeName}"; + } + else + { + properties[WellKnownGrainTypeProperties.PlacementFilter] = typeName; + } + + foreach (var additionalGrainProperty in GetAdditionalGrainProperties(services, grainClass, grainType, properties)) + { + properties[$"{WellKnownGrainTypeProperties.PlacementFilter}.{typeName}.{additionalGrainProperty.Key}"] = additionalGrainProperty.Value; + } + } + + protected string GetPlacementFilterGrainProperty(string key, GrainProperties properties) + { + var typeName = GetType().Name; + return properties.Properties.TryGetValue($"{WellKnownGrainTypeProperties.PlacementFilter}.{typeName}.{key}", out var value) ? value : null; + } + + protected virtual IEnumerable> GetAdditionalGrainProperties(IServiceProvider services, Type grainClass, GrainType grainType, IReadOnlyDictionary existingProperties) + => Array.Empty>(); +} diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs new file mode 100644 index 0000000000..cb36817056 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using Microsoft.Extensions.DependencyInjection; +using Orleans.Metadata; + +namespace Orleans.Runtime.Placement.Filtering; + +/// +/// Responsible for resolving an for a . +/// +public sealed class PlacementFilterStrategyResolver +{ + private readonly ConcurrentDictionary _resolvedFilters = new(); + private readonly Func _getFiltersInternal; + private readonly GrainPropertiesResolver _grainPropertiesResolver; + private readonly IServiceProvider _services; + + /// + /// Create a instance. + /// + public PlacementFilterStrategyResolver( + IServiceProvider services, + GrainPropertiesResolver grainPropertiesResolver) + { + _services = services; + _getFiltersInternal = GetPlacementFilterStrategyInternal; + _grainPropertiesResolver = grainPropertiesResolver; + } + + /// + /// Gets the placement filter strategy associated with the provided grain type. + /// + public PlacementFilterStrategy[] GetPlacementFilterStrategies(GrainType grainType) => _resolvedFilters.GetOrAdd(grainType, _getFiltersInternal); + + private PlacementFilterStrategy[] GetPlacementFilterStrategyInternal(GrainType grainType) + { + _grainPropertiesResolver.TryGetGrainProperties(grainType, out var properties); + + if (properties is not null + && properties.Properties.TryGetValue(WellKnownGrainTypeProperties.PlacementFilter, out var placementFilterIds) + && !string.IsNullOrWhiteSpace(placementFilterIds)) + { + var filterList = new List(); + foreach (var filterId in placementFilterIds.Split(",")) + { + var filter = _services.GetKeyedService(filterId); + if (filter is not null) + { + filter.Initialize(properties); + filterList.Add(filter); + } + else + { + throw new KeyNotFoundException($"Could not resolve placement filter strategy {filterId} for grain type {grainType}. Ensure that dependencies for that filter have been configured in the Container. This is often through a .Use* extension method provided by the implementation."); + } + } + return filterList.ToArray(); + } + + return []; + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterAttribute.cs new file mode 100644 index 0000000000..1c2183fcf1 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterAttribute.cs @@ -0,0 +1,7 @@ +using System; + +namespace Orleans.Runtime.Placement.Filtering; + +[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] +public class PreferredSiloMetadataPlacementFilterAttribute(string[] orderedMetadataKeys) + : PlacementFilterAttribute(new PreferredSiloMetadataPlacementFilterStrategy(orderedMetadataKeys)); \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterDirector.cs new file mode 100644 index 0000000000..e6d08aa060 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterDirector.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Orleans.Runtime.MembershipService.SiloMetadata; +#nullable enable +namespace Orleans.Runtime.Placement.Filtering; + +internal class PreferredSiloMetadataPlacementFilterDirector( + ILocalSiloDetails localSiloDetails, + ISiloMetadataCache siloMetadataCache) + : IPlacementFilterDirector +{ + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + { + var orderedMetadataKeys = (filterStrategy as PreferredSiloMetadataPlacementFilterStrategy)?.OrderedMetadataKeys ?? []; + var localSiloMetadata = siloMetadataCache.GetMetadata(localSiloDetails.SiloAddress).Metadata; + + if (localSiloMetadata.Count == 0) + { + // yield return all silos if no metadata keys are configured + foreach (var silo in silos) + { + yield return silo; + } + } + else + { + // return the list of silos that match the most metadata keys. The first key in the list is the least important. + // This means that the last key in the list is the most important. + // If no silos match any metadata keys, return the original list of silos. + + var siloList = silos.ToList(); + var maxScore = 0; + var siloScores = new int[siloList.Count]; + for (var i = 0; i < siloList.Count; i++) + { + var siloMetadata = siloMetadataCache.GetMetadata(siloList[i]).Metadata; + for (var j = orderedMetadataKeys.Length - 1; j >= 0; --j) + { + if (siloMetadata.TryGetValue(orderedMetadataKeys[j], out var siloMetadataValue) && + localSiloMetadata.TryGetValue(orderedMetadataKeys[j], out var localSiloMetadataValue) && + siloMetadataValue == localSiloMetadataValue) + { + var newScore = siloScores[i]++; + maxScore = Math.Max(maxScore, newScore); + } + else + { + break; + } + } + } + + if (maxScore == 0) + { + // yield return all silos if no silos match any metadata keys + foreach (var silo in siloList) + { + yield return silo; + } + } + + // return the list of silos that match the most metadata keys + for (var i = 0; i < siloScores.Length; i++) + { + if (siloScores[i] == maxScore) + { + yield return siloList[i]; + } + } + } + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterStrategy.cs new file mode 100644 index 0000000000..6316fd0c0b --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterStrategy.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using Orleans.Metadata; + +namespace Orleans.Runtime.Placement.Filtering; + +public class PreferredSiloMetadataPlacementFilterStrategy(string[] orderedMetadataKeys) : PlacementFilterStrategy +{ + public string[] OrderedMetadataKeys { get; set; } = orderedMetadataKeys; + + public PreferredSiloMetadataPlacementFilterStrategy() : this([]) + { + } + + public override void Initialize(GrainProperties properties) + { + base.Initialize(properties); + OrderedMetadataKeys = GetPlacementFilterGrainProperty("ordered-metadata-keys", properties).Split(","); + } + + protected override IEnumerable> GetAdditionalGrainProperties(IServiceProvider services, Type grainClass, GrainType grainType, + IReadOnlyDictionary existingProperties) + { + yield return new KeyValuePair("ordered-metadata-keys", string.Join(",", OrderedMetadataKeys)); + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataFilterDirector.cs new file mode 100644 index 0000000000..ba1fd242bd --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataFilterDirector.cs @@ -0,0 +1,59 @@ +using System.Collections.Generic; +using System.Linq; +using Orleans.Runtime.MembershipService.SiloMetadata; + +namespace Orleans.Runtime.Placement.Filtering; + +internal class RequiredSiloMetadataFilterDirector(ILocalSiloDetails localSiloDetails, ISiloMetadataCache siloMetadataCache) + : IPlacementFilterDirector +{ + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + { + var metadataKeys = (filterStrategy as RequiredSiloMetadataPlacementFilterStrategy)?.MetadataKeys ?? []; + + // yield return all silos if no silos match any metadata keys + if (metadataKeys.Length == 0) + { + foreach (var silo in silos) + { + yield return silo; + } + } + else + { + var localMetadata = siloMetadataCache.GetMetadata(localSiloDetails.SiloAddress); + var localRequiredMetadata = GetMetadata(localMetadata, metadataKeys); + + foreach (var silo in silos) + { + var remoteMetadata = siloMetadataCache.GetMetadata(silo); + if(DoesMetadataMatch(localRequiredMetadata, remoteMetadata, metadataKeys)) + { + yield return silo; + } + } + } + } + + private static bool DoesMetadataMatch(string[] localMetadata, SiloMetadata siloMetadata, string[] metadataKeys) + { + for (var i = 0; i < metadataKeys.Length; i++) + { + if(localMetadata[i] != siloMetadata.Metadata?.GetValueOrDefault(metadataKeys[i])) + { + return false; + } + } + + return true; + } + private static string[] GetMetadata(SiloMetadata siloMetadata, string[] metadataKeys) + { + var result = new string[metadataKeys.Length]; + for (var i = 0; i < metadataKeys.Length; i++) + { + result[i] = siloMetadata.Metadata?.GetValueOrDefault(metadataKeys[i]); + } + return result; + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterAttribute.cs new file mode 100644 index 0000000000..180a409c46 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterAttribute.cs @@ -0,0 +1,7 @@ +using System; + +namespace Orleans.Runtime.Placement.Filtering; + +[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] +public class RequiredSiloMetadataPlacementFilterAttribute(string[] orderedMetadataKeys) + : PlacementFilterAttribute(new RequiredSiloMetadataPlacementFilterStrategy(orderedMetadataKeys)); \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterStrategy.cs new file mode 100644 index 0000000000..ac79c94080 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterStrategy.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using Orleans.Metadata; + +namespace Orleans.Runtime.Placement.Filtering; + +public class RequiredSiloMetadataPlacementFilterStrategy(string[] metadataKeys) : PlacementFilterStrategy +{ + public string[] MetadataKeys { get; private set; } = metadataKeys; + + public RequiredSiloMetadataPlacementFilterStrategy() : this([]) + { + } + + public override void Initialize(GrainProperties properties) + { + base.Initialize(properties); + MetadataKeys = GetPlacementFilterGrainProperty("metadata-keys", properties).Split(","); + } + + protected override IEnumerable> GetAdditionalGrainProperties(IServiceProvider services, Type grainClass, GrainType grainType, + IReadOnlyDictionary existingProperties) + { + yield return new KeyValuePair("metadata-keys", String.Join(",", MetadataKeys)); + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/PlacementService.cs b/src/Orleans.Runtime/Placement/PlacementService.cs index cd17709a05..4688ccafeb 100644 --- a/src/Orleans.Runtime/Placement/PlacementService.cs +++ b/src/Orleans.Runtime/Placement/PlacementService.cs @@ -9,6 +9,7 @@ using Orleans.Configuration; using Orleans.Runtime.GrainDirectory; using Orleans.Runtime.Internal; +using Orleans.Runtime.Placement.Filtering; using Orleans.Runtime.Versions; namespace Orleans.Runtime.Placement @@ -28,6 +29,8 @@ internal class PlacementService : IPlacementContext private readonly ISiloStatusOracle _siloStatusOracle; private readonly bool _assumeHomogeneousSilosForTesting; private readonly PlacementWorker[] _workers; + private readonly PlacementFilterStrategyResolver _filterStrategyResolver; + private readonly PlacementFilterDirectorResolver _placementFilterDirectoryResolver; /// /// Create a instance. @@ -41,11 +44,15 @@ public PlacementService( GrainVersionManifest grainInterfaceVersions, CachedVersionSelectorManager versionSelectorManager, PlacementDirectorResolver directorResolver, - PlacementStrategyResolver strategyResolver) + PlacementStrategyResolver strategyResolver, + PlacementFilterStrategyResolver filterStrategyResolver, + PlacementFilterDirectorResolver placementFilterDirectoryResolver) { LocalSilo = localSiloDetails.SiloAddress; _strategyResolver = strategyResolver; _directorResolver = directorResolver; + _filterStrategyResolver = filterStrategyResolver; + _placementFilterDirectoryResolver = placementFilterDirectoryResolver; _logger = logger; _grainLocator = grainLocator; _grainInterfaceVersions = grainInterfaceVersions; @@ -117,6 +124,22 @@ public SiloAddress[] GetCompatibleSilos(PlacementTarget target) : _grainInterfaceVersions.GetSupportedSilos(grainType).Result; var compatibleSilos = silos.Intersect(AllActiveSilos).ToArray(); + + + var filters = _filterStrategyResolver.GetPlacementFilterStrategies(grainType); + if (filters.Length > 0) + { + IEnumerable filteredSilos = compatibleSilos; + foreach (var placementFilter in filters) + { + var director = _placementFilterDirectoryResolver.GetFilterDirector(placementFilter); + filteredSilos = director.Filter(placementFilter, target, filteredSilos); + } + + compatibleSilos = filteredSilos.ToArray(); + } + + if (compatibleSilos.Length == 0) { var allWithType = _grainInterfaceVersions.GetSupportedSilos(grainType).Result; From 3c5542eb0f97c33a21f49bb1a8e6e76049c74729 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Thu, 19 Dec 2024 13:16:32 -0800 Subject: [PATCH 02/16] Remove dead silos from metadata cache --- .../MembershipService/SiloMetadata/SiloMetadaCache.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs index afdcb39c95..d7bdf011ed 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs @@ -52,7 +52,7 @@ private async Task ProcessMembershipUpdates(CancellationToken ct) await foreach (var update in membershipTableManager.MembershipTableUpdates.WithCancellation(ct)) { // Add entries for members that aren't already in the cache - foreach (var membershipEntry in update.Entries) + foreach (var membershipEntry in update.Entries.Where(e => e.Value.Status != SiloStatus.Dead)) { if (!_metadata.ContainsKey(membershipEntry.Key)) { @@ -68,6 +68,12 @@ private async Task ProcessMembershipUpdates(CancellationToken ct) } } + // Add entries for members that aren't already in the cache + foreach (var membershipEntry in update.Entries.Where(e => e.Value.Status == SiloStatus.Dead)) + { + _metadata.TryRemove(membershipEntry.Key, out _); + } + // Remove entries for members that are no longer in the table foreach (var silo in _metadata.Keys.ToList()) { From a070530641d26c2b476b4ccb0d144e8ce4889ab5 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Thu, 19 Dec 2024 13:16:54 -0800 Subject: [PATCH 03/16] Allow overwriting of metadata values --- .../MembershipService/SiloMetadata/SiloMetadata.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs index a8adf8397c..395aa9d3df 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs @@ -12,6 +12,6 @@ public record SiloMetadata [Id(0)] public ImmutableDictionary Metadata { get; private set; } = ImmutableDictionary.Empty; - internal void AddMetadata(IEnumerable> metadata) => Metadata = Metadata.AddRange(metadata); - internal void AddMetadata(string key, string value) => Metadata = Metadata.Add(key, value); + internal void AddMetadata(IEnumerable> metadata) => Metadata = Metadata.SetItems(metadata); + internal void AddMetadata(string key, string value) => Metadata = Metadata.SetItem(key, value); } \ No newline at end of file From a6f1c8d4cab7a31929844bf2c8eeeca08ea971b0 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Thu, 19 Dec 2024 13:17:40 -0800 Subject: [PATCH 04/16] Adding Silo Metadata tests --- .../General/SiloMetadataTests.cs | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 test/TesterInternal/General/SiloMetadataTests.cs diff --git a/test/TesterInternal/General/SiloMetadataTests.cs b/test/TesterInternal/General/SiloMetadataTests.cs new file mode 100644 index 0000000000..32540ee52d --- /dev/null +++ b/test/TesterInternal/General/SiloMetadataTests.cs @@ -0,0 +1,115 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime.MembershipService.SiloMetadata; +using Orleans.TestingHost; +using TestExtensions; +using Xunit; + +namespace UnitTests.General; + +[TestCategory("SiloMetadata")] +public class SiloMetadataTests : TestClusterPerTest +{ + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.AddSiloBuilderConfigurator(); + } + + private class SiloConfigurator : ISiloConfigurator + { + public void Configure(ISiloBuilder hostBuilder) + { + hostBuilder.UseSiloMetadata(new Dictionary + { + {"host.id", Guid.NewGuid().ToString()} + }); + } + } + + [Fact, TestCategory("Functional")] + public async Task SiloMetadata_CanBeSetAndRead() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + AssertAllSiloMetadataMatchesOnAllSilos(); + } + + [Fact, TestCategory("Functional")] + public async Task SiloMetadata_NewSilosHaveMetadata() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.StartAdditionalSiloAsync(); + AssertAllSiloMetadataMatchesOnAllSilos(); + } + + [Fact, TestCategory("Functional")] + public async Task SiloMetadata_RemovedSiloHasNoMetadata() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + AssertAllSiloMetadataMatchesOnAllSilos(); + var first = HostedCluster.Silos.First(); + var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + + var second = HostedCluster.Silos.Skip(1).First(); + var metadata = firstSiloMetadataCache.GetMetadata(second.SiloAddress); + Assert.NotNull(metadata); + Assert.NotEmpty(metadata.Metadata); + + await HostedCluster.StopSiloAsync(second); + metadata = firstSiloMetadataCache.GetMetadata(second.SiloAddress); + Assert.NotNull(metadata); + Assert.Empty(metadata.Metadata); + } + + [Fact, TestCategory("Functional")] + public async Task SiloMetadata_BadSiloAddressHasNoMetadata() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + var first = this.HostedCluster.Silos.First(); + var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var metadata = firstSiloMetadataCache.GetMetadata(SiloAddress.Zero); + Assert.NotNull(metadata); + Assert.Empty(metadata.Metadata); + } + + private void AssertAllSiloMetadataMatchesOnAllSilos() + { + var exampleSiloMetadata = new Dictionary(); + var first = this.HostedCluster.Silos.First(); + var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + foreach (var otherSilo in this.HostedCluster.Silos) + { + var metadata = firstSiloMetadataCache.GetMetadata(otherSilo.SiloAddress); + Assert.NotNull(metadata); + Assert.NotNull(metadata.Metadata); + Assert.True(metadata.Metadata.ContainsKey("host.id")); + exampleSiloMetadata.Add(otherSilo.SiloAddress, metadata); + } + foreach (var hostedClusterSilo in this.HostedCluster.Silos.Skip(1)) + { + var sp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var siloMetadataCache = sp.GetRequiredService(); + var remoteMetadata = new Dictionary(); + foreach (var otherSilo in this.HostedCluster.Silos) + { + var metadata = siloMetadataCache.GetMetadata(otherSilo.SiloAddress); + Assert.NotNull(metadata); + Assert.NotNull(metadata.Metadata); + Assert.True(metadata.Metadata.ContainsKey("host.id")); + remoteMetadata.Add(otherSilo.SiloAddress, metadata); + } + //Assert that the two dictionaries have the same keys and the values for those keys are the same + Assert.Equal(exampleSiloMetadata.Count, remoteMetadata.Count); + foreach (var kvp in exampleSiloMetadata) + { + Assert.Equal(kvp.Value.Metadata.Count, remoteMetadata[kvp.Key].Metadata.Count); + foreach (var kvp2 in kvp.Value.Metadata) + { + Assert.True(remoteMetadata[kvp.Key].Metadata.TryGetValue(kvp2.Key, out var value), $"Key '{kvp2.Key}' not found in actual dictionary."); + Assert.Equal(kvp2.Value, value); + } + } + } + } +} \ No newline at end of file From e18d5edcc25685e166dfbdff89bb56cd035f8c9a Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Thu, 19 Dec 2024 20:19:49 -0800 Subject: [PATCH 05/16] Tests for Grain Placement Filter --- .../General/GrainPlacementFilterTests.cs | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 test/TesterInternal/General/GrainPlacementFilterTests.cs diff --git a/test/TesterInternal/General/GrainPlacementFilterTests.cs b/test/TesterInternal/General/GrainPlacementFilterTests.cs new file mode 100644 index 0000000000..79d59ed2ac --- /dev/null +++ b/test/TesterInternal/General/GrainPlacementFilterTests.cs @@ -0,0 +1,78 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Runtime.Placement; +using Orleans.Runtime.Placement.Filtering; +using Orleans.TestingHost; +using TestExtensions; +using Xunit; + +namespace UnitTests.General; + +[TestCategory("Placement"), TestCategory("Filters")] +public class GrainPlacementFilterTests : TestClusterPerTest +{ + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.AddSiloBuilderConfigurator(); + } + + private class SiloConfigurator : ISiloConfigurator + { + public void Configure(ISiloBuilder hostBuilder) + { + hostBuilder.ConfigureServices(services => + { + services.AddPlacementFilter(ServiceLifetime.Singleton); + }); + } + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_GrainWithoutFilterCanBeCalled() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + var managementGrain = this.Client.GetGrain(0); + var silos = await managementGrain.GetHosts(true); + Assert.NotNull(silos); + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_FilterIsTriggered() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var triggered = false; + var task = Task.Run(async () => + { + triggered = await TestPlacementFilterDirector.Triggered.WaitAsync(TimeSpan.FromSeconds(1)); + }); + var localOnlyGrain = Client.GetGrain(0); + await localOnlyGrain.Ping(); + await task; + Assert.True(triggered); + } +} + +[TestPlacementFilter] +public class TestFilteredGrain : Grain, ITestFilteredGrain +{ + public Task Ping() => Task.CompletedTask; +} + +public interface ITestFilteredGrain : IGrainWithIntegerKey +{ + Task Ping(); +} + +public class TestPlacementFilterAttribute() : PlacementFilterAttribute(new TestPlacementFilterStrategy()); + +public class TestPlacementFilterStrategy : PlacementFilterStrategy; + +public class TestPlacementFilterDirector() : IPlacementFilterDirector +{ + public static SemaphoreSlim Triggered { get; } = new(0); + + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + { + Triggered.Release(1); + return silos; + } +} From 8fcf93b0f3d48f25ff0ba5e6b1e4f20dd8ecfc51 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Fri, 20 Dec 2024 11:44:05 -0800 Subject: [PATCH 06/16] Tests for silo metadata placement filters --- .../SiloMetadataHostingExtensions.cs | 4 +- ...tchSiloMetadataPlacementFilterAttribute.cs | 15 ++ ...atchSiloMetadataPlacementFilterDirector.cs | 78 ++++++ ...tchSiloMetadataPlacementFilterStrategy.cs} | 14 +- ...redSiloMetadataPlacementFilterAttribute.cs | 7 - ...rredSiloMetadataPlacementFilterDirector.cs | 73 ------ ...equiredMatchSiloMetadataFilterDirector.cs} | 30 +-- ...tchSiloMetadataPlacementFilterAttribute.cs | 13 + ...tchSiloMetadataPlacementFilterStrategy.cs} | 4 +- ...redSiloMetadataPlacementFilterAttribute.cs | 7 - .../SiloMetadataPlacementFilterTests.cs | 247 ++++++++++++++++++ 11 files changed, 380 insertions(+), 112 deletions(-) create mode 100644 src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs create mode 100644 src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs rename src/Orleans.Runtime/Placement/Filtering/{PreferredSiloMetadataPlacementFilterStrategy.cs => PreferredMatchSiloMetadataPlacementFilterStrategy.cs} (52%) delete mode 100644 src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterAttribute.cs delete mode 100644 src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterDirector.cs rename src/Orleans.Runtime/Placement/Filtering/{RequiredSiloMetadataFilterDirector.cs => RequiredMatchSiloMetadataFilterDirector.cs} (58%) create mode 100644 src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs rename src/Orleans.Runtime/Placement/Filtering/{RequiredSiloMetadataPlacementFilterStrategy.cs => RequiredMatchSiloMetadataPlacementFilterStrategy.cs} (80%) delete mode 100644 src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterAttribute.cs create mode 100644 test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs index 3457bd0984..7afef23762 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs @@ -82,8 +82,8 @@ public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, Dictionary services.AddFromExisting, SiloMetadataCache>(); services.AddSingleton(); // Placement filters - services.AddPlacementFilter(ServiceLifetime.Transient); - services.AddPlacementFilter(ServiceLifetime.Transient); + services.AddPlacementFilter(ServiceLifetime.Transient); + services.AddPlacementFilter(ServiceLifetime.Transient); }); return builder; } diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs new file mode 100644 index 0000000000..f80955e855 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs @@ -0,0 +1,15 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Orleans.Runtime.Placement.Filtering; + +/// +/// Attribute to specify the preferred match silo metadata placement filter that preferentially filters down to silos where the metadata matches the local (calling) silo metadata. +/// +/// Ordered set of metadata keys to try to match. The earlier entries are considered less important and will be dropped to find a less-specific match if sufficient more-specific matches do not have enough results. +/// The minimum desired candidates to filter. This is to balance meeting the metadata preferences with not overloading a single or small set of silos with activations. Set this to 1 if you only want the best matches, even if there's only one silo that is currently the best match. +/// Example: If keys ["first","second"] are specified, then it will attempt to return only silos where both keys match the local silo's metadata values. If there are not sufficient silos matching both, then it will also include silos matching only the second key. Finally, if there are still fewer than minCandidates results then it will include all silos. +[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] +[Experimental("ORLEANSEXP004")] +public class PreferredMatchSiloMetadataPlacementFilterAttribute(string[] orderedMetadataKeys, int minCandidates = 2) + : PlacementFilterAttribute(new PreferredMatchSiloMetadataPlacementFilterStrategy(orderedMetadataKeys, minCandidates)); \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs new file mode 100644 index 0000000000..b847ffbf08 --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs @@ -0,0 +1,78 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Orleans.Runtime.MembershipService.SiloMetadata; +#nullable enable +namespace Orleans.Runtime.Placement.Filtering; + +internal class PreferredMatchSiloMetadataPlacementFilterDirector( + ILocalSiloDetails localSiloDetails, + ISiloMetadataCache siloMetadataCache) + : IPlacementFilterDirector +{ + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + { + var preferredMatchSiloMetadataPlacementFilterStrategy = (filterStrategy as PreferredMatchSiloMetadataPlacementFilterStrategy); + var minCandidates = preferredMatchSiloMetadataPlacementFilterStrategy?.MinCandidates ?? 1; + var orderedMetadataKeys = preferredMatchSiloMetadataPlacementFilterStrategy?.OrderedMetadataKeys ?? []; + + var localSiloMetadata = siloMetadataCache.GetMetadata(localSiloDetails.SiloAddress).Metadata; + + if (localSiloMetadata.Count == 0) + { + return silos; + } + + var siloList = silos.ToList(); + if (siloList.Count <= minCandidates) + { + return siloList; + } + + // return the list of silos that match the most metadata keys. The first key in the list is the least important. + // This means that the last key in the list is the most important. + // If no silos match any metadata keys, return the original list of silos. + var maxScore = 0; + var siloScores = new int[siloList.Count]; + var scoreCounts = new int[orderedMetadataKeys.Length+1]; + for (var i = 0; i < siloList.Count; i++) + { + var siloMetadata = siloMetadataCache.GetMetadata(siloList[i]).Metadata; + var siloScore = 0; + for (var j = orderedMetadataKeys.Length - 1; j >= 0; --j) + { + if (siloMetadata.TryGetValue(orderedMetadataKeys[j], out var siloMetadataValue) && + localSiloMetadata.TryGetValue(orderedMetadataKeys[j], out var localSiloMetadataValue) && + siloMetadataValue == localSiloMetadataValue) + { + siloScore = ++siloScores[i]; + maxScore = Math.Max(maxScore, siloScore); + } + else + { + break; + } + } + scoreCounts[siloScore]++; + } + + if (maxScore == 0) + { + return siloList; + } + + var candidateCount = 0; + var scoreCutOff = orderedMetadataKeys.Length; + for (var i = scoreCounts.Length-1; i >= 0; i--) + { + candidateCount += scoreCounts[i]; + if (candidateCount >= minCandidates) + { + scoreCutOff = i; + break; + } + } + + return siloList.Where((_, i) => siloScores[i] >= scoreCutOff); + } +} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs similarity index 52% rename from src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterStrategy.cs rename to src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs index 6316fd0c0b..74c551b5c8 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterStrategy.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs @@ -4,11 +4,13 @@ namespace Orleans.Runtime.Placement.Filtering; -public class PreferredSiloMetadataPlacementFilterStrategy(string[] orderedMetadataKeys) : PlacementFilterStrategy +public class PreferredMatchSiloMetadataPlacementFilterStrategy(string[] orderedMetadataKeys, int minCandidates) + : PlacementFilterStrategy { public string[] OrderedMetadataKeys { get; set; } = orderedMetadataKeys; + public int MinCandidates { get; set; } = minCandidates; - public PreferredSiloMetadataPlacementFilterStrategy() : this([]) + public PreferredMatchSiloMetadataPlacementFilterStrategy() : this([], 1) { } @@ -16,11 +18,19 @@ public override void Initialize(GrainProperties properties) { base.Initialize(properties); OrderedMetadataKeys = GetPlacementFilterGrainProperty("ordered-metadata-keys", properties).Split(","); + var minCandidatesProperty = GetPlacementFilterGrainProperty("min-candidates", properties); + if (!int.TryParse(minCandidatesProperty, out var parsedMinCandidates)) + { + throw new ArgumentException("Invalid min-candidates property value."); + } + + MinCandidates = parsedMinCandidates; } protected override IEnumerable> GetAdditionalGrainProperties(IServiceProvider services, Type grainClass, GrainType grainType, IReadOnlyDictionary existingProperties) { yield return new KeyValuePair("ordered-metadata-keys", string.Join(",", OrderedMetadataKeys)); + yield return new KeyValuePair("min-candidates", MinCandidates.ToString()); } } \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterAttribute.cs deleted file mode 100644 index 1c2183fcf1..0000000000 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterAttribute.cs +++ /dev/null @@ -1,7 +0,0 @@ -using System; - -namespace Orleans.Runtime.Placement.Filtering; - -[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] -public class PreferredSiloMetadataPlacementFilterAttribute(string[] orderedMetadataKeys) - : PlacementFilterAttribute(new PreferredSiloMetadataPlacementFilterStrategy(orderedMetadataKeys)); \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterDirector.cs deleted file mode 100644 index e6d08aa060..0000000000 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredSiloMetadataPlacementFilterDirector.cs +++ /dev/null @@ -1,73 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Orleans.Runtime.MembershipService.SiloMetadata; -#nullable enable -namespace Orleans.Runtime.Placement.Filtering; - -internal class PreferredSiloMetadataPlacementFilterDirector( - ILocalSiloDetails localSiloDetails, - ISiloMetadataCache siloMetadataCache) - : IPlacementFilterDirector -{ - public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) - { - var orderedMetadataKeys = (filterStrategy as PreferredSiloMetadataPlacementFilterStrategy)?.OrderedMetadataKeys ?? []; - var localSiloMetadata = siloMetadataCache.GetMetadata(localSiloDetails.SiloAddress).Metadata; - - if (localSiloMetadata.Count == 0) - { - // yield return all silos if no metadata keys are configured - foreach (var silo in silos) - { - yield return silo; - } - } - else - { - // return the list of silos that match the most metadata keys. The first key in the list is the least important. - // This means that the last key in the list is the most important. - // If no silos match any metadata keys, return the original list of silos. - - var siloList = silos.ToList(); - var maxScore = 0; - var siloScores = new int[siloList.Count]; - for (var i = 0; i < siloList.Count; i++) - { - var siloMetadata = siloMetadataCache.GetMetadata(siloList[i]).Metadata; - for (var j = orderedMetadataKeys.Length - 1; j >= 0; --j) - { - if (siloMetadata.TryGetValue(orderedMetadataKeys[j], out var siloMetadataValue) && - localSiloMetadata.TryGetValue(orderedMetadataKeys[j], out var localSiloMetadataValue) && - siloMetadataValue == localSiloMetadataValue) - { - var newScore = siloScores[i]++; - maxScore = Math.Max(maxScore, newScore); - } - else - { - break; - } - } - } - - if (maxScore == 0) - { - // yield return all silos if no silos match any metadata keys - foreach (var silo in siloList) - { - yield return silo; - } - } - - // return the list of silos that match the most metadata keys - for (var i = 0; i < siloScores.Length; i++) - { - if (siloScores[i] == maxScore) - { - yield return siloList[i]; - } - } - } - } -} \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataFilterDirector.cs similarity index 58% rename from src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataFilterDirector.cs rename to src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataFilterDirector.cs index ba1fd242bd..4b5a3f85f4 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataFilterDirector.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataFilterDirector.cs @@ -4,35 +4,27 @@ namespace Orleans.Runtime.Placement.Filtering; -internal class RequiredSiloMetadataFilterDirector(ILocalSiloDetails localSiloDetails, ISiloMetadataCache siloMetadataCache) +internal class RequiredMatchSiloMetadataFilterDirector(ILocalSiloDetails localSiloDetails, ISiloMetadataCache siloMetadataCache) : IPlacementFilterDirector { public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) { - var metadataKeys = (filterStrategy as RequiredSiloMetadataPlacementFilterStrategy)?.MetadataKeys ?? []; + var metadataKeys = (filterStrategy as RequiredMatchSiloMetadataPlacementFilterStrategy)?.MetadataKeys ?? []; // yield return all silos if no silos match any metadata keys if (metadataKeys.Length == 0) { - foreach (var silo in silos) - { - yield return silo; - } + return silos; } - else + + var localMetadata = siloMetadataCache.GetMetadata(localSiloDetails.SiloAddress); + var localRequiredMetadata = GetMetadata(localMetadata, metadataKeys); + + return silos.Where(silo => { - var localMetadata = siloMetadataCache.GetMetadata(localSiloDetails.SiloAddress); - var localRequiredMetadata = GetMetadata(localMetadata, metadataKeys); - - foreach (var silo in silos) - { - var remoteMetadata = siloMetadataCache.GetMetadata(silo); - if(DoesMetadataMatch(localRequiredMetadata, remoteMetadata, metadataKeys)) - { - yield return silo; - } - } - } + var remoteMetadata = siloMetadataCache.GetMetadata(silo); + return DoesMetadataMatch(localRequiredMetadata, remoteMetadata, metadataKeys); + }); } private static bool DoesMetadataMatch(string[] localMetadata, SiloMetadata siloMetadata, string[] metadataKeys) diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs new file mode 100644 index 0000000000..2bdda2645f --- /dev/null +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs @@ -0,0 +1,13 @@ +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Orleans.Runtime.Placement.Filtering; + +/// +/// Attribute to specify that a silo must have a specific metadata key-value pair matching the local (calling) silo to be considered for placement. +/// +/// +[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] +[Experimental("ORLEANSEXP004")] +public class RequiredMatchSiloMetadataPlacementFilterAttribute(string[] metadataKeys) + : PlacementFilterAttribute(new RequiredMatchSiloMetadataPlacementFilterStrategy(metadataKeys)); \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs similarity index 80% rename from src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterStrategy.cs rename to src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs index ac79c94080..fc0ca730fe 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterStrategy.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs @@ -4,11 +4,11 @@ namespace Orleans.Runtime.Placement.Filtering; -public class RequiredSiloMetadataPlacementFilterStrategy(string[] metadataKeys) : PlacementFilterStrategy +public class RequiredMatchSiloMetadataPlacementFilterStrategy(string[] metadataKeys) : PlacementFilterStrategy { public string[] MetadataKeys { get; private set; } = metadataKeys; - public RequiredSiloMetadataPlacementFilterStrategy() : this([]) + public RequiredMatchSiloMetadataPlacementFilterStrategy() : this([]) { } diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterAttribute.cs deleted file mode 100644 index 180a409c46..0000000000 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredSiloMetadataPlacementFilterAttribute.cs +++ /dev/null @@ -1,7 +0,0 @@ -using System; - -namespace Orleans.Runtime.Placement.Filtering; - -[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] -public class RequiredSiloMetadataPlacementFilterAttribute(string[] orderedMetadataKeys) - : PlacementFilterAttribute(new RequiredSiloMetadataPlacementFilterStrategy(orderedMetadataKeys)); \ No newline at end of file diff --git a/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs b/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs new file mode 100644 index 0000000000..8a3604a2c6 --- /dev/null +++ b/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs @@ -0,0 +1,247 @@ +using Microsoft.Extensions.DependencyInjection; +using Orleans.Placement; +using Orleans.Runtime.MembershipService.SiloMetadata; +using Orleans.Runtime.Placement.Filtering; +using Orleans.TestingHost; +using TestExtensions; +using Xunit; + +namespace UnitTests.General; + +[TestCategory("Placement"), TestCategory("Filters")] +public class SiloMetadataPlacementFilterTests : TestClusterPerTest +{ + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.AddSiloBuilderConfigurator(); + } + + private class SiloConfigurator : ISiloConfigurator + { + public void Configure(ISiloBuilder hostBuilder) + { + hostBuilder.UseSiloMetadata(new Dictionary + { + {"first", "1"}, + {"second", "2"}, + {"third", "3"}, + {"unique", Guid.NewGuid().ToString()} + }); + } + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_GrainWithoutFilterCanBeCalled() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + var managementGrain = this.Client.GetGrain(0); + var silos = await managementGrain.GetHosts(true); + Assert.NotNull(silos); + } + + /// + /// Unique silo metadata is set up to be different for each silo, so this will require that placement happens on the calling silo. + /// + /// + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_RequiredFilterCanBeCalled() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + int id = 0; + foreach (var hostedClusterSilo in HostedCluster.Silos) + { + for (int i = 0; i < 50; i++) + { + ++id; + var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var managementGrain = firstSiloMetadataCache.GetGrain(id); + var hostingSilo = await managementGrain.GetHostingSilo(); + Assert.NotNull(hostingSilo); + Assert.Equal(hostedClusterSilo.SiloAddress, hostingSilo); + } + } + } + + /// + /// Unique silo metadata is set up to be different for each silo, so this will require that placement happens on the calling silo because it is the only one that matches. + /// + /// + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_PreferredFilterCanBeCalled() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + int id = 0; + foreach (var hostedClusterSilo in HostedCluster.Silos) + { + for (int i = 0; i < 50; i++) + { + ++id; + var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var managementGrain = firstSiloMetadataCache.GetGrain(id); + var hostingSilo = await managementGrain.GetHostingSilo(); + Assert.NotNull(hostingSilo); + Assert.Equal(hostedClusterSilo.SiloAddress, hostingSilo); + } + } + } + + /// + /// Unique silo metadata is set up to be different for each silo, so this will still place on any of the two silos since just the matching silos (just the one) is not enough to make the minimum desired candidates. + /// + /// + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_PreferredMin2FilterCanBeCalled() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + int id = 0; + foreach (var hostedClusterSilo in HostedCluster.Silos) + { + var dict = new Dictionary(); + foreach (var clusterSilo in HostedCluster.Silos) + { + dict[clusterSilo.SiloAddress] = 0; + } + for (int i = 0; i < 50; i++) + { + ++id; + var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var managementGrain = firstSiloMetadataCache.GetGrain(id); + var hostingSilo = await managementGrain.GetHostingSilo(); + Assert.NotNull(hostingSilo); + dict[hostingSilo] = dict.TryGetValue(hostingSilo, out var count) ? count + 1 : 1; + } + + foreach (var kv in dict) + { + Assert.True(kv.Value >= 1, $"Silo {kv.Key} did not host at least 1 grain"); + } + } + } + + /// + /// Unique silo metadata is set up to be different for each silo, so this will still place on any of the two silos since just the matching silos (just the one) is not enough to make the minimum desired candidates. + /// + /// + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_PreferredMin2FilterCanBeCalledWithLargerCluster() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.StartAdditionalSiloAsync(); + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + int id = 0; + foreach (var hostedClusterSilo in HostedCluster.Silos) + { + var dict = new Dictionary(); + foreach (var clusterSilo in HostedCluster.Silos) + { + dict[clusterSilo.SiloAddress] = 0; + } + for (int i = 0; i < 50; i++) + { + ++id; + var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var managementGrain = firstSiloMetadataCache.GetGrain(id); + var hostingSilo = await managementGrain.GetHostingSilo(); + Assert.NotNull(hostingSilo); + dict[hostingSilo] = dict.TryGetValue(hostingSilo, out var count) ? count + 1 : 1; + } + + foreach (var kv in dict) + { + Assert.True(kv.Value >= 1, $"Silo {kv.Key} did not host at least 1 grain"); + } + } + } + + /// + /// If no metadata key is defined then it should fall back to matching all silos + /// + /// + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_PreferredNoMetadataFilterCanBeCalled() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.StartAdditionalSiloAsync(); + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + int id = 0; + foreach (var hostedClusterSilo in HostedCluster.Silos) + { + var dict = new Dictionary(); + foreach (var clusterSilo in HostedCluster.Silos) + { + dict[clusterSilo.SiloAddress] = 0; + } + for (int i = 0; i < 50; i++) + { + ++id; + var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var managementGrain = firstSiloMetadataCache.GetGrain(id); + var hostingSilo = await managementGrain.GetHostingSilo(); + Assert.NotNull(hostingSilo); + dict[hostingSilo] = dict.TryGetValue(hostingSilo, out var count) ? count + 1 : 1; + } + + foreach (var kv in dict) + { + Assert.True(kv.Value >= 1, $"Silo {kv.Key} did not host at least 1 grain"); + } + } + } +} + +public interface IUniqueRequiredMatchFilteredGrain : IGrainWithIntegerKey +{ + Task GetHostingSilo(); +} + +#pragma warning disable ORLEANSEXP004 +[RequiredMatchSiloMetadataPlacementFilter(["unique"])] +#pragma warning restore ORLEANSEXP004 +public class UniqueRequiredMatchFilteredGrain(ILocalSiloDetails localSiloDetails) : Grain, IUniqueRequiredMatchFilteredGrain +{ + public Task GetHostingSilo() => Task.FromResult(localSiloDetails.SiloAddress); +} +public interface IPreferredMatchFilteredGrain : IGrainWithIntegerKey +{ + Task GetHostingSilo(); +} + +#pragma warning disable ORLEANSEXP004 +[PreferredMatchSiloMetadataPlacementFilter(["unique"], 1)] +#pragma warning restore ORLEANSEXP004 +public class PreferredMatchFilteredGrain(ILocalSiloDetails localSiloDetails) : Grain, IPreferredMatchFilteredGrain +{ + public Task GetHostingSilo() => Task.FromResult(localSiloDetails.SiloAddress); +} + + +public interface IPreferredMatchMin2FilteredGrain : IGrainWithIntegerKey +{ + Task GetHostingSilo(); +} + +#pragma warning disable ORLEANSEXP004 +[PreferredMatchSiloMetadataPlacementFilter(["unique"])] +#pragma warning restore ORLEANSEXP004 +public class PreferredMatchMinTwoFilteredGrain(ILocalSiloDetails localSiloDetails) : Grain, IPreferredMatchMin2FilteredGrain +{ + public Task GetHostingSilo() => Task.FromResult(localSiloDetails.SiloAddress); +} + +#pragma warning disable ORLEANSEXP004 +[PreferredMatchSiloMetadataPlacementFilter(["not.there"])] +#pragma warning restore ORLEANSEXP004 +public class PreferredMatchNoMetadataFilteredGrain(ILocalSiloDetails localSiloDetails) : Grain, IPreferredMatchNoMetadataFilteredGrain +{ + public Task GetHostingSilo() => Task.FromResult(localSiloDetails.SiloAddress); +} + +public interface IPreferredMatchNoMetadataFilteredGrain : IGrainWithIntegerKey +{ + Task GetHostingSilo(); +} \ No newline at end of file From 72c514de32d440f10d7c3c367880be6a30268842 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Fri, 20 Dec 2024 11:45:07 -0800 Subject: [PATCH 07/16] Adding test category --- test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs b/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs index 8a3604a2c6..7b765c7b52 100644 --- a/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs +++ b/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs @@ -8,7 +8,7 @@ namespace UnitTests.General; -[TestCategory("Placement"), TestCategory("Filters")] +[TestCategory("Placement"), TestCategory("Filters"), TestCategory("SiloMetadata")] public class SiloMetadataPlacementFilterTests : TestClusterPerTest { protected override void ConfigureTestCluster(TestClusterBuilder builder) From 5109fb6c2c6e2fcb5fae2a3fddeb7520eb7d18e7 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Fri, 20 Dec 2024 12:21:47 -0800 Subject: [PATCH 08/16] Testing loading silo metadata from config --- .../SiloMetadataHostingExtensions.cs | 6 +- .../SiloMetadataPlacementFilterTests.cs | 1 - .../General/SiloMetadataTests.cs | 92 ++++++++++++++++--- 3 files changed, 82 insertions(+), 17 deletions(-) diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs index 7afef23762..1530d0316e 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs @@ -28,15 +28,15 @@ public static class SiloMetadataHostingExtensions /// Silo builder /// Configuration to pull from /// - /// Get the ORLEANS__METADATA section from config + /// Get the Orleans:Metadata section from config /// Key/value pairs in configuration as a will look like this as environment variables: - /// ORLEANS__METADATA__key1=value1 + /// Orleans:Metadata:key1=value1 /// /// public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, IConfiguration configuration) { - var metadataConfigSection = builder.Configuration.GetSection("ORLEANS").GetSection("METADATA"); + var metadataConfigSection = configuration.GetSection("Orleans").GetSection("Metadata"); return builder.UseSiloMetadata(metadataConfigSection); } diff --git a/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs b/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs index 7b765c7b52..15fcce09a1 100644 --- a/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs +++ b/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs @@ -1,5 +1,4 @@ using Microsoft.Extensions.DependencyInjection; -using Orleans.Placement; using Orleans.Runtime.MembershipService.SiloMetadata; using Orleans.Runtime.Placement.Filtering; using Orleans.TestingHost; diff --git a/test/TesterInternal/General/SiloMetadataTests.cs b/test/TesterInternal/General/SiloMetadataTests.cs index 32540ee52d..fbd847c5de 100644 --- a/test/TesterInternal/General/SiloMetadataTests.cs +++ b/test/TesterInternal/General/SiloMetadataTests.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Orleans.Runtime.MembershipService.SiloMetadata; using Orleans.TestingHost; @@ -6,6 +7,59 @@ namespace UnitTests.General; + +[TestCategory("SiloMetadata")] +public class SiloMetadataConfigTests : TestClusterPerTest +{ + protected override void ConfigureTestCluster(TestClusterBuilder builder) + { + builder.AddSiloBuilderConfigurator(); + } + + private class SiloConfigurator : ISiloConfigurator + { + public static readonly List> Metadata = + [ + new("Orleans:Metadata:first", "1"), + new("Orleans:Metadata:second", "2"), + new("Orleans:Metadata:third", "3") + ]; + + public void Configure(ISiloBuilder hostBuilder) + { + var config = new ConfigurationBuilder() + .AddInMemoryCollection(Metadata) + .Build(); + hostBuilder.UseSiloMetadata(config); + } + } + + [Fact, TestCategory("Functional")] + public async Task SiloMetadata_CanBeSetAndRead() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(SiloConfigurator.Metadata.Select(kv => kv.Key.Split(':').Last()).ToArray()); + } + + [Fact, TestCategory("Functional")] + public async Task SiloMetadata_HasConfiguredValues() + { + await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + + var first = HostedCluster.Silos.First(); + var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var metadata = firstSiloMetadataCache.GetMetadata(first.SiloAddress); + Assert.NotNull(metadata); + Assert.NotNull(metadata.Metadata); + Assert.Equal(SiloConfigurator.Metadata.Count, metadata.Metadata.Count); + foreach (var kv in SiloConfigurator.Metadata) + { + Assert.Equal(kv.Value, metadata.Metadata[kv.Key.Split(':').Last()]); + } + } +} + [TestCategory("SiloMetadata")] public class SiloMetadataTests : TestClusterPerTest { @@ -29,7 +83,7 @@ public void Configure(ISiloBuilder hostBuilder) public async Task SiloMetadata_CanBeSetAndRead() { await this.HostedCluster.WaitForLivenessToStabilizeAsync(); - AssertAllSiloMetadataMatchesOnAllSilos(); + HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); } [Fact, TestCategory("Functional")] @@ -37,14 +91,14 @@ public async Task SiloMetadata_NewSilosHaveMetadata() { await this.HostedCluster.WaitForLivenessToStabilizeAsync(); await HostedCluster.StartAdditionalSiloAsync(); - AssertAllSiloMetadataMatchesOnAllSilos(); + HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); } [Fact, TestCategory("Functional")] public async Task SiloMetadata_RemovedSiloHasNoMetadata() { await this.HostedCluster.WaitForLivenessToStabilizeAsync(); - AssertAllSiloMetadataMatchesOnAllSilos(); + HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); var first = HostedCluster.Silos.First(); var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); var firstSiloMetadataCache = firstSp.GetRequiredService(); @@ -71,34 +125,45 @@ public async Task SiloMetadata_BadSiloAddressHasNoMetadata() Assert.NotNull(metadata); Assert.Empty(metadata.Metadata); } +} - private void AssertAllSiloMetadataMatchesOnAllSilos() +public static class SiloMetadataTestExtensions +{ + public static void AssertAllSiloMetadataMatchesOnAllSilos(this TestCluster hostedCluster, string[] expectedKeys) { var exampleSiloMetadata = new Dictionary(); - var first = this.HostedCluster.Silos.First(); - var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); + var first = hostedCluster.Silos.First(); + var firstSp = hostedCluster.GetSiloServiceProvider(first.SiloAddress); var firstSiloMetadataCache = firstSp.GetRequiredService(); - foreach (var otherSilo in this.HostedCluster.Silos) + foreach (var otherSilo in hostedCluster.Silos) { var metadata = firstSiloMetadataCache.GetMetadata(otherSilo.SiloAddress); Assert.NotNull(metadata); Assert.NotNull(metadata.Metadata); - Assert.True(metadata.Metadata.ContainsKey("host.id")); + foreach (var expectedKey in expectedKeys) + { + Assert.True(metadata.Metadata.ContainsKey(expectedKey)); + } exampleSiloMetadata.Add(otherSilo.SiloAddress, metadata); } - foreach (var hostedClusterSilo in this.HostedCluster.Silos.Skip(1)) + + foreach (var hostedClusterSilo in hostedCluster.Silos.Skip(1)) { - var sp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var sp = hostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); var siloMetadataCache = sp.GetRequiredService(); var remoteMetadata = new Dictionary(); - foreach (var otherSilo in this.HostedCluster.Silos) + foreach (var otherSilo in hostedCluster.Silos) { var metadata = siloMetadataCache.GetMetadata(otherSilo.SiloAddress); Assert.NotNull(metadata); Assert.NotNull(metadata.Metadata); - Assert.True(metadata.Metadata.ContainsKey("host.id")); + foreach (var expectedKey in expectedKeys) + { + Assert.True(metadata.Metadata.ContainsKey(expectedKey)); + } remoteMetadata.Add(otherSilo.SiloAddress, metadata); } + //Assert that the two dictionaries have the same keys and the values for those keys are the same Assert.Equal(exampleSiloMetadata.Count, remoteMetadata.Count); foreach (var kvp in exampleSiloMetadata) @@ -106,7 +171,8 @@ private void AssertAllSiloMetadataMatchesOnAllSilos() Assert.Equal(kvp.Value.Metadata.Count, remoteMetadata[kvp.Key].Metadata.Count); foreach (var kvp2 in kvp.Value.Metadata) { - Assert.True(remoteMetadata[kvp.Key].Metadata.TryGetValue(kvp2.Key, out var value), $"Key '{kvp2.Key}' not found in actual dictionary."); + Assert.True(remoteMetadata[kvp.Key].Metadata.TryGetValue(kvp2.Key, out var value), + $"Key '{kvp2.Key}' not found in actual dictionary."); Assert.Equal(kvp2.Value, value); } } From 5779205dacecdd76f0faf87d81671ee753afc4b5 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Fri, 20 Dec 2024 13:41:07 -0800 Subject: [PATCH 09/16] Required and Preferred match filtering unit tests --- .../SiloMetadataHostingExtensions.cs | 2 +- ...tchSiloMetadataPlacementFilterDirector.cs} | 2 +- .../GrainPlacementFilterTests.cs | 6 +- ...iloMetadataPlacementFilterDirectorTests.cs | 176 ++++++++++++++++++ ...iloMetadataPlacementFilterDirectorTests.cs | 141 ++++++++++++++ .../SiloMetadataPlacementFilterTests.cs | 40 ++-- .../TestLocalSiloDetails.cs | 19 ++ .../TestSiloMetadataCache.cs | 15 ++ .../SiloMetadataTests.cs | 16 +- test/TesterInternal/TesterInternal.csproj | 2 +- 10 files changed, 385 insertions(+), 34 deletions(-) rename src/Orleans.Runtime/Placement/Filtering/{RequiredMatchSiloMetadataFilterDirector.cs => RequiredMatchSiloMetadataPlacementFilterDirector.cs} (92%) rename test/TesterInternal/{General => PlacementFilterTests}/GrainPlacementFilterTests.cs (92%) create mode 100644 test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs create mode 100644 test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs rename test/TesterInternal/{General => PlacementFilterTests}/SiloMetadataPlacementFilterTests.cs (90%) create mode 100644 test/TesterInternal/PlacementFilterTests/TestLocalSiloDetails.cs create mode 100644 test/TesterInternal/PlacementFilterTests/TestSiloMetadataCache.cs rename test/TesterInternal/{General => SiloMetadataTests}/SiloMetadataTests.cs (93%) diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs index 1530d0316e..200577e263 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs @@ -83,7 +83,7 @@ public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, Dictionary services.AddSingleton(); // Placement filters services.AddPlacementFilter(ServiceLifetime.Transient); - services.AddPlacementFilter(ServiceLifetime.Transient); + services.AddPlacementFilter(ServiceLifetime.Transient); }); return builder; } diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs similarity index 92% rename from src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataFilterDirector.cs rename to src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs index 4b5a3f85f4..ffbc4782f6 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataFilterDirector.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs @@ -4,7 +4,7 @@ namespace Orleans.Runtime.Placement.Filtering; -internal class RequiredMatchSiloMetadataFilterDirector(ILocalSiloDetails localSiloDetails, ISiloMetadataCache siloMetadataCache) +internal class RequiredMatchSiloMetadataPlacementFilterDirector(ILocalSiloDetails localSiloDetails, ISiloMetadataCache siloMetadataCache) : IPlacementFilterDirector { public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) diff --git a/test/TesterInternal/General/GrainPlacementFilterTests.cs b/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs similarity index 92% rename from test/TesterInternal/General/GrainPlacementFilterTests.cs rename to test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs index 79d59ed2ac..9f34e11e9b 100644 --- a/test/TesterInternal/General/GrainPlacementFilterTests.cs +++ b/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs @@ -5,7 +5,7 @@ using TestExtensions; using Xunit; -namespace UnitTests.General; +namespace UnitTests.PlacementFilterTests; [TestCategory("Placement"), TestCategory("Filters")] public class GrainPlacementFilterTests : TestClusterPerTest @@ -29,8 +29,8 @@ public void Configure(ISiloBuilder hostBuilder) [Fact, TestCategory("Functional")] public async Task PlacementFilter_GrainWithoutFilterCanBeCalled() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); - var managementGrain = this.Client.GetGrain(0); + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var managementGrain = Client.GetGrain(0); var silos = await managementGrain.GetHosts(true); Assert.NotNull(silos); } diff --git a/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs b/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs new file mode 100644 index 0000000000..9274d75f96 --- /dev/null +++ b/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs @@ -0,0 +1,176 @@ +using System.Net; +using Orleans.Metadata; +using Orleans.Runtime.MembershipService.SiloMetadata; +using Orleans.Runtime.Placement; +using Orleans.Runtime.Placement.Filtering; +using Xunit; + +namespace UnitTests.PlacementFilterTests; + +[TestCategory("Placement"), TestCategory("Filters"), TestCategory("SiloMetadata")] +public class PreferredMatchSiloMetadataPlacementFilterDirectorTests +{ + [Fact, TestCategory("Functional")] + public void CanBeCreated() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var director = new PreferredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + { testLocalSiloAddress, SiloMetadata.Empty } + })); + Assert.NotNull(director); + } + + [Fact, TestCategory("Functional")] + public void CanBeCalled() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var director = new PreferredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testLocalSiloAddress, SiloMetadata.Empty} + })); + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(), new PlacementTarget(), + new List() { testLocalSiloAddress } + ).ToList(); + Assert.NotNull(result); + Assert.NotEmpty(result); + } + + [Fact, TestCategory("Functional")] + public void FiltersToAllWhenNoEntry() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var siloMetadata = new SiloMetadata(); + siloMetadata.AddMetadata("metadata.key", "something"); + var director = new PreferredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress, SiloMetadata.Empty}, + {testLocalSiloAddress, siloMetadata}, + })); + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 1), new PlacementTarget(), + new List() { testOtherSiloAddress }).ToList(); + Assert.NotEmpty(result); + } + + + [Theory, TestCategory("Functional")] + [InlineData(1, 3, "no.match")] + [InlineData(2, 3, "no.match")] + [InlineData( 1, 1, "one.match")] + [InlineData( 2, 3, "one.match")] + [InlineData( 1, 2, "two.match")] + [InlineData( 2, 2, "two.match")] + [InlineData( 3, 3, "two.match")] + [InlineData( 1, 3, "all.match")] + [InlineData( 2, 3, "all.match")] + + public void FiltersOnSingleMetadata(int minCandidates, int expectedCount, string key) + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress1 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var testOtherSiloAddress2 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1002, 1); + var testOtherSiloAddress3 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1003, 1); + var localSiloMetadata = new SiloMetadata(); + localSiloMetadata.AddMetadata("all.match", "match"); + localSiloMetadata.AddMetadata("one.match", "match"); + localSiloMetadata.AddMetadata("two.match", "match"); + localSiloMetadata.AddMetadata("no.match", "match"); + var otherSiloMetadata1 = new SiloMetadata(); + otherSiloMetadata1.AddMetadata("all.match", "match"); + otherSiloMetadata1.AddMetadata("one.match", "match"); + otherSiloMetadata1.AddMetadata("two.match", "match"); + otherSiloMetadata1.AddMetadata("no.match", "nomatch"); + var otherSiloMetadata2 = new SiloMetadata(); + otherSiloMetadata2.AddMetadata("all.match", "match"); + otherSiloMetadata2.AddMetadata("one.match", "nomatch"); + otherSiloMetadata2.AddMetadata("two.match", "match"); + otherSiloMetadata2.AddMetadata("no.match", "nomatch"); + var otherSiloMetadata3 = new SiloMetadata(); + otherSiloMetadata3.AddMetadata("all.match", "match"); + otherSiloMetadata3.AddMetadata("one.match", "nomatch"); + otherSiloMetadata3.AddMetadata("two.match", "nomatch"); + otherSiloMetadata3.AddMetadata("no.match", "nomatch"); + var director = new PreferredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress1, otherSiloMetadata1}, + {testOtherSiloAddress2, otherSiloMetadata2}, + {testOtherSiloAddress3, otherSiloMetadata3}, + {testLocalSiloAddress, localSiloMetadata}, + })); + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy([key], minCandidates), new PlacementTarget(), + new List() { testOtherSiloAddress1, testOtherSiloAddress2, testOtherSiloAddress3 }).ToList(); + Assert.NotEmpty(result); + Assert.Equal(expectedCount, result.Count); + } + + [Theory, TestCategory("Functional")] + [InlineData(1, 3, "no.match", "all.match")] + [InlineData(1, 1, "no.match", "one.match", "two.match")] + [InlineData(2, 2, "no.match", "one.match", "two.match")] + [InlineData(3, 3, "no.match", "one.match", "two.match")] + [InlineData(1, 3, "all.match", "no.match")] + [InlineData(1, 1, "one.match", "all.match")] + [InlineData(2, 3, "one.match", "all.match")] + [InlineData(1, 1, "no.match", "one.match", "all.match")] + [InlineData(2, 3, "no.match", "one.match", "all.match")] + + public void FiltersOnMultipleMetadata(int minCandidates, int expectedCount, params string[] keys) + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress1 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var testOtherSiloAddress2 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1002, 1); + var testOtherSiloAddress3 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1003, 1); + var localSiloMetadata = new SiloMetadata(); + localSiloMetadata.AddMetadata("all.match", "match"); + localSiloMetadata.AddMetadata("one.match", "match"); + localSiloMetadata.AddMetadata("two.match", "match"); + localSiloMetadata.AddMetadata("no.match", "match"); + var otherSiloMetadata1 = new SiloMetadata(); + otherSiloMetadata1.AddMetadata("all.match", "match"); + otherSiloMetadata1.AddMetadata("one.match", "match"); + otherSiloMetadata1.AddMetadata("two.match", "match"); + otherSiloMetadata1.AddMetadata("no.match", "not.match"); + var otherSiloMetadata2 = new SiloMetadata(); + otherSiloMetadata2.AddMetadata("all.match", "match"); + otherSiloMetadata2.AddMetadata("one.match", "nomatch"); + otherSiloMetadata2.AddMetadata("two.match", "match"); + otherSiloMetadata2.AddMetadata("no.match", "not.match"); + var otherSiloMetadata3 = new SiloMetadata(); + otherSiloMetadata3.AddMetadata("all.match", "match"); + otherSiloMetadata3.AddMetadata("one.match", "not.match"); + otherSiloMetadata3.AddMetadata("two.match", "not.match"); + otherSiloMetadata3.AddMetadata("no.match", "not.match"); + var director = new PreferredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress1, otherSiloMetadata1}, + {testOtherSiloAddress2, otherSiloMetadata2}, + {testOtherSiloAddress3, otherSiloMetadata3}, + {testLocalSiloAddress, localSiloMetadata}, + })); + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(keys, minCandidates), new PlacementTarget(), + new List() { testOtherSiloAddress1, testOtherSiloAddress2, testOtherSiloAddress3 }).ToList(); + Assert.NotEmpty(result); + Assert.Equal(expectedCount, result.Count); + } +} \ No newline at end of file diff --git a/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs b/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs new file mode 100644 index 0000000000..f63afed282 --- /dev/null +++ b/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs @@ -0,0 +1,141 @@ +using System.Net; +using Orleans.Runtime.MembershipService.SiloMetadata; +using Orleans.Runtime.Placement; +using Orleans.Runtime.Placement.Filtering; +using Xunit; + +namespace UnitTests.PlacementFilterTests; + +[TestCategory("Placement"), TestCategory("Filters"), TestCategory("SiloMetadata")] +public class RequiredMatchSiloMetadataPlacementFilterDirectorTests +{ + [Fact, TestCategory("Functional")] + public void RequiredMatchSiloMetadataPlacementFilterDirector_CanBeCreated() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var director = new RequiredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testLocalSiloAddress, SiloMetadata.Empty} + })); + Assert.NotNull(director); + } + + [Fact, TestCategory("Functional")] + public void RequiredMatchSiloMetadataPlacementFilterDirector_CanBeCalled() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var director = new RequiredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testLocalSiloAddress, SiloMetadata.Empty} + })); + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(), new PlacementTarget(), + new List() { testLocalSiloAddress } + ).ToList(); + Assert.NotNull(result); + Assert.NotEmpty(result); + } + + [Fact, TestCategory("Functional")] + public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToNothingWhenNoEntry() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var siloMetadata = new SiloMetadata(); + siloMetadata.AddMetadata("metadata.key", "something"); + var director = new RequiredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress, SiloMetadata.Empty}, + {testLocalSiloAddress, siloMetadata}, + })); + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"]), new PlacementTarget(), + new List() { testOtherSiloAddress }).ToList(); + Assert.Empty(result); + } + + [Fact, TestCategory("Functional")] + public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToNothingWhenDifferentValue() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var localSiloMetadata = new SiloMetadata(); + localSiloMetadata.AddMetadata("metadata.key", "local"); + var otherSiloMetadata = new SiloMetadata(); + otherSiloMetadata.AddMetadata("metadata.key", "other"); + var director = new RequiredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress, otherSiloMetadata}, + {testLocalSiloAddress, localSiloMetadata}, + })); + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"]), new PlacementTarget(), + new List() { testOtherSiloAddress }).ToList(); + Assert.Empty(result); + } + + [Fact, TestCategory("Functional")] + public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToSiloWhenMatching() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var localSiloMetadata = new SiloMetadata(); + localSiloMetadata.AddMetadata("metadata.key", "same"); + var otherSiloMetadata = new SiloMetadata(); + otherSiloMetadata.AddMetadata("metadata.key", "same"); + var director = new RequiredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress, otherSiloMetadata}, + {testLocalSiloAddress, localSiloMetadata}, + })); + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"]), new PlacementTarget(), + new List() { testOtherSiloAddress }).ToList(); + Assert.NotEmpty(result); + } + + [Fact, TestCategory("Functional")] + public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToMultipleSilosWhenMatching() + { + var testLocalSiloAddress = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1000, 1); + var testOtherSiloAddress1 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1001, 1); + var testOtherSiloAddress2 = SiloAddress.New(IPAddress.Parse("1.1.1.1"), 1002, 1); + var localSiloMetadata = new SiloMetadata(); + localSiloMetadata.AddMetadata("metadata.key", "same"); + var otherSiloMetadata1 = new SiloMetadata(); + otherSiloMetadata1.AddMetadata("metadata.key", "same"); + var otherSiloMetadata2 = new SiloMetadata(); + otherSiloMetadata2.AddMetadata("metadata.key", "same"); + var director = new RequiredMatchSiloMetadataPlacementFilterDirector( + new TestLocalSiloDetails("name", "clusterId", "dnsHostName", + testLocalSiloAddress, + testLocalSiloAddress), + new TestSiloMetadataCache(new Dictionary() + { + {testOtherSiloAddress1, otherSiloMetadata1}, + {testOtherSiloAddress2, otherSiloMetadata2}, + {testLocalSiloAddress, localSiloMetadata}, + })); + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"]), new PlacementTarget(), + new List() { testOtherSiloAddress1, testOtherSiloAddress2 }).ToList(); + Assert.NotEmpty(result); + Assert.Equal(2, result.Count); + } + +} \ No newline at end of file diff --git a/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs b/test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs similarity index 90% rename from test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs rename to test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs index 15fcce09a1..c19aff7d82 100644 --- a/test/TesterInternal/General/SiloMetadataPlacementFilterTests.cs +++ b/test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs @@ -5,7 +5,7 @@ using TestExtensions; using Xunit; -namespace UnitTests.General; +namespace UnitTests.PlacementFilterTests; [TestCategory("Placement"), TestCategory("Filters"), TestCategory("SiloMetadata")] public class SiloMetadataPlacementFilterTests : TestClusterPerTest @@ -32,8 +32,8 @@ public void Configure(ISiloBuilder hostBuilder) [Fact, TestCategory("Functional")] public async Task PlacementFilter_GrainWithoutFilterCanBeCalled() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); - var managementGrain = this.Client.GetGrain(0); + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var managementGrain = Client.GetGrain(0); var silos = await managementGrain.GetHosts(true); Assert.NotNull(silos); } @@ -45,11 +45,11 @@ public async Task PlacementFilter_GrainWithoutFilterCanBeCalled() [Fact, TestCategory("Functional")] public async Task PlacementFilter_RequiredFilterCanBeCalled() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); - int id = 0; + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var id = 0; foreach (var hostedClusterSilo in HostedCluster.Silos) { - for (int i = 0; i < 50; i++) + for (var i = 0; i < 50; i++) { ++id; var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); @@ -69,11 +69,11 @@ public async Task PlacementFilter_RequiredFilterCanBeCalled() [Fact, TestCategory("Functional")] public async Task PlacementFilter_PreferredFilterCanBeCalled() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); - int id = 0; + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var id = 0; foreach (var hostedClusterSilo in HostedCluster.Silos) { - for (int i = 0; i < 50; i++) + for (var i = 0; i < 50; i++) { ++id; var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); @@ -93,8 +93,8 @@ public async Task PlacementFilter_PreferredFilterCanBeCalled() [Fact, TestCategory("Functional")] public async Task PlacementFilter_PreferredMin2FilterCanBeCalled() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); - int id = 0; + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var id = 0; foreach (var hostedClusterSilo in HostedCluster.Silos) { var dict = new Dictionary(); @@ -102,7 +102,7 @@ public async Task PlacementFilter_PreferredMin2FilterCanBeCalled() { dict[clusterSilo.SiloAddress] = 0; } - for (int i = 0; i < 50; i++) + for (var i = 0; i < 50; i++) { ++id; var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); @@ -127,10 +127,10 @@ public async Task PlacementFilter_PreferredMin2FilterCanBeCalled() [Fact, TestCategory("Functional")] public async Task PlacementFilter_PreferredMin2FilterCanBeCalledWithLargerCluster() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.WaitForLivenessToStabilizeAsync(); await HostedCluster.StartAdditionalSiloAsync(); - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); - int id = 0; + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var id = 0; foreach (var hostedClusterSilo in HostedCluster.Silos) { var dict = new Dictionary(); @@ -138,7 +138,7 @@ public async Task PlacementFilter_PreferredMin2FilterCanBeCalledWithLargerCluste { dict[clusterSilo.SiloAddress] = 0; } - for (int i = 0; i < 50; i++) + for (var i = 0; i < 50; i++) { ++id; var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); @@ -163,10 +163,10 @@ public async Task PlacementFilter_PreferredMin2FilterCanBeCalledWithLargerCluste [Fact, TestCategory("Functional")] public async Task PlacementFilter_PreferredNoMetadataFilterCanBeCalled() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.WaitForLivenessToStabilizeAsync(); await HostedCluster.StartAdditionalSiloAsync(); - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); - int id = 0; + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var id = 0; foreach (var hostedClusterSilo in HostedCluster.Silos) { var dict = new Dictionary(); @@ -174,7 +174,7 @@ public async Task PlacementFilter_PreferredNoMetadataFilterCanBeCalled() { dict[clusterSilo.SiloAddress] = 0; } - for (int i = 0; i < 50; i++) + for (var i = 0; i < 50; i++) { ++id; var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); diff --git a/test/TesterInternal/PlacementFilterTests/TestLocalSiloDetails.cs b/test/TesterInternal/PlacementFilterTests/TestLocalSiloDetails.cs new file mode 100644 index 0000000000..849480f0e9 --- /dev/null +++ b/test/TesterInternal/PlacementFilterTests/TestLocalSiloDetails.cs @@ -0,0 +1,19 @@ +namespace UnitTests.PlacementFilterTests; + +internal class TestLocalSiloDetails : ILocalSiloDetails +{ + public TestLocalSiloDetails(string name, string clusterId, string dnsHostName, SiloAddress siloAddress, SiloAddress gatewayAddress) + { + Name = name; + ClusterId = clusterId; + DnsHostName = dnsHostName; + SiloAddress = siloAddress; + GatewayAddress = gatewayAddress; + } + + public string Name { get; } + public string ClusterId { get; } + public string DnsHostName { get; } + public SiloAddress SiloAddress { get; } + public SiloAddress GatewayAddress { get; } +} \ No newline at end of file diff --git a/test/TesterInternal/PlacementFilterTests/TestSiloMetadataCache.cs b/test/TesterInternal/PlacementFilterTests/TestSiloMetadataCache.cs new file mode 100644 index 0000000000..a6ab78dda4 --- /dev/null +++ b/test/TesterInternal/PlacementFilterTests/TestSiloMetadataCache.cs @@ -0,0 +1,15 @@ +using Orleans.Runtime.MembershipService.SiloMetadata; + +namespace UnitTests.PlacementFilterTests; + +internal class TestSiloMetadataCache : ISiloMetadataCache +{ + private readonly Dictionary _metadata; + + public TestSiloMetadataCache(Dictionary metadata) + { + _metadata = metadata; + } + + public SiloMetadata GetMetadata(SiloAddress siloAddress) => _metadata.GetValueOrDefault(siloAddress) ?? new SiloMetadata(); +} \ No newline at end of file diff --git a/test/TesterInternal/General/SiloMetadataTests.cs b/test/TesterInternal/SiloMetadataTests/SiloMetadataTests.cs similarity index 93% rename from test/TesterInternal/General/SiloMetadataTests.cs rename to test/TesterInternal/SiloMetadataTests/SiloMetadataTests.cs index fbd847c5de..30cfc94ec3 100644 --- a/test/TesterInternal/General/SiloMetadataTests.cs +++ b/test/TesterInternal/SiloMetadataTests/SiloMetadataTests.cs @@ -5,7 +5,7 @@ using TestExtensions; using Xunit; -namespace UnitTests.General; +namespace UnitTests.SiloMetadataTests; [TestCategory("SiloMetadata")] @@ -37,14 +37,14 @@ public void Configure(ISiloBuilder hostBuilder) [Fact, TestCategory("Functional")] public async Task SiloMetadata_CanBeSetAndRead() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.WaitForLivenessToStabilizeAsync(); HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(SiloConfigurator.Metadata.Select(kv => kv.Key.Split(':').Last()).ToArray()); } [Fact, TestCategory("Functional")] public async Task SiloMetadata_HasConfiguredValues() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.WaitForLivenessToStabilizeAsync(); var first = HostedCluster.Silos.First(); var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); @@ -82,14 +82,14 @@ public void Configure(ISiloBuilder hostBuilder) [Fact, TestCategory("Functional")] public async Task SiloMetadata_CanBeSetAndRead() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.WaitForLivenessToStabilizeAsync(); HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); } [Fact, TestCategory("Functional")] public async Task SiloMetadata_NewSilosHaveMetadata() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.WaitForLivenessToStabilizeAsync(); await HostedCluster.StartAdditionalSiloAsync(); HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); } @@ -97,7 +97,7 @@ public async Task SiloMetadata_NewSilosHaveMetadata() [Fact, TestCategory("Functional")] public async Task SiloMetadata_RemovedSiloHasNoMetadata() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); + await HostedCluster.WaitForLivenessToStabilizeAsync(); HostedCluster.AssertAllSiloMetadataMatchesOnAllSilos(["host.id"]); var first = HostedCluster.Silos.First(); var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); @@ -117,8 +117,8 @@ public async Task SiloMetadata_RemovedSiloHasNoMetadata() [Fact, TestCategory("Functional")] public async Task SiloMetadata_BadSiloAddressHasNoMetadata() { - await this.HostedCluster.WaitForLivenessToStabilizeAsync(); - var first = this.HostedCluster.Silos.First(); + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var first = HostedCluster.Silos.First(); var firstSp = HostedCluster.GetSiloServiceProvider(first.SiloAddress); var firstSiloMetadataCache = firstSp.GetRequiredService(); var metadata = firstSiloMetadataCache.GetMetadata(SiloAddress.Zero); diff --git a/test/TesterInternal/TesterInternal.csproj b/test/TesterInternal/TesterInternal.csproj index ceefb25f91..0adc91be07 100644 --- a/test/TesterInternal/TesterInternal.csproj +++ b/test/TesterInternal/TesterInternal.csproj @@ -1,4 +1,4 @@ - + UnitTests true From 046db6cc0f462341fcc3beed0258d498ffe894c2 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Fri, 20 Dec 2024 13:52:10 -0800 Subject: [PATCH 10/16] Testing multiple metadata keys in attribute --- .../SiloMetadataPlacementFilterTests.cs | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs b/test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs index c19aff7d82..e0fca917bb 100644 --- a/test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs +++ b/test/TesterInternal/PlacementFilterTests/SiloMetadataPlacementFilterTests.cs @@ -120,6 +120,40 @@ public async Task PlacementFilter_PreferredMin2FilterCanBeCalled() } } + /// + /// Unique silo metadata is set up to be different for each silo, so this will still place on any of the two silos since just the matching silos (just the one) is not enough to make the minimum desired candidates. + /// + /// + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_PreferredMultipleFilterCanBeCalled() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + var id = 0; + foreach (var hostedClusterSilo in HostedCluster.Silos) + { + var dict = new Dictionary(); + foreach (var clusterSilo in HostedCluster.Silos) + { + dict[clusterSilo.SiloAddress] = 0; + } + for (var i = 0; i < 50; i++) + { + ++id; + var firstSp = HostedCluster.GetSiloServiceProvider(hostedClusterSilo.SiloAddress); + var firstSiloMetadataCache = firstSp.GetRequiredService(); + var managementGrain = firstSiloMetadataCache.GetGrain(id); + var hostingSilo = await managementGrain.GetHostingSilo(); + Assert.NotNull(hostingSilo); + dict[hostingSilo] = dict.TryGetValue(hostingSilo, out var count) ? count + 1 : 1; + } + + foreach (var kv in dict) + { + Assert.True(kv.Value >= 1, $"Silo {kv.Key} did not host at least 1 grain"); + } + } + } + /// /// Unique silo metadata is set up to be different for each silo, so this will still place on any of the two silos since just the matching silos (just the one) is not enough to make the minimum desired candidates. /// @@ -232,6 +266,19 @@ public class PreferredMatchMinTwoFilteredGrain(ILocalSiloDetails localSiloDetail public Task GetHostingSilo() => Task.FromResult(localSiloDetails.SiloAddress); } +public interface IPreferredMatchMultipleFilteredGrain : IGrainWithIntegerKey +{ + Task GetHostingSilo(); +} + +#pragma warning disable ORLEANSEXP004 +[PreferredMatchSiloMetadataPlacementFilter(["unique", "other"], 2)] +#pragma warning restore ORLEANSEXP004 +public class PreferredMatchMultipleFilteredGrain(ILocalSiloDetails localSiloDetails) : Grain, IPreferredMatchMultipleFilteredGrain +{ + public Task GetHostingSilo() => Task.FromResult(localSiloDetails.SiloAddress); +} + #pragma warning disable ORLEANSEXP004 [PreferredMatchSiloMetadataPlacementFilter(["not.there"])] #pragma warning restore ORLEANSEXP004 From 1e0e08dea635d23fc4810e2d7c347cfcfbe3f433 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Tue, 7 Jan 2025 15:38:02 -0800 Subject: [PATCH 11/16] Ordering support for filters --- .../Filtering/PlacementFilterStrategy.cs | 25 ++- .../PlacementFilterStrategyResolver.cs | 10 +- ...tchSiloMetadataPlacementFilterAttribute.cs | 4 +- ...atchSiloMetadataPlacementFilterStrategy.cs | 9 +- ...tchSiloMetadataPlacementFilterAttribute.cs | 4 +- ...atchSiloMetadataPlacementFilterStrategy.cs | 8 +- .../GrainPlacementFilterTests.cs | 195 +++++++++++++++++- ...iloMetadataPlacementFilterDirectorTests.cs | 6 +- ...iloMetadataPlacementFilterDirectorTests.cs | 8 +- 9 files changed, 244 insertions(+), 25 deletions(-) diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs index 614b42ec99..28ffaf202d 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs @@ -6,14 +6,35 @@ namespace Orleans.Runtime.Placement.Filtering; public abstract class PlacementFilterStrategy { + public int Order { get; private set; } + + protected PlacementFilterStrategy(int order) + { + Order = order; + } + /// /// Initializes an instance of this type using the provided grain properties. /// /// /// The grain properties. /// - public virtual void Initialize(GrainProperties properties) + public void Initialize(GrainProperties properties) { + var orderProperty = GetPlacementFilterGrainProperty("order", properties); + if (!int.TryParse(orderProperty, out var parsedOrder)) + { + throw new ArgumentException("Invalid order property value."); + } + + Order = parsedOrder; + + AdditionalInitialize(properties); + } + + public virtual void AdditionalInitialize(GrainProperties properties) + { + } /// @@ -35,6 +56,8 @@ public void PopulateGrainProperties(IServiceProvider services, Type grainClass, properties[WellKnownGrainTypeProperties.PlacementFilter] = typeName; } + properties[$"{WellKnownGrainTypeProperties.PlacementFilter}.{typeName}.order"] = Order.ToString(); + foreach (var additionalGrainProperty in GetAdditionalGrainProperties(services, grainClass, grainType, properties)) { properties[$"{WellKnownGrainTypeProperties.PlacementFilter}.{typeName}.{additionalGrainProperty.Key}"] = additionalGrainProperty.Value; diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs index cb36817056..97c9aa3705 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Linq; using Microsoft.Extensions.DependencyInjection; using Orleans.Metadata; @@ -55,7 +56,14 @@ private PlacementFilterStrategy[] GetPlacementFilterStrategyInternal(GrainType g throw new KeyNotFoundException($"Could not resolve placement filter strategy {filterId} for grain type {grainType}. Ensure that dependencies for that filter have been configured in the Container. This is often through a .Use* extension method provided by the implementation."); } } - return filterList.ToArray(); + + var orderedFilters = filterList.OrderBy(f => f.Order).ToArray(); + // check that the order is unique + if (orderedFilters.Select(f => f.Order).Distinct().Count() != orderedFilters.Length) + { + throw new InvalidOperationException($"Placement filters for grain type {grainType} have duplicate order values. Order values must be specified if more than one filter is applied and must be unique."); + } + return orderedFilters; } return []; diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs index f80955e855..8d5afcfce2 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs @@ -11,5 +11,5 @@ namespace Orleans.Runtime.Placement.Filtering; /// Example: If keys ["first","second"] are specified, then it will attempt to return only silos where both keys match the local silo's metadata values. If there are not sufficient silos matching both, then it will also include silos matching only the second key. Finally, if there are still fewer than minCandidates results then it will include all silos. [AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] [Experimental("ORLEANSEXP004")] -public class PreferredMatchSiloMetadataPlacementFilterAttribute(string[] orderedMetadataKeys, int minCandidates = 2) - : PlacementFilterAttribute(new PreferredMatchSiloMetadataPlacementFilterStrategy(orderedMetadataKeys, minCandidates)); \ No newline at end of file +public class PreferredMatchSiloMetadataPlacementFilterAttribute(string[] orderedMetadataKeys, int minCandidates = 2, int order = 0) + : PlacementFilterAttribute(new PreferredMatchSiloMetadataPlacementFilterStrategy(orderedMetadataKeys, minCandidates, order)); \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs index 74c551b5c8..772076c7c9 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs @@ -4,19 +4,18 @@ namespace Orleans.Runtime.Placement.Filtering; -public class PreferredMatchSiloMetadataPlacementFilterStrategy(string[] orderedMetadataKeys, int minCandidates) - : PlacementFilterStrategy +public class PreferredMatchSiloMetadataPlacementFilterStrategy(string[] orderedMetadataKeys, int minCandidates, int order) + : PlacementFilterStrategy(order) { public string[] OrderedMetadataKeys { get; set; } = orderedMetadataKeys; public int MinCandidates { get; set; } = minCandidates; - public PreferredMatchSiloMetadataPlacementFilterStrategy() : this([], 1) + public PreferredMatchSiloMetadataPlacementFilterStrategy() : this([], 1, 0) { } - public override void Initialize(GrainProperties properties) + public override void AdditionalInitialize(GrainProperties properties) { - base.Initialize(properties); OrderedMetadataKeys = GetPlacementFilterGrainProperty("ordered-metadata-keys", properties).Split(","); var minCandidatesProperty = GetPlacementFilterGrainProperty("min-candidates", properties); if (!int.TryParse(minCandidatesProperty, out var parsedMinCandidates)) diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs index 2bdda2645f..8db4f0f675 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs @@ -9,5 +9,5 @@ namespace Orleans.Runtime.Placement.Filtering; /// [AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] [Experimental("ORLEANSEXP004")] -public class RequiredMatchSiloMetadataPlacementFilterAttribute(string[] metadataKeys) - : PlacementFilterAttribute(new RequiredMatchSiloMetadataPlacementFilterStrategy(metadataKeys)); \ No newline at end of file +public class RequiredMatchSiloMetadataPlacementFilterAttribute(string[] metadataKeys, int order = 0) + : PlacementFilterAttribute(new RequiredMatchSiloMetadataPlacementFilterStrategy(metadataKeys, order)); \ No newline at end of file diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs index fc0ca730fe..db3a8fac73 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs @@ -4,17 +4,17 @@ namespace Orleans.Runtime.Placement.Filtering; -public class RequiredMatchSiloMetadataPlacementFilterStrategy(string[] metadataKeys) : PlacementFilterStrategy +public class RequiredMatchSiloMetadataPlacementFilterStrategy(string[] metadataKeys, int order) + : PlacementFilterStrategy(order) { public string[] MetadataKeys { get; private set; } = metadataKeys; - public RequiredMatchSiloMetadataPlacementFilterStrategy() : this([]) + public RequiredMatchSiloMetadataPlacementFilterStrategy() : this([], 0) { } - public override void Initialize(GrainProperties properties) + public override void AdditionalInitialize(GrainProperties properties) { - base.Initialize(properties); MetadataKeys = GetPlacementFilterGrainProperty("metadata-keys", properties).Split(","); } diff --git a/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs b/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs index 9f34e11e9b..c38730c0d0 100644 --- a/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs +++ b/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Orleans.Runtime.Placement; using Orleans.Runtime.Placement.Filtering; @@ -10,6 +11,9 @@ namespace UnitTests.PlacementFilterTests; [TestCategory("Placement"), TestCategory("Filters")] public class GrainPlacementFilterTests : TestClusterPerTest { + public static Dictionary> FilterScratchpad = new(); + private static Random random = new(); + protected override void ConfigureTestCluster(TestClusterBuilder builder) { builder.AddSiloBuilderConfigurator(); @@ -22,10 +26,13 @@ public void Configure(ISiloBuilder hostBuilder) hostBuilder.ConfigureServices(services => { services.AddPlacementFilter(ServiceLifetime.Singleton); + services.AddPlacementFilter(ServiceLifetime.Singleton); + services.AddPlacementFilter(ServiceLifetime.Singleton); }); } } + [Fact, TestCategory("Functional")] public async Task PlacementFilter_GrainWithoutFilterCanBeCalled() { @@ -49,9 +56,79 @@ public async Task PlacementFilter_FilterIsTriggered() await task; Assert.True(triggered); } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_OrderAB12() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + + var primaryKey = random.Next(); + var testGrain = Client.GetGrain(primaryKey); + await testGrain.Ping(); + var list = FilterScratchpad.GetValueOrAddNew(testGrain.GetGrainId().ToString()); + Assert.Equal(2, list.Count); + Assert.Equal("A", list[0]); + Assert.Equal("B", list[1]); + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_OrderAB21() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + + var primaryKey = random.Next(); + var testGrain = Client.GetGrain(primaryKey); + await testGrain.Ping(); + var list = FilterScratchpad.GetValueOrAddNew(testGrain.GetGrainId().ToString()); + Assert.Equal(2, list.Count); + Assert.Equal("B", list[0]); + Assert.Equal("A", list[1]); + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_OrderBA12() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + + var primaryKey = random.Next(); + var testGrain = Client.GetGrain(primaryKey); + await testGrain.Ping(); + var list = FilterScratchpad.GetValueOrAddNew(testGrain.GetGrainId().ToString()); + Assert.Equal(2, list.Count); + Assert.Equal("B", list[0]); + Assert.Equal("A", list[1]); + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_OrderBA21() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + + var primaryKey = random.Next(); + var testGrain = Client.GetGrain(primaryKey); + await testGrain.Ping(); + + var list = FilterScratchpad.GetValueOrAddNew(testGrain.GetGrainId().ToString()); + Assert.Equal(2, list.Count); + Assert.Equal("A", list[0]); + Assert.Equal("B", list[1]); + } + + [Fact, TestCategory("Functional")] + public async Task PlacementFilter_DuplicateOrder() + { + await HostedCluster.WaitForLivenessToStabilizeAsync(); + + var primaryKey = random.Next(); + var testGrain = Client.GetGrain(primaryKey); + await Assert.ThrowsAsync(async () => + { + await testGrain.Ping(); + }); + } } -[TestPlacementFilter] +[TestPlacementFilter(order: 1)] public class TestFilteredGrain : Grain, ITestFilteredGrain { public Task Ping() => Task.CompletedTask; @@ -62,9 +139,14 @@ public interface ITestFilteredGrain : IGrainWithIntegerKey Task Ping(); } -public class TestPlacementFilterAttribute() : PlacementFilterAttribute(new TestPlacementFilterStrategy()); +public class TestPlacementFilterAttribute(int order) : PlacementFilterAttribute(new TestPlacementFilterStrategy(order)); -public class TestPlacementFilterStrategy : PlacementFilterStrategy; +public class TestPlacementFilterStrategy(int order) : PlacementFilterStrategy(order) +{ + public TestPlacementFilterStrategy() : this(0) + { + } +} public class TestPlacementFilterDirector() : IPlacementFilterDirector { @@ -76,3 +158,110 @@ public IEnumerable Filter(PlacementFilterStrategy filterStrategy, P return silos; } } + + + +public class OrderAPlacementFilterAttribute(int order) : PlacementFilterAttribute(new OrderAPlacementFilterStrategy(order)); + +public class OrderAPlacementFilterStrategy(int order) : PlacementFilterStrategy(order) +{ + public OrderAPlacementFilterStrategy() : this(0) + { + } +} + +public class OrderAPlacementFilterDirector : IPlacementFilterDirector +{ + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + { + var dict = GrainPlacementFilterTests.FilterScratchpad; + var list = dict.GetValueOrAddNew(target.GrainIdentity.ToString()); + list.Add("A"); + return silos; + } +} + + +public class OrderBPlacementFilterAttribute(int order) : PlacementFilterAttribute(new OrderBPlacementFilterStrategy(order)); + +public class OrderBPlacementFilterStrategy(int order) : PlacementFilterStrategy(order) +{ + + public OrderBPlacementFilterStrategy() : this(0) + { + } +} + +public class OrderBPlacementFilterDirector() : IPlacementFilterDirector +{ + public IEnumerable Filter(PlacementFilterStrategy filterStrategy, PlacementTarget target, IEnumerable silos) + { + var dict = GrainPlacementFilterTests.FilterScratchpad; + var list = dict.GetValueOrAddNew(target.GrainIdentity.ToString()); + list.Add("B"); + return silos; + } +} + +[OrderAPlacementFilter(order: 1)] +[OrderBPlacementFilter(order: 2)] +public class TestAB12FilteredGrain : Grain, ITestAB12FilteredGrain +{ + public Task Ping() => Task.CompletedTask; +} + +public interface ITestAB12FilteredGrain : IGrainWithIntegerKey +{ + Task Ping(); +} + +[OrderAPlacementFilter(order: 2)] +[OrderBPlacementFilter(order: 1)] +public class TestAB21FilteredGrain : Grain, ITestAB21FilteredGrain +{ + public Task Ping() => Task.CompletedTask; +} + +public interface ITestAB21FilteredGrain : IGrainWithIntegerKey +{ + Task Ping(); +} + +[OrderBPlacementFilter(order: 1)] +[OrderAPlacementFilter(order: 2)] +public class TestBA12FilteredGrain : Grain, ITestBA12FilteredGrain +{ + public Task Ping() => Task.CompletedTask; +} + +public interface ITestBA12FilteredGrain : IGrainWithIntegerKey +{ + Task Ping(); +} + + +[OrderBPlacementFilter(order: 2)] +[OrderAPlacementFilter(order: 1)] +public class TestBA121FilteredGrain : Grain, ITestBA21FilteredGrain +{ + public Task Ping() => Task.CompletedTask; +} + +public interface ITestBA21FilteredGrain : IGrainWithIntegerKey +{ + Task Ping(); +} + + + +[OrderBPlacementFilter(order: 2)] +[OrderAPlacementFilter(order: 2)] +public class TestDuplicateOrderFilteredGrain : Grain, ITestDuplicateOrderFilteredGrain +{ + public Task Ping() => Task.CompletedTask; +} + +public interface ITestDuplicateOrderFilteredGrain : IGrainWithIntegerKey +{ + Task Ping(); +} \ No newline at end of file diff --git a/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs b/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs index 9274d75f96..369829fb7b 100644 --- a/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs +++ b/test/TesterInternal/PlacementFilterTests/PreferredMatchSiloMetadataPlacementFilterDirectorTests.cs @@ -60,7 +60,7 @@ public void FiltersToAllWhenNoEntry() {testOtherSiloAddress, SiloMetadata.Empty}, {testLocalSiloAddress, siloMetadata}, })); - var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 1), new PlacementTarget(), + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 1, 0), new PlacementTarget(), new List() { testOtherSiloAddress }).ToList(); Assert.NotEmpty(result); } @@ -114,7 +114,7 @@ public void FiltersOnSingleMetadata(int minCandidates, int expectedCount, string {testOtherSiloAddress3, otherSiloMetadata3}, {testLocalSiloAddress, localSiloMetadata}, })); - var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy([key], minCandidates), new PlacementTarget(), + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy([key], minCandidates, 0), new PlacementTarget(), new List() { testOtherSiloAddress1, testOtherSiloAddress2, testOtherSiloAddress3 }).ToList(); Assert.NotEmpty(result); Assert.Equal(expectedCount, result.Count); @@ -168,7 +168,7 @@ public void FiltersOnMultipleMetadata(int minCandidates, int expectedCount, para {testOtherSiloAddress3, otherSiloMetadata3}, {testLocalSiloAddress, localSiloMetadata}, })); - var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(keys, minCandidates), new PlacementTarget(), + var result = director.Filter(new PreferredMatchSiloMetadataPlacementFilterStrategy(keys, minCandidates, 0), new PlacementTarget(), new List() { testOtherSiloAddress1, testOtherSiloAddress2, testOtherSiloAddress3 }).ToList(); Assert.NotEmpty(result); Assert.Equal(expectedCount, result.Count); diff --git a/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs b/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs index f63afed282..19d03626c9 100644 --- a/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs +++ b/test/TesterInternal/PlacementFilterTests/RequiredMatchSiloMetadataPlacementFilterDirectorTests.cs @@ -59,7 +59,7 @@ public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToNothingWhe {testOtherSiloAddress, SiloMetadata.Empty}, {testLocalSiloAddress, siloMetadata}, })); - var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"]), new PlacementTarget(), + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 0), new PlacementTarget(), new List() { testOtherSiloAddress }).ToList(); Assert.Empty(result); } @@ -82,7 +82,7 @@ public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToNothingWhe {testOtherSiloAddress, otherSiloMetadata}, {testLocalSiloAddress, localSiloMetadata}, })); - var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"]), new PlacementTarget(), + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 0), new PlacementTarget(), new List() { testOtherSiloAddress }).ToList(); Assert.Empty(result); } @@ -105,7 +105,7 @@ public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToSiloWhenMa {testOtherSiloAddress, otherSiloMetadata}, {testLocalSiloAddress, localSiloMetadata}, })); - var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"]), new PlacementTarget(), + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 0), new PlacementTarget(), new List() { testOtherSiloAddress }).ToList(); Assert.NotEmpty(result); } @@ -132,7 +132,7 @@ public void RequiredMatchSiloMetadataPlacementFilterDirector_FiltersToMultipleSi {testOtherSiloAddress2, otherSiloMetadata2}, {testLocalSiloAddress, localSiloMetadata}, })); - var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"]), new PlacementTarget(), + var result = director.Filter(new RequiredMatchSiloMetadataPlacementFilterStrategy(["metadata.key"], 0), new PlacementTarget(), new List() { testOtherSiloAddress1, testOtherSiloAddress2 }).ToList(); Assert.NotEmpty(result); Assert.Equal(2, result.Count); From 2a40b034c9f104fc5ed865c50db0972a32a23ce5 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Wed, 8 Jan 2025 08:44:52 -0800 Subject: [PATCH 12/16] Moving reusable types to Core projects Orleans.Core.Abstractions: - PlacementFilterAttribute - PlacementFilterStrategy Orelans.Core: - IPlacementFilterDirector - PlacementFilterExtensions --- .../Placement}/PlacementFilterAttribute.cs | 3 ++- .../Placement}/PlacementFilterStrategy.cs | 3 ++- .../Placement}/IPlacementFilterDirector.cs | 4 +++- .../Placement}/PlacementFilterExtensions.cs | 2 +- .../SiloMetadata/SiloMetadataHostingExtensions.cs | 1 + .../Placement/Filtering/PlacementFilterDirectorResolver.cs | 1 + .../Placement/Filtering/PlacementFilterStrategyResolver.cs | 1 + .../PreferredMatchSiloMetadataPlacementFilterAttribute.cs | 1 + .../PreferredMatchSiloMetadataPlacementFilterDirector.cs | 1 + .../PreferredMatchSiloMetadataPlacementFilterStrategy.cs | 1 + .../RequiredMatchSiloMetadataPlacementFilterAttribute.cs | 1 + .../RequiredMatchSiloMetadataPlacementFilterDirector.cs | 1 + .../RequiredMatchSiloMetadataPlacementFilterStrategy.cs | 1 + .../PlacementFilterTests/GrainPlacementFilterTests.cs | 3 +-- 14 files changed, 18 insertions(+), 6 deletions(-) rename src/{Orleans.Runtime/Placement/Filtering => Orleans.Core.Abstractions/Placement}/PlacementFilterAttribute.cs (94%) rename src/{Orleans.Runtime/Placement/Filtering => Orleans.Core.Abstractions/Placement}/PlacementFilterStrategy.cs (98%) rename src/{Orleans.Runtime/Placement/Filtering => Orleans.Core/Placement}/IPlacementFilterDirector.cs (72%) rename src/{Orleans.Runtime/Placement/Filtering => Orleans.Core/Placement}/PlacementFilterExtensions.cs (95%) diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterAttribute.cs b/src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs similarity index 94% rename from src/Orleans.Runtime/Placement/Filtering/PlacementFilterAttribute.cs rename to src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs index b67e23fd10..9cb03476db 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterAttribute.cs +++ b/src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs @@ -1,8 +1,9 @@ using System; using System.Collections.Generic; using Orleans.Metadata; +using Orleans.Runtime; -namespace Orleans.Runtime.Placement.Filtering; +namespace Orleans.Placement; /// /// Base for all placement filter marker attributes. diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs b/src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs similarity index 98% rename from src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs rename to src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs index 28ffaf202d..1ec787803b 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategy.cs +++ b/src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs @@ -1,8 +1,9 @@ using System; using System.Collections.Generic; using Orleans.Metadata; +using Orleans.Runtime; -namespace Orleans.Runtime.Placement.Filtering; +namespace Orleans.Placement; public abstract class PlacementFilterStrategy { diff --git a/src/Orleans.Runtime/Placement/Filtering/IPlacementFilterDirector.cs b/src/Orleans.Core/Placement/IPlacementFilterDirector.cs similarity index 72% rename from src/Orleans.Runtime/Placement/Filtering/IPlacementFilterDirector.cs rename to src/Orleans.Core/Placement/IPlacementFilterDirector.cs index 23b2437771..18c9486ed5 100644 --- a/src/Orleans.Runtime/Placement/Filtering/IPlacementFilterDirector.cs +++ b/src/Orleans.Core/Placement/IPlacementFilterDirector.cs @@ -1,6 +1,8 @@ using System.Collections.Generic; +using Orleans.Runtime; +using Orleans.Runtime.Placement; -namespace Orleans.Runtime.Placement.Filtering; +namespace Orleans.Placement; public interface IPlacementFilterDirector { diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterExtensions.cs b/src/Orleans.Core/Placement/PlacementFilterExtensions.cs similarity index 95% rename from src/Orleans.Runtime/Placement/Filtering/PlacementFilterExtensions.cs rename to src/Orleans.Core/Placement/PlacementFilterExtensions.cs index faf9367f27..05805ed15f 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterExtensions.cs +++ b/src/Orleans.Core/Placement/PlacementFilterExtensions.cs @@ -1,6 +1,6 @@ using Microsoft.Extensions.DependencyInjection; -namespace Orleans.Runtime.Placement.Filtering; +namespace Orleans.Placement; public static class PlacementFilterExtensions { diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs index 200577e263..cb3c8f5d3b 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs @@ -3,6 +3,7 @@ using Microsoft.Extensions.DependencyInjection; using Orleans.Configuration.Internal; using Orleans.Hosting; +using Orleans.Placement; using Orleans.Runtime.Placement.Filtering; namespace Orleans.Runtime.MembershipService.SiloMetadata; diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs index 8401f5eac7..167daff151 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs @@ -1,5 +1,6 @@ using System; using Microsoft.Extensions.DependencyInjection; +using Orleans.Placement; namespace Orleans.Runtime.Placement.Filtering; diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs index 97c9aa3705..0c462b1fdd 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs @@ -4,6 +4,7 @@ using System.Linq; using Microsoft.Extensions.DependencyInjection; using Orleans.Metadata; +using Orleans.Placement; namespace Orleans.Runtime.Placement.Filtering; diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs index 8d5afcfce2..b8c6ca356e 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs @@ -1,5 +1,6 @@ using System; using System.Diagnostics.CodeAnalysis; +using Orleans.Placement; namespace Orleans.Runtime.Placement.Filtering; diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs index b847ffbf08..24a9ba7c1e 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Orleans.Placement; using Orleans.Runtime.MembershipService.SiloMetadata; #nullable enable namespace Orleans.Runtime.Placement.Filtering; diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs index 772076c7c9..acef5ecb16 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using Orleans.Metadata; +using Orleans.Placement; namespace Orleans.Runtime.Placement.Filtering; diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs index 8db4f0f675..3c929dd297 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs @@ -1,5 +1,6 @@ using System; using System.Diagnostics.CodeAnalysis; +using Orleans.Placement; namespace Orleans.Runtime.Placement.Filtering; diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs index ffbc4782f6..df3e579fbc 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using System.Linq; +using Orleans.Placement; using Orleans.Runtime.MembershipService.SiloMetadata; namespace Orleans.Runtime.Placement.Filtering; diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs index db3a8fac73..413f218af9 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using Orleans.Metadata; +using Orleans.Placement; namespace Orleans.Runtime.Placement.Filtering; diff --git a/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs b/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs index c38730c0d0..ce80d6ae5b 100644 --- a/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs +++ b/test/TesterInternal/PlacementFilterTests/GrainPlacementFilterTests.cs @@ -1,7 +1,6 @@ -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Orleans.Placement; using Orleans.Runtime.Placement; -using Orleans.Runtime.Placement.Filtering; using Orleans.TestingHost; using TestExtensions; using Xunit; From 8a98696a5a0b824098f5bbfd2ce73ba5e957b671 Mon Sep 17 00:00:00 2001 From: rkargMsft <164392675+rkargMsft@users.noreply.github.com> Date: Wed, 8 Jan 2025 13:55:22 -0800 Subject: [PATCH 13/16] Correct comment Co-authored-by: dmorganMsft --- .../MembershipService/SiloMetadata/SiloMetadaCache.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs index d7bdf011ed..bc594fb928 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs @@ -68,7 +68,7 @@ private async Task ProcessMembershipUpdates(CancellationToken ct) } } - // Add entries for members that aren't already in the cache + // Remove entries for members that are now dead foreach (var membershipEntry in update.Entries.Where(e => e.Value.Status == SiloStatus.Dead)) { _metadata.TryRemove(membershipEntry.Key, out _); From e530f3205761c1c4dcb7b6df226874c29eb7d872 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Wed, 8 Jan 2025 13:57:13 -0800 Subject: [PATCH 14/16] Correcting documentation --- .../SiloMetadata/SiloMetadataHostingExtensions.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs index cb3c8f5d3b..02a0bb8cb9 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs @@ -29,9 +29,9 @@ public static class SiloMetadataHostingExtensions /// Silo builder /// Configuration to pull from /// - /// Get the Orleans:Metadata section from config + /// Get the ORLEANS__METADATA section from config /// Key/value pairs in configuration as a will look like this as environment variables: - /// Orleans:Metadata:key1=value1 + /// ORLEANS__METADATA__key1=value1 /// /// public static ISiloBuilder UseSiloMetadata(this ISiloBuilder builder, IConfiguration configuration) From 3d7bfae62ef12e2f0a941335aa1be09d362b2c96 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Wed, 8 Jan 2025 13:59:52 -0800 Subject: [PATCH 15/16] Allowing chaining extension method --- src/Orleans.Core/Placement/PlacementFilterExtensions.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Orleans.Core/Placement/PlacementFilterExtensions.cs b/src/Orleans.Core/Placement/PlacementFilterExtensions.cs index 05805ed15f..087d81fbb0 100644 --- a/src/Orleans.Core/Placement/PlacementFilterExtensions.cs +++ b/src/Orleans.Core/Placement/PlacementFilterExtensions.cs @@ -12,12 +12,14 @@ public static class PlacementFilterExtensions /// The service collection. /// The lifetime of the placement strategy. /// The service collection. - public static void AddPlacementFilter(this IServiceCollection services, ServiceLifetime strategyLifetime) + public static IServiceCollection AddPlacementFilter(this IServiceCollection services, ServiceLifetime strategyLifetime) where TFilter : PlacementFilterStrategy where TDirector : class, IPlacementFilterDirector { services.Add(ServiceDescriptor.DescribeKeyed(typeof(PlacementFilterStrategy), typeof(TFilter).Name, typeof(TFilter), strategyLifetime)); services.AddKeyedSingleton(typeof(TFilter)); + + return services; } } \ No newline at end of file From 4c4bce18aca7764c61b57ea0c51a40971e130a75 Mon Sep 17 00:00:00 2001 From: Ryan Karg Date: Wed, 8 Jan 2025 14:54:13 -0800 Subject: [PATCH 16/16] #nullable enable for new files --- .../Placement/PlacementFilterAttribute.cs | 1 + .../Placement/PlacementFilterStrategy.cs | 3 ++- src/Orleans.Core/Placement/IPlacementFilterDirector.cs | 1 + .../Placement/PlacementFilterExtensions.cs | 1 + .../SiloMetadata/ISiloMetadataCache.cs | 3 ++- .../SiloMetadata/ISiloMetadataClient.cs | 1 + .../SiloMetadata/ISiloMetadataGrainService.cs | 1 + .../MembershipService/SiloMetadata/SiloMetadaCache.cs | 3 ++- .../MembershipService/SiloMetadata/SiloMetadata.cs | 1 + .../SiloMetadata/SiloMetadataClient.cs | 1 + .../SiloMetadata/SiloMetadataGrainService.cs | 1 + .../SiloMetadata/SiloMetadataHostingExtensions.cs | 1 + .../Filtering/PlacementFilterDirectorResolver.cs | 1 + .../Filtering/PlacementFilterStrategyResolver.cs | 1 + ...eferredMatchSiloMetadataPlacementFilterAttribute.cs | 1 + ...referredMatchSiloMetadataPlacementFilterDirector.cs | 1 + ...referredMatchSiloMetadataPlacementFilterStrategy.cs | 10 ++++++++-- ...equiredMatchSiloMetadataPlacementFilterAttribute.cs | 1 + ...RequiredMatchSiloMetadataPlacementFilterDirector.cs | 7 ++++--- ...RequiredMatchSiloMetadataPlacementFilterStrategy.cs | 8 +++++++- 20 files changed, 39 insertions(+), 9 deletions(-) diff --git a/src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs b/src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs index 9cb03476db..0203d6439b 100644 --- a/src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs +++ b/src/Orleans.Core.Abstractions/Placement/PlacementFilterAttribute.cs @@ -3,6 +3,7 @@ using Orleans.Metadata; using Orleans.Runtime; +#nullable enable namespace Orleans.Placement; /// diff --git a/src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs b/src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs index 1ec787803b..eede6af61b 100644 --- a/src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs +++ b/src/Orleans.Core.Abstractions/Placement/PlacementFilterStrategy.cs @@ -3,6 +3,7 @@ using Orleans.Metadata; using Orleans.Runtime; +#nullable enable namespace Orleans.Placement; public abstract class PlacementFilterStrategy @@ -65,7 +66,7 @@ public void PopulateGrainProperties(IServiceProvider services, Type grainClass, } } - protected string GetPlacementFilterGrainProperty(string key, GrainProperties properties) + protected string? GetPlacementFilterGrainProperty(string key, GrainProperties properties) { var typeName = GetType().Name; return properties.Properties.TryGetValue($"{WellKnownGrainTypeProperties.PlacementFilter}.{typeName}.{key}", out var value) ? value : null; diff --git a/src/Orleans.Core/Placement/IPlacementFilterDirector.cs b/src/Orleans.Core/Placement/IPlacementFilterDirector.cs index 18c9486ed5..fe9a225a0e 100644 --- a/src/Orleans.Core/Placement/IPlacementFilterDirector.cs +++ b/src/Orleans.Core/Placement/IPlacementFilterDirector.cs @@ -2,6 +2,7 @@ using Orleans.Runtime; using Orleans.Runtime.Placement; +#nullable enable namespace Orleans.Placement; public interface IPlacementFilterDirector diff --git a/src/Orleans.Core/Placement/PlacementFilterExtensions.cs b/src/Orleans.Core/Placement/PlacementFilterExtensions.cs index 087d81fbb0..7592656607 100644 --- a/src/Orleans.Core/Placement/PlacementFilterExtensions.cs +++ b/src/Orleans.Core/Placement/PlacementFilterExtensions.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.DependencyInjection; +#nullable enable namespace Orleans.Placement; public static class PlacementFilterExtensions diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs index a15cb67a36..96f5d5bbfb 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataCache.cs @@ -1,5 +1,6 @@ -namespace Orleans.Runtime.MembershipService.SiloMetadata; #nullable enable +namespace Orleans.Runtime.MembershipService.SiloMetadata; + public interface ISiloMetadataCache { SiloMetadata GetMetadata(SiloAddress siloAddress); diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs index eb622eaaa4..e2ab060bea 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataClient.cs @@ -1,6 +1,7 @@ using System.Threading.Tasks; using Orleans.Services; +#nullable enable namespace Orleans.Runtime.MembershipService.SiloMetadata; public interface ISiloMetadataClient : IGrainServiceClient diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs index 97e3c7e600..06121ae40c 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/ISiloMetadataGrainService.cs @@ -1,6 +1,7 @@ using System.Threading.Tasks; using Orleans.Services; +#nullable enable namespace Orleans.Runtime.MembershipService.SiloMetadata; [Alias("Orleans.Runtime.MembershipService.SiloMetadata.ISiloMetadataGrainService")] diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs index bc594fb928..8483c570a1 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadaCache.cs @@ -8,8 +8,9 @@ using Orleans.Configuration; using Orleans.Internal; -namespace Orleans.Runtime.MembershipService.SiloMetadata; #nullable enable +namespace Orleans.Runtime.MembershipService.SiloMetadata; + internal class SiloMetadataCache( ISiloMetadataClient siloMetadataClient, MembershipTableManager membershipTableManager, diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs index 395aa9d3df..278a7aa76a 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadata.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.Collections.Immutable; +#nullable enable namespace Orleans.Runtime.MembershipService.SiloMetadata; [GenerateSerializer] diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs index 377bf30288..5d507bde72 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataClient.cs @@ -2,6 +2,7 @@ using System.Threading.Tasks; using Orleans.Runtime.Services; +#nullable enable namespace Orleans.Runtime.MembershipService.SiloMetadata; public class SiloMetadataClient(IServiceProvider serviceProvider) diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs index 336b527458..8e60bff08d 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataGrainService.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +#nullable enable namespace Orleans.Runtime.MembershipService.SiloMetadata; public class SiloMetadataGrainService : GrainService, ISiloMetadataGrainService diff --git a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs index 02a0bb8cb9..a2adf33f55 100644 --- a/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs +++ b/src/Orleans.Runtime/MembershipService/SiloMetadata/SiloMetadataHostingExtensions.cs @@ -6,6 +6,7 @@ using Orleans.Placement; using Orleans.Runtime.Placement.Filtering; +#nullable enable namespace Orleans.Runtime.MembershipService.SiloMetadata; public static class SiloMetadataHostingExtensions diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs index 167daff151..f6ef6cba0a 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterDirectorResolver.cs @@ -2,6 +2,7 @@ using Microsoft.Extensions.DependencyInjection; using Orleans.Placement; +#nullable enable namespace Orleans.Runtime.Placement.Filtering; /// diff --git a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs index 0c462b1fdd..41524681cf 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PlacementFilterStrategyResolver.cs @@ -6,6 +6,7 @@ using Orleans.Metadata; using Orleans.Placement; +#nullable enable namespace Orleans.Runtime.Placement.Filtering; /// diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs index b8c6ca356e..bde94878c9 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterAttribute.cs @@ -2,6 +2,7 @@ using System.Diagnostics.CodeAnalysis; using Orleans.Placement; +#nullable enable namespace Orleans.Runtime.Placement.Filtering; /// diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs index 24a9ba7c1e..7b92a41c0e 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterDirector.cs @@ -3,6 +3,7 @@ using System.Linq; using Orleans.Placement; using Orleans.Runtime.MembershipService.SiloMetadata; + #nullable enable namespace Orleans.Runtime.Placement.Filtering; diff --git a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs index acef5ecb16..e6a160adef 100644 --- a/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs +++ b/src/Orleans.Runtime/Placement/Filtering/PreferredMatchSiloMetadataPlacementFilterStrategy.cs @@ -3,6 +3,7 @@ using Orleans.Metadata; using Orleans.Placement; +#nullable enable namespace Orleans.Runtime.Placement.Filtering; public class PreferredMatchSiloMetadataPlacementFilterStrategy(string[] orderedMetadataKeys, int minCandidates, int order) @@ -11,13 +12,18 @@ public class PreferredMatchSiloMetadataPlacementFilterStrategy(string[] orderedM public string[] OrderedMetadataKeys { get; set; } = orderedMetadataKeys; public int MinCandidates { get; set; } = minCandidates; - public PreferredMatchSiloMetadataPlacementFilterStrategy() : this([], 1, 0) + public PreferredMatchSiloMetadataPlacementFilterStrategy() : this([], 2, 0) { } public override void AdditionalInitialize(GrainProperties properties) { - OrderedMetadataKeys = GetPlacementFilterGrainProperty("ordered-metadata-keys", properties).Split(","); + var placementFilterGrainProperty = GetPlacementFilterGrainProperty("ordered-metadata-keys", properties); + if (placementFilterGrainProperty is null) + { + throw new ArgumentException("Invalid ordered-metadata-keys property value."); + } + OrderedMetadataKeys = placementFilterGrainProperty.Split(","); var minCandidatesProperty = GetPlacementFilterGrainProperty("min-candidates", properties); if (!int.TryParse(minCandidatesProperty, out var parsedMinCandidates)) { diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs index 3c929dd297..ec317ad2c6 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterAttribute.cs @@ -2,6 +2,7 @@ using System.Diagnostics.CodeAnalysis; using Orleans.Placement; +#nullable enable namespace Orleans.Runtime.Placement.Filtering; /// diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs index df3e579fbc..91ed8cbac7 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterDirector.cs @@ -3,6 +3,7 @@ using Orleans.Placement; using Orleans.Runtime.MembershipService.SiloMetadata; +#nullable enable namespace Orleans.Runtime.Placement.Filtering; internal class RequiredMatchSiloMetadataPlacementFilterDirector(ILocalSiloDetails localSiloDetails, ISiloMetadataCache siloMetadataCache) @@ -28,7 +29,7 @@ public IEnumerable Filter(PlacementFilterStrategy filterStrategy, P }); } - private static bool DoesMetadataMatch(string[] localMetadata, SiloMetadata siloMetadata, string[] metadataKeys) + private static bool DoesMetadataMatch(string?[] localMetadata, SiloMetadata siloMetadata, string[] metadataKeys) { for (var i = 0; i < metadataKeys.Length; i++) { @@ -40,9 +41,9 @@ private static bool DoesMetadataMatch(string[] localMetadata, SiloMetadata siloM return true; } - private static string[] GetMetadata(SiloMetadata siloMetadata, string[] metadataKeys) + private static string?[] GetMetadata(SiloMetadata siloMetadata, string[] metadataKeys) { - var result = new string[metadataKeys.Length]; + var result = new string?[metadataKeys.Length]; for (var i = 0; i < metadataKeys.Length; i++) { result[i] = siloMetadata.Metadata?.GetValueOrDefault(metadataKeys[i]); diff --git a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs index 413f218af9..c2c19a995b 100644 --- a/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs +++ b/src/Orleans.Runtime/Placement/Filtering/RequiredMatchSiloMetadataPlacementFilterStrategy.cs @@ -3,6 +3,7 @@ using Orleans.Metadata; using Orleans.Placement; +#nullable enable namespace Orleans.Runtime.Placement.Filtering; public class RequiredMatchSiloMetadataPlacementFilterStrategy(string[] metadataKeys, int order) @@ -16,7 +17,12 @@ public RequiredMatchSiloMetadataPlacementFilterStrategy() : this([], 0) public override void AdditionalInitialize(GrainProperties properties) { - MetadataKeys = GetPlacementFilterGrainProperty("metadata-keys", properties).Split(","); + var placementFilterGrainProperty = GetPlacementFilterGrainProperty("metadata-keys", properties); + if (placementFilterGrainProperty is null) + { + throw new ArgumentException("Invalid metadata-keys property value."); + } + MetadataKeys = placementFilterGrainProperty.Split(","); } protected override IEnumerable> GetAdditionalGrainProperties(IServiceProvider services, Type grainClass, GrainType grainType,