Skip to content

Commit

Permalink
feat: dynamic object storage
Browse files Browse the repository at this point in the history
  • Loading branch information
aneojgurhem committed Oct 21, 2024
1 parent 5b30b24 commit 5194f79
Show file tree
Hide file tree
Showing 67 changed files with 457 additions and 364 deletions.
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<RuntimeIdentifiers>win-x64;linux-x64;linux-arm64</RuntimeIdentifiers>
Expand All @@ -10,10 +10,18 @@
<GeneratePackageOnBuild>True</GeneratePackageOnBuild>
<IsPackable>true</IsPackable>
<Nullable>enable</Nullable>
<EnableDynamicLoading>true</EnableDynamicLoading>
</PropertyGroup>

<ItemGroup>
<ProjectReference Include="..\..\..\Common\src\ArmoniK.Core.Common.csproj" />
<ProjectReference Include="..\..\..\Utils\src\ArmoniK.Core.Utils.csproj">
<Private>false</Private>
<ExcludeAssets>runtime</ExcludeAssets>
</ProjectReference>
<ProjectReference Include="..\..\..\Base\src\ArmoniK.Core.Base.csproj">
<Private>false</Private>
<ExcludeAssets>runtime</ExcludeAssets>
</ProjectReference>
</ItemGroup>

<ItemGroup>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

using ArmoniK.Api.Common.Utils;
using ArmoniK.Core.Common.Injection.Options;
using ArmoniK.Core.Base;
using ArmoniK.Core.Common.Storage;
using ArmoniK.Core.Utils;

Expand All @@ -28,34 +27,27 @@

namespace ArmoniK.Core.Adapters.LocalStorage;

public static class ServiceCollectionExt
/// <summary>
/// Class for building RabbitMQ object and Queue interfaces through Dependency Injection
/// </summary>
[PublicAPI]
public class ObjectBuilder : IDependencyInjectionBuildable
{
/// <inheritdoc />
[PublicAPI]
public static IServiceCollection AddLocalStorage(this IServiceCollection serviceCollection,
ConfigurationManager configuration,
ILogger logger)
public void Build(IServiceCollection serviceCollection,
ConfigurationManager configuration,
ILogger logger)
{
var components = configuration.GetSection(Components.SettingSection);

if (components["ObjectStorage"] != "ArmoniK.Adapters.LocalStorage.ObjectStorage")
{
return serviceCollection;
}

serviceCollection.AddOption(configuration,
Options.LocalStorage.SettingSection,
out Options.LocalStorage storageOptions);

using var _ = logger.BeginNamedScope("Object Local configuration",
("Path", storageOptions.Path));

logger.LogDebug("setup local storage");

serviceCollection.AddSingletonWithHealthCheck<IObjectStorage>(nameof(IObjectStorage),
sp => new ObjectStorage(storageOptions.Path,
storageOptions.ChunkSize,
sp.GetRequiredService<ILogger<ObjectStorage>>()));

return serviceCollection;
}
}
8 changes: 1 addition & 7 deletions Adaptors/LocalStorage/src/ObjectStorage.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,8 @@
using System.Threading;
using System.Threading.Tasks;

using ArmoniK.Api.Common.Utils;
using ArmoniK.Core.Base.DataStructures;
using ArmoniK.Core.Common.Exceptions;
using ArmoniK.Core.Base.Exceptions;
using ArmoniK.Core.Common.Storage;

using Microsoft.Extensions.Diagnostics.HealthChecks;
Expand Down Expand Up @@ -102,7 +101,6 @@ public async Task<long> AddOrUpdateAsync(string
var filename = Path.Combine(path_,
key);

using var _ = logger_.LogFunction(filename);

// Write to temporary file
await using var file = File.Open(filename,
Expand Down Expand Up @@ -145,8 +143,6 @@ public async IAsyncEnumerable<byte[]> GetValuesAsync(string
var filename = Path.Combine(path_,
key);

using var _ = logger_.LogFunction(filename);

if (!File.Exists(filename))
{
throw new ObjectDataNotFoundException($"The object {key} has not been found in {path_}");
Expand Down Expand Up @@ -208,8 +204,6 @@ public Task<bool> TryDeleteAsync(string key,
var filename = Path.Combine(path_,
key);

using var _ = logger_.LogFunction(filename);

File.Delete(filename);

return Task.FromResult(true);
Expand Down
1 change: 1 addition & 0 deletions Adaptors/Memory/src/ResultTable.cs
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
using System.Threading.Tasks;

using ArmoniK.Core.Base.DataStructures;
using ArmoniK.Core.Base.Exceptions;
using ArmoniK.Core.Common.Exceptions;
using ArmoniK.Core.Common.Storage;

Expand Down
2 changes: 1 addition & 1 deletion Adaptors/MongoDB/src/ObjectStorage.cs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
using ArmoniK.Core.Adapters.MongoDB.Common;
using ArmoniK.Core.Adapters.MongoDB.Object;
using ArmoniK.Core.Base.DataStructures;
using ArmoniK.Core.Common.Exceptions;
using ArmoniK.Core.Base.Exceptions;
using ArmoniK.Core.Common.Storage;
using ArmoniK.Utils;

Expand Down
1 change: 1 addition & 0 deletions Adaptors/MongoDB/src/ResultTable.cs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
using ArmoniK.Core.Adapters.MongoDB.Common;
using ArmoniK.Core.Adapters.MongoDB.Table.DataModel;
using ArmoniK.Core.Base.DataStructures;
using ArmoniK.Core.Base.Exceptions;
using ArmoniK.Core.Common.Exceptions;
using ArmoniK.Core.Common.Storage;
using ArmoniK.Utils;
Expand Down
1 change: 1 addition & 0 deletions Adaptors/MongoDB/src/TaskTable.cs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
using ArmoniK.Core.Adapters.MongoDB.Options;
using ArmoniK.Core.Adapters.MongoDB.Table.DataModel;
using ArmoniK.Core.Base.DataStructures;
using ArmoniK.Core.Base.Exceptions;
using ArmoniK.Core.Common.Exceptions;
using ArmoniK.Core.Common.Storage;
using ArmoniK.Utils;
Expand Down
3 changes: 0 additions & 3 deletions Adaptors/MongoDB/tests/MongoDatabaseProvider.cs
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,6 @@ public MongoDatabaseProvider(bool useSingleNodeReplicaSet
{
$"{Components.SettingSection}:{nameof(Components.TableStorage)}", "ArmoniK.Adapters.MongoDB.TableStorage"
},
{
$"{Components.SettingSection}:{nameof(Components.ObjectStorage)}", "ArmoniK.Adapters.MongoDB.ObjectStorage"
},
{
$"{Components.SettingSection}:{nameof(Components.AuthenticationStorage)}",
"ArmoniK.Adapters.MongoDB.AuthenticationTable"
Expand Down
53 changes: 51 additions & 2 deletions Adaptors/Redis/src/ArmoniK.Core.Adapters.Redis.csproj
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<RuntimeIdentifiers>win-x64;linux-x64;linux-arm64</RuntimeIdentifiers>
Expand All @@ -9,6 +9,19 @@
<PackageRequireLicenseAcceptance>True</PackageRequireLicenseAcceptance>
<IsPackable>true</IsPackable>
<Nullable>enable</Nullable>
<EnableDynamicLoading>true</EnableDynamicLoading>
</PropertyGroup>

<PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
<DebugType>Embedded</DebugType>
<IncludeSymbols>true</IncludeSymbols>
<DefineConstants>DEBUG;TRACE</DefineConstants>
</PropertyGroup>

<PropertyGroup Condition="'$(Configuration)'=='Release'">
<Optimize>true</Optimize>
<IncludeSymbols>true</IncludeSymbols>
<SymbolPackageFormat>snupkg</SymbolPackageFormat>
</PropertyGroup>

<ItemGroup>
Expand All @@ -20,7 +33,43 @@
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\..\..\Common\src\ArmoniK.Core.Common.csproj" />
<PackageReference Include="JetBrains.Annotations" Version="2024.2.0">
<ExcludeAssets>runtime</ExcludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0">
<ExcludeAssets>runtime</ExcludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.2">
<ExcludeAssets>runtime</ExcludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.1">
<Private>false</Private>
<ExcludeAssets>runtime</ExcludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.2">
<ExcludeAssets>runtime</ExcludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.2">
<ExcludeAssets>runtime</ExcludeAssets>
</PackageReference>
<PackageReference Include="ArmoniK.Utils.Diagnostics" Version="0.5.1">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="ArmoniK.Utils" Version="0.5.1">
<ExcludeAssets>runtime</ExcludeAssets>
</PackageReference>
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\..\..\Utils\src\ArmoniK.Core.Utils.csproj">
<Private>false</Private>
<ExcludeAssets>runtime</ExcludeAssets>
</ProjectReference>
<ProjectReference Include="..\..\..\Base\src\ArmoniK.Core.Base.csproj">
<Private>false</Private>
<ExcludeAssets>runtime</ExcludeAssets>
</ProjectReference>
</ItemGroup>

</Project>
118 changes: 118 additions & 0 deletions Adaptors/Redis/src/ObjectBuilder.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
// This file is part of the ArmoniK project
//
// Copyright (C) ANEO, 2021-2024. All rights reserved.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY, without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

using System;
using System.IO;
using System.Security.Cryptography.X509Certificates;

using ArmoniK.Core.Base;
using ArmoniK.Core.Common.Storage;
using ArmoniK.Core.Utils;

using JetBrains.Annotations;

using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;

using StackExchange.Redis;

namespace ArmoniK.Core.Adapters.Redis;

/// <summary>
/// Class for building RabbitMQ object and Queue interfaces through Dependency Injection
/// </summary>
[PublicAPI]
public class ObjectBuilder : IDependencyInjectionBuildable
{
/// <inheritdoc />
[PublicAPI]
public void Build(IServiceCollection serviceCollection,
ConfigurationManager configuration,
ILogger logger)
{
// ReSharper disable once InlineOutVariableDeclaration
Options.Redis redisOptions;
serviceCollection.AddOption(configuration,
Options.Redis.SettingSection,
out redisOptions);

if (!string.IsNullOrEmpty(redisOptions.CredentialsPath))
{
configuration.AddJsonFile(redisOptions.CredentialsPath,
false,
false);

serviceCollection.AddOption(configuration,
Options.Redis.SettingSection,
out redisOptions);

logger.LogTrace("Loaded Redis credentials from file {path}",
redisOptions.CredentialsPath);
}

if (!string.IsNullOrEmpty(redisOptions.CaPath))
{
var localTrustStore = new X509Store(StoreName.Root);
var certificateCollection = new X509Certificate2Collection();
try
{
certificateCollection.ImportFromPemFile(redisOptions.CaPath);
localTrustStore.Open(OpenFlags.ReadWrite);
localTrustStore.AddRange(certificateCollection);
logger.LogTrace("Imported Redis certificate from file {path}",
redisOptions.CaPath);
}
catch (Exception ex)
{
logger.LogError("Root certificate import failed: {error}",
ex.Message);
throw;
}
finally
{
localTrustStore.Close();
}
}

var config = new ConfigurationOptions
{
ClientName = redisOptions.ClientName,
ReconnectRetryPolicy = new ExponentialRetry(10),
Ssl = redisOptions.Ssl,
AbortOnConnectFail = true,
SslHost = redisOptions.SslHost,
Password = redisOptions.Password,
User = redisOptions.User,
};
config.EndPoints.Add(redisOptions.EndpointUrl);

if (redisOptions.Timeout > 0)
{
config.ConnectTimeout = redisOptions.Timeout;
}

logger.LogDebug("setup connection to Redis at {EndpointUrl} with user {user}",
redisOptions.EndpointUrl,
redisOptions.User);

serviceCollection.AddSingleton<IDatabaseAsync>(_ => ConnectionMultiplexer.Connect(config,
TextWriter.Null)
.GetDatabase());
serviceCollection.AddSingletonWithHealthCheck<IObjectStorage, ObjectStorage>(nameof(IObjectStorage));
}
}
14 changes: 3 additions & 11 deletions Adaptors/Redis/src/ObjectStorage.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,8 @@
using System.Threading;
using System.Threading.Tasks;

using ArmoniK.Api.Common.Utils;
using ArmoniK.Core.Base.DataStructures;
using ArmoniK.Core.Common.Exceptions;
using ArmoniK.Core.Base.Exceptions;
using ArmoniK.Core.Common.Storage;
using ArmoniK.Utils;

Expand Down Expand Up @@ -91,9 +90,8 @@ public async Task<long> AddOrUpdateAsync(string
IAsyncEnumerable<ReadOnlyMemory<byte>> valueChunks,
CancellationToken cancellationToken = default)
{
long size = 0;
var storageNameKey = objectStorageName_ + key;
using var _ = logger_.LogFunction(storageNameKey);
long size = 0;
var storageNameKey = objectStorageName_ + key;

var idx = 0;
var taskList = new List<Task>();
Expand All @@ -119,7 +117,6 @@ await taskList.WhenAll()
public async IAsyncEnumerable<byte[]> GetValuesAsync(string key,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{
using var _ = logger_.LogFunction(objectStorageName_ + key);
var value = await PerformActionWithRetry(() => redis_.StringGetAsync(objectStorageName_ + key + "_count"))
.ConfigureAwait(false);

Expand Down Expand Up @@ -151,14 +148,9 @@ public async Task TryDeleteAsync(IEnumerable<string> keys,
cancellationToken))
.ConfigureAwait(false);

/// <inheritdoc />
public IAsyncEnumerable<string> ListKeysAsync(CancellationToken cancellationToken = default)
=> throw new NotImplementedException();

private async Task TryDeleteAsync(string key,
CancellationToken cancellationToken = default)
{
using var _ = logger_.LogFunction(objectStorageName_ + key);
var value = await PerformActionWithRetry(() => redis_.StringGetAsync(objectStorageName_ + key + "_count"))
.ConfigureAwait(false);

Expand Down
Loading

0 comments on commit 5194f79

Please sign in to comment.