feat: add embedded object storage that stores object data into the da…

…tabase
aneoconsulting · Nov 7, 2024 · 6021ec2 · 6021ec2
1 parent 48dd3f2
commit 6021ec2
Show file tree

Hide file tree

Showing 15 changed files with 367 additions and 46 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -40,6 +40,7 @@ jobs:
           - Adaptors/MongoDB/tests
           - Adaptors/Memory/tests
           - Adaptors/S3/tests
+          - Adaptors/Embed/tests
         os:
           - ubuntu-latest
       fail-fast: false
@@ -169,6 +170,7 @@ jobs:
           - Common/tests
           - Adaptors/MongoDB/tests
           - Adaptors/Memory/tests
+          - Adaptors/Embed/tests
       fail-fast: false
     runs-on: windows-latest
     steps:
@@ -179,14 +181,10 @@ jobs:
         submodules: true
 
     - name: Dotnet Restore
-      run: |
-        cd ${{ matrix.projects }}
-        dotnet restore
+      run: dotnet restore
 
     - name: Dotnet Build
-      run: |
-        cd ${{ matrix.projects }}
-        dotnet build
+      run: dotnet build
 
     - name: Run tests
       run: |
@@ -430,6 +428,7 @@ jobs:
         object:
           - redis
           - minio
+          - embed
         log-level:
           - Information
           - Verbose

diff --git a/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj b/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj
@@ -0,0 +1,75 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <RuntimeIdentifiers>win-x64;linux-x64;linux-arm64</RuntimeIdentifiers>
+    <GeneratePackageOnBuild>True</GeneratePackageOnBuild>
+    <Company>ANEO</Company>
+    <Copyright>Copyright (C) ANEO, 2021-2022</Copyright>
+    <PackageLicenseExpression>AGPL-3.0-or-later</PackageLicenseExpression>
+    <PackageRequireLicenseAcceptance>True</PackageRequireLicenseAcceptance>
+    <IsPackable>true</IsPackable>
+    <Nullable>enable</Nullable>
+    <EnableDynamicLoading>true</EnableDynamicLoading>
+  </PropertyGroup>
+
+  <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
+    <DebugType>Embedded</DebugType>
+    <IncludeSymbols>true</IncludeSymbols>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)'=='Release'">
+    <Optimize>true</Optimize>
+    <IncludeSymbols>true</IncludeSymbols>
+    <SymbolPackageFormat>snupkg</SymbolPackageFormat>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="ArmoniK.Utils.Diagnostics" Version="0.5.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="StackExchange.Redis" Version="2.8.16" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="JetBrains.Annotations" Version="2024.2.0">
+      <ExcludeAssets>runtime</ExcludeAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0">
+      <ExcludeAssets>runtime</ExcludeAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.2">
+      <ExcludeAssets>runtime</ExcludeAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.1">
+      <Private>false</Private>
+      <ExcludeAssets>runtime</ExcludeAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.2">
+      <ExcludeAssets>runtime</ExcludeAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.2">
+      <ExcludeAssets>runtime</ExcludeAssets>
+    </PackageReference>
+    <PackageReference Include="ArmoniK.Utils.Diagnostics" Version="0.5.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="ArmoniK.Utils" Version="0.5.1">
+      <ExcludeAssets>runtime</ExcludeAssets>
+    </PackageReference>
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\Utils\src\ArmoniK.Core.Utils.csproj">
+      <Private>false</Private>
+      <ExcludeAssets>runtime</ExcludeAssets>
+    </ProjectReference>
+    <ProjectReference Include="..\..\..\Base\src\ArmoniK.Core.Base.csproj">
+      <Private>false</Private>
+      <ExcludeAssets>runtime</ExcludeAssets>
+    </ProjectReference>
+  </ItemGroup>
+
+</Project>
diff --git a/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj.DotSettings b/Adaptors/Embed/src/ArmoniK.Core.Adapters.Embed.csproj.DotSettings
@@ -0,0 +1,2 @@
+<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:String x:Key="/Default/CodeInspection/Daemon/ConfigureAwaitAnalysisMode/@EntryValue">Library</s:String></wpf:ResourceDictionary>
diff --git a/Adaptors/Embed/src/ObjectBuilder.cs b/Adaptors/Embed/src/ObjectBuilder.cs
@@ -0,0 +1,41 @@
+// This file is part of the ArmoniK project
+// 
+// Copyright (C) ANEO, 2021-2024. All rights reserved.
+// 
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+// 
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY, without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+// 
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+using ArmoniK.Core.Base;
+using ArmoniK.Core.Utils;
+
+using JetBrains.Annotations;
+
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+
+namespace ArmoniK.Core.Adapters.Embed;
+
+/// <summary>
+///   Class for building RabbitMQ object and Queue interfaces through Dependency Injection
+/// </summary>
+[PublicAPI]
+public class ObjectBuilder : IDependencyInjectionBuildable
+{
+  /// <inheritdoc />
+  [PublicAPI]
+  public void Build(IServiceCollection   serviceCollection,
+                    ConfigurationManager configuration,
+                    ILogger              logger)
+    => serviceCollection.AddSingletonWithHealthCheck<IObjectStorage, ObjectStorage>(nameof(IObjectStorage));
+}
diff --git a/Adaptors/Embed/src/ObjectStorage.cs b/Adaptors/Embed/src/ObjectStorage.cs
@@ -0,0 +1,86 @@
+// This file is part of the ArmoniK project
+// 
+// Copyright (C) ANEO, 2021-2024. All rights reserved.
+// 
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+// 
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY, without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+// 
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+using System;
+using System.Collections.Generic;
+using System.Runtime.CompilerServices;
+using System.Threading;
+using System.Threading.Tasks;
+
+using ArmoniK.Core.Base;
+using ArmoniK.Core.Base.DataStructures;
+
+using JetBrains.Annotations;
+
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Logging;
+
+namespace ArmoniK.Core.Adapters.Embed;
+
+[UsedImplicitly]
+public class ObjectStorage : IObjectStorage
+{
+  private readonly ILogger<ObjectStorage> logger_;
+  private          bool                   isInitialized_;
+
+  /// <summary>
+  ///   <see cref="IObjectStorage" /> implementation for Redis
+  /// </summary>
+  /// <param name="logger">Logger used to print logs</param>
+  public ObjectStorage(ILogger<ObjectStorage> logger)
+    => logger_ = logger;
+
+  /// <inheritdoc />
+  public Task Init(CancellationToken cancellationToken)
+  {
+    isInitialized_ = true;
+    return Task.CompletedTask;
+  }
+
+  /// <inheritdoc />
+  public Task<HealthCheckResult> Check(HealthCheckTag tag)
+    => Task.FromResult(isInitialized_
+                         ? HealthCheckResult.Healthy()
+                         : HealthCheckResult.Unhealthy("Object storage not initialized yet."));
+
+  /// <inheritdoc />
+  public async Task<(byte[] id, long size)> AddOrUpdateAsync(IAsyncEnumerable<ReadOnlyMemory<byte>> valueChunks,
+                                                             CancellationToken                      cancellationToken = default)
+  {
+    var array = new List<byte>();
+
+    await foreach (var val in valueChunks.WithCancellation(cancellationToken)
+                                         .ConfigureAwait(false))
+    {
+      array.AddRange(val.ToArray());
+    }
+
+    return (array.ToArray(), array.Count);
+  }
+
+  /// <inheritdoc />
+  public async IAsyncEnumerable<byte[]> GetValuesAsync(byte[]                                     id,
+                                                       [EnumeratorCancellation] CancellationToken cancellationToken = default)
+  {
+    yield return id;
+  }
+
+  /// <inheritdoc />
+  public Task TryDeleteAsync(IEnumerable<byte[]> ids,
+                             CancellationToken   cancellationToken = default)
+    => Task.CompletedTask;
+}
diff --git a/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj b/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj
@@ -0,0 +1,28 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <RuntimeIdentifiers>win-x64;linux-x64;linux-arm64</RuntimeIdentifiers>
+    <IsPackable>false</IsPackable>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="ArmoniK.Utils.Diagnostics" Version="0.5.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="NUnit3TestAdapter" Version="4.5.0" />
+    <PackageReference Include="NUnit" Version="3.14.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
+    <PackageReference Include="redis-inside" Version="3.3.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <AssemblyAttribute Include="System.Diagnostics.CodeAnalysis.ExcludeFromCodeCoverageAttribute" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\Common\tests\ArmoniK.Core.Common.Tests.csproj" />
+  </ItemGroup>
+
+</Project>
diff --git a/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj.DotSettings b/Adaptors/Embed/tests/ArmoniK.Core.Adapters.Embed.Tests.csproj.DotSettings
@@ -0,0 +1,2 @@
+<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:String x:Key="/Default/CodeInspection/Daemon/ConfigureAwaitAnalysisMode/@EntryValue">Library</s:String></wpf:ResourceDictionary>
diff --git a/Adaptors/Embed/tests/ObjectStorageTests.cs b/Adaptors/Embed/tests/ObjectStorageTests.cs
@@ -0,0 +1,81 @@
+// This file is part of the ArmoniK project
+// 
+// Copyright (C) ANEO, 2021-2024. All rights reserved.
+// 
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+// 
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY, without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+// 
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+using System.Collections.Generic;
+using System.IO;
+
+using ArmoniK.Core.Base;
+using ArmoniK.Core.Common.Injection;
+using ArmoniK.Core.Common.Injection.Options;
+using ArmoniK.Core.Common.Tests.TestBase;
+using ArmoniK.Core.Common.Utils;
+
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+
+using NUnit.Framework;
+
+namespace ArmoniK.Core.Adapters.Embed.Tests;
+
+[TestFixture]
+public class ObjectStorageTests : ObjectStorageTestBase
+{
+  private static readonly string SolutionRoot =
+    Path.GetFullPath(Path.Combine(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(Path.GetDirectoryName(typeof(ObjectStorageTests)
+                                                                                                                                                .Assembly
+                                                                                                                                                .Location))))) ??
+                                  string.Empty));
+
+  private static readonly string S3Path =
+    $"{Path.DirectorySeparatorChar}src{Path.DirectorySeparatorChar}bin{Path.DirectorySeparatorChar}Debug{Path.DirectorySeparatorChar}net8.0{Path.DirectorySeparatorChar}ArmoniK.Core.Adapters.Embed.dll";
+
+
+  protected override void GetObjectStorageInstance()
+  {
+    Dictionary<string, string?> minimalConfig = new()
+                                                {
+                                                  {
+                                                    "Components:ObjectStorageAdaptorSettings:ClassName", "ArmoniK.Core.Adapters.Embed.ObjectBuilder"
+                                                  },
+                                                  {
+                                                    "Components:ObjectStorageAdaptorSettings:AdapterAbsolutePath", $"{SolutionRoot}{S3Path}"
+                                                  },
+                                                };
+
+    var configuration = new ConfigurationManager();
+    configuration.AddInMemoryCollection(minimalConfig);
+
+    var services = new ServiceCollection();
+    services.AddLogging();
+    var logger = new LoggerInit(configuration);
+
+    services.AddAdapter(configuration,
+                        nameof(Components.ObjectStorageAdaptorSettings),
+                        logger.GetLogger());
+
+    var provider = services.BuildServiceProvider(new ServiceProviderOptions
+                                                 {
+                                                   ValidateOnBuild = true,
+                                                 });
+
+    ObjectStorage = provider.GetRequiredService<IObjectStorage>();
+    RunTests      = true;
+  }
+
+  protected override bool HasErrors()
+    => false;
+}