Merge branch 'v5.4-merge' into v6.0

bench/Micro.Benchmark/Micro.Benchmark.csproj

@@ -16,6 +16,9 @@
     <PackageReference Include="BenchmarkDotNet.Diagnostics.Windows" Version="0.13.12" />
     <PackageReference Include="BenchmarkDotNet" Version="0.13.12" />
     <PackageReference Include="xunit" Version="2.8.1" />
+    <PackageReference Include="System.Net.Http" Version="4.3.4" />
+    <PackageReference Include="System.Text.RegularExpressions" Version="4.3.1" />
+    <PackageReference Include="xunit" Version="2.4.2" />
     <PackageReference Include="System.Runtime.CompilerServices.Unsafe" Version="6.0.0" />
   </ItemGroup>
   <ItemGroup>

bench/Regression.Benchmark/Regression.Benchmark.csproj

@@ -18,6 +18,8 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
     <PackageReference Include="Microsoft.NETCore.Jit" Version="2.0.8" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.8.1">
+    <PackageReference Include="System.Text.RegularExpressions" Version="4.3.1" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>

scripts/linux/pkg/deb/assets/build.sh

@@ -5,7 +5,7 @@ DEST_DIR=/build
 release=$(lsb_release -sr | cut -d. -f1)
 
 if [[ $release -ge 22 ]]; then
-    mv -v $ASSETS_DIR/ravendb/debian/control_$release $ASSETS_DIR/ravendb/debian/control
+    mv -v $ASSETS_DIR/ravendb/debian/control_22 $ASSETS_DIR/ravendb/debian/control
 else
     apt install dh-systemd
     mv -v $ASSETS_DIR/ravendb/debian/control_legacy $ASSETS_DIR/ravendb/debian/control
@@ -41,15 +41,26 @@ DOTNET_VERSION_MINOR=$(egrep -o -e '^[0-9]+.[0-9]+' <<< $DOTNET_FULL_VERSION)
 export DOTNET_DEPS_VERSION="$DOTNET_FULL_VERSION"
 export DOTNET_RUNTIME_VERSION="$DOTNET_VERSION_MINOR"
 
-# Show dependencies for amd64 since that's the only platform Microsoft ships package for,
-# however the dependencies are the same at the moment.
-DOTNET_RUNTIME_DEPS_PKG="dotnet-runtime-deps-$DOTNET_RUNTIME_VERSION:amd64"
-DOTNET_RUNTIME_DEPS=$(apt show $DOTNET_RUNTIME_DEPS_PKG 2>/dev/null | sed -n -e 's/Depends: //p')
-if [ -z "$DOTNET_RUNTIME_DEPS" ]; then
-    echo "Could not extract dependencies from $DOTNET_RUNTIME_DEPS_PKG package."
-    exit 1
+if [[ $release -eq 24 && $RAVEN_PLATFORM == "raspberry-pi" ]]; then
+    DOTNET_RUNTIME_DEPS="libicu74, libc6 (>= 2.38), libgcc-s1 (>= 3.0), liblttng-ust1t64 (>= 2.13.0), libssl3t64 (>= 3.0.0), libstdc++6 (>= 13.1), zlib1g (>= 1:1.1.4)"
+else
+    if [[ $release -ge 24 ]]; then
+        DOTNET_RUNTIME_DEPS_PKG="dotnet-runtime-$DOTNET_RUNTIME_VERSION"
+    else
+        # Show dependencies for amd64 since that's the only platform Microsoft ships package for,
+        # however the dependencies are the same at the moment.
+        DOTNET_RUNTIME_DEPS_PKG="dotnet-runtime-$DOTNET_RUNTIME_VERSION:amd64"
+    fi
+
+    # get depenencies and remove dotnet-host* dependencies
+    DOTNET_RUNTIME_DEPS=$(apt show $DOTNET_RUNTIME_DEPS_PKG 2>/dev/null | sed -n -e 's/Depends: //p' | sed -E 's/(^|, )dotnet-host[^,]*(, |$)/\1/; s/, $//')
+    if [ -z "$DOTNET_RUNTIME_DEPS" ]; then
+        echo "Could not extract dependencies from $DOTNET_RUNTIME_DEPS_PKG package."
+        exit 1
+    fi
 fi
 
+
 export DEB_DEPS="${DOTNET_RUNTIME_DEPS}, libc6-dev (>= 2.27)"
 
 echo ".NET Runtime: $DOTNET_FULL_VERSION"

scripts/linux/pkg/deb/build-deb_ubuntu-noble_amd64.ps1

@@ -0,0 +1,7 @@
+$env:OUTPUT_DIR = "$PSScriptRoot/dist"
+
+.\set-ubuntu-noble.ps1
+.\set-raven-platform-amd64.ps1
+.\set-raven-version-env.ps1
+
+.\build-deb.ps1

scripts/linux/pkg/deb/set-ubuntu-noble.ps1

@@ -0,0 +1,3 @@
+$env:DISTRO_NAME = "ubuntu"
+$env:DISTRO_VERSION = "24.04"
+$env:DISTRO_VERSION_NAME ="noble"

scripts/linux/pkg/deb/set-ubuntu-noble.sh

@@ -0,0 +1,3 @@
+export DISTRO_NAME="ubuntu"
+export DISTRO_VERSION="24.04"
+export DISTRO_VERSION_NAME="noble"

src/Raven.Server/Monitoring/Snmp/Objects/Database/5.1/5.1.11/ActiveOngoingTasksBase.cs

@@ -0,0 +1,105 @@
+using System.Collections.Generic;
+using System.Linq;
+using Lextm.SharpSnmpLib;
+using Raven.Client.Documents.Subscriptions;
+using Raven.Client.Json.Serialization;
+using Raven.Client.ServerWide;
+using Raven.Server.ServerWide;
+using Raven.Server.ServerWide.Context;
+
+namespace Raven.Server.Monitoring.Snmp.Objects.Database;
+
+public abstract class ActiveOngoingTasksBase : DatabaseBase<Integer32>
+{
+    protected ActiveOngoingTasksBase(ServerStore serverStore, string dots)
+        : base(serverStore, dots)
+    {
+    }
+
+    protected override IEnumerable<RawDatabaseRecord> GetDatabases(TransactionOperationContext context)
+    {
+        foreach (var database in base.GetDatabases(context))
+        {
+            if (database.IsDisabled)
+                continue;
+
+            if (ServerStore.DatabasesLandlord.IsDatabaseLoaded(database.DatabaseName) == false)
+                continue;
+
+            yield return database;
+        }
+    }
+
+    protected abstract int GetCount(TransactionOperationContext context, RachisState rachisState, string nodeTag, RawDatabaseRecord database);
+
+    protected override Integer32 GetData()
+    {
+        var count = 0;
+        using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
+        using (context.OpenReadTransaction())
+        {
+            var rachisState = ServerStore.CurrentRachisState;
+            var nodeTag = ServerStore.NodeTag;
+
+            foreach (var database in GetDatabases(context))
+            {
+                count += GetCount(context, rachisState, nodeTag, database);
+            }
+        }
+
+        return new Integer32(count);
+    }
+
+    protected static int GetNumberOfActiveElasticSearchEtls(RachisState rachisState, string nodeTag, RawDatabaseRecord database) => CountTasks(rachisState, nodeTag, database.Topology, database.ElasticSearchEtls.Where(x => x.Disabled == false));
+
+    protected static int GetNumberOfActiveExternalReplications(RachisState rachisState, string nodeTag, RawDatabaseRecord database) => CountTasks(rachisState, nodeTag, database.Topology, database.ExternalReplications.Where(x => x.Disabled == false));
+
+    protected static int GetNumberOfActiveOlapEtls(RachisState rachisState, string nodeTag, RawDatabaseRecord database) => CountTasks(rachisState, nodeTag, database.Topology, database.OlapEtls.Where(x => x.Disabled == false));
+
+    protected static int GetNumberOfActivePeriodicBackups(RachisState rachisState, string nodeTag, RawDatabaseRecord database) => CountTasks(rachisState, nodeTag, database.Topology, database.PeriodicBackups.Where(x => x.Disabled == false));
+
+    protected static int GetNumberOfActiveQueueEtls(RachisState rachisState, string nodeTag, RawDatabaseRecord database) => CountTasks(rachisState, nodeTag, database.Topology, database.QueueEtls.Where(x => x.Disabled == false));
+
+    protected static int GetNumberOfActiveRavenEtls(RachisState rachisState, string nodeTag, RawDatabaseRecord database) => CountTasks(rachisState, nodeTag, database.Topology, database.RavenEtls.Where(x => x.Disabled == false));
+
+    protected static int GetNumberOfActiveSinkPullReplications(RachisState rachisState, string nodeTag, RawDatabaseRecord database) => CountTasks(rachisState, nodeTag, database.Topology, database.SinkPullReplications.Where(x => x.Disabled == false));
+
+    protected static int GetNumberOfActiveSqlEtls(RachisState rachisState, string nodeTag, RawDatabaseRecord database) => CountTasks(rachisState, nodeTag, database.Topology, database.SqlEtls.Where(x => x.Disabled == false));
+
+    protected static int GetNumberOfActiveSubscriptions(TransactionOperationContext context, RachisState rachisState, string nodeTag, RawDatabaseRecord database)
+    {
+        var count = 0;
+        foreach (var kvp in ClusterStateMachine.ReadValuesStartingWith(context, SubscriptionState.SubscriptionPrefix(database.DatabaseName)))
+        {
+            var subscriptionState = JsonDeserializationClient.SubscriptionState(kvp.Value);
+            if (subscriptionState.Disabled)
+                continue;
+
+            var responsibleNode = database.Topology.WhoseTaskIsIt(rachisState, subscriptionState);
+            if (responsibleNode != nodeTag)
+                continue;
+
+            count++;
+        }
+
+        return count;
+    }
+
+    private static int CountTasks(RachisState rachisState, string nodeTag, DatabaseTopology databaseTopology, IEnumerable<IDatabaseTask> tasks)
+    {
+        if (tasks == null)
+            return 0;
+
+        var count = 0;
+        foreach (var task in tasks)
+        {
+            var responsibleNode = databaseTopology.WhoseTaskIsIt(rachisState, task);
+            if (responsibleNode != nodeTag)
+                continue;
+
+            count++;
+        }
+
+        return count;
+    }
+}

src/Raven.Server/Monitoring/Snmp/Objects/Database/5.1/5.1.11/OngoingTasksBase.cs

@@ -0,0 +1,74 @@
+using System.Collections.Generic;
+using System.Linq;
+using Lextm.SharpSnmpLib;
+using Raven.Client.Documents.Subscriptions;
+using Raven.Server.ServerWide;
+using Raven.Server.ServerWide.Context;
+
+namespace Raven.Server.Monitoring.Snmp.Objects.Database;
+
+public abstract class OngoingTasksBase : DatabaseBase<Integer32>
+{
+    protected OngoingTasksBase(ServerStore serverStore, string dots)
+        : base(serverStore, dots)
+    {
+    }
+
+    protected override IEnumerable<RawDatabaseRecord> GetDatabases(TransactionOperationContext context)
+    {
+        foreach (var database in base.GetDatabases(context))
+        {
+            if (database.IsDisabled)
+                continue;
+
+            yield return database;
+        }
+    }
+
+    protected abstract int GetCount(TransactionOperationContext context, RawDatabaseRecord database);
+
+    protected override Integer32 GetData()
+    {
+        var count = 0;
+        using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context))
+        using (context.OpenReadTransaction())
+        {
+            foreach (var database in GetDatabases(context))
+            {
+                count += GetCount(context, database);
+            }
+        }
+
+        return new Integer32(count);
+    }
+
+    protected internal static int GetNumberOfElasticSearchEtls(RawDatabaseRecord database) => database.ElasticSearchEtls?.Count(x => x.Disabled == false) ?? 0;
+
+    protected internal static int GetNumberOfExternalReplications(RawDatabaseRecord database) => database.ExternalReplications?.Count(x => x.Disabled == false) ?? 0;
+
+    protected internal static int GetNumberOfOlapEtls(RawDatabaseRecord database) => database.OlapEtls?.Count(x => x.Disabled == false) ?? 0;
+
+    protected internal static int GetNumberOfPeriodicBackups(RawDatabaseRecord database) => database.PeriodicBackups?.Count(x => x.Disabled == false) ?? 0;
+
+    protected internal static int GetNumberOfQueueEtls(RawDatabaseRecord database) => database.QueueEtls?.Count(x => x.Disabled == false) ?? 0;
+
+    protected internal static int GetNumberOfRavenEtls(RawDatabaseRecord database) => database.RavenEtls?.Count(x => x.Disabled == false) ?? 0;
+
+    protected internal static int GetNumberOfSinkPullReplications(RawDatabaseRecord database) => database.SinkPullReplications?.Count(x => x.Disabled == false) ?? 0;
+
+    protected internal static int GetNumberOfSqlEtls(RawDatabaseRecord database) => database.SqlEtls?.Count(x => x.Disabled == false) ?? 0;
+
+    protected internal static int GetNumberOfSubscriptions(TransactionOperationContext context, RawDatabaseRecord database)
+    {
+        var count = 0;
+        foreach (var kvp in ClusterStateMachine.ReadValuesStartingWith(context, SubscriptionState.SubscriptionPrefix(database.DatabaseName)))
+        {
+            if (kvp.Value.TryGet(nameof(SubscriptionState.Disabled), out bool disabled) && disabled)
+                continue;
+
+            count++;
+        }
+
+        return count;
+    }
+}

src/Raven.Server/Monitoring/Snmp/Objects/Database/5.1/5.1.11/TotalNumberOfActiveBackupTasks.cs

@@ -0,0 +1,19 @@
+using Raven.Client.ServerWide;
+using Raven.Server.ServerWide;
+using Raven.Server.ServerWide.Context;
+
+namespace Raven.Server.Monitoring.Snmp.Objects.Database
+{
+    public class TotalNumberOfActiveBackupTasks : ActiveOngoingTasksBase
+    {
+        public TotalNumberOfActiveBackupTasks(ServerStore serverStore)
+            : base(serverStore, SnmpOids.Databases.General.TotalNumberOfActiveBackupTasks)
+        {
+        }
+
+        protected override int GetCount(TransactionOperationContext context, RachisState rachisState, string nodeTag, RawDatabaseRecord database)
+        {
+            return GetNumberOfActivePeriodicBackups(rachisState, nodeTag, database);
+        }
+    }
+}

src/Raven.Server/Monitoring/Snmp/Objects/Database/5.1/5.1.11/TotalNumberOfActiveElasticSearchEtlTasks.cs

@@ -0,0 +1,19 @@
+using Raven.Client.ServerWide;
+using Raven.Server.ServerWide;
+using Raven.Server.ServerWide.Context;
+
+namespace Raven.Server.Monitoring.Snmp.Objects.Database
+{
+    public class TotalNumberOfActiveElasticSearchEtlTasks : ActiveOngoingTasksBase
+    {
+        public TotalNumberOfActiveElasticSearchEtlTasks(ServerStore serverStore)
+            : base(serverStore, SnmpOids.Databases.General.TotalNumberOfActiveElasticSearchEtlTasks)
+        {
+        }
+
+        protected override int GetCount(TransactionOperationContext context, RachisState rachisState, string nodeTag, RawDatabaseRecord database)
+        {
+            return GetNumberOfActiveElasticSearchEtls(rachisState, nodeTag, database);
+        }
+    }
+}

src/Raven.Server/Monitoring/Snmp/Objects/Database/5.1/5.1.11/TotalNumberOfActiveExternalReplicationTasks.cs

@@ -0,0 +1,26 @@
+// -----------------------------------------------------------------------
+//  <copyright file="DatabaseOpenedCount.cs" company="Hibernating Rhinos LTD">
+//      Copyright (c) Hibernating Rhinos LTD. All rights reserved.
+//  </copyright>
+// -----------------------------------------------------------------------
+
+using Lextm.SharpSnmpLib;
+using Raven.Client.ServerWide;
+using Raven.Server.ServerWide;
+using Raven.Server.ServerWide.Context;
+
+namespace Raven.Server.Monitoring.Snmp.Objects.Database
+{
+    public class TotalNumberOfActiveExternalReplicationTasks : ActiveOngoingTasksBase
+    {
+        public TotalNumberOfActiveExternalReplicationTasks(ServerStore serverStore)
+            : base(serverStore, SnmpOids.Databases.General.TotalNumberOfActiveExternalReplicationTasks)
+        {
+        }
+
+        protected override int GetCount(TransactionOperationContext context, RachisState rachisState, string nodeTag, RawDatabaseRecord database)
+        {
+            return GetNumberOfActiveExternalReplications(rachisState, nodeTag, database);
+        }
+    }
+}

src/Raven.Server/Monitoring/Snmp/Objects/Database/5.1/5.1.11/TotalNumberOfActiveOlapEtlTasks.cs

@@ -0,0 +1,19 @@
+using Raven.Client.ServerWide;
+using Raven.Server.ServerWide;
+using Raven.Server.ServerWide.Context;
+
+namespace Raven.Server.Monitoring.Snmp.Objects.Database
+{
+    public class TotalNumberOfActiveOlapEtlTasks : ActiveOngoingTasksBase
+    {
+        public TotalNumberOfActiveOlapEtlTasks(ServerStore serverStore)
+            : base(serverStore, SnmpOids.Databases.General.TotalNumberOfActiveOlapEtlTasks)
+        {
+        }
+
+        protected override int GetCount(TransactionOperationContext context, RachisState rachisState, string nodeTag, RawDatabaseRecord database)
+        {
+            return GetNumberOfActiveOlapEtls(rachisState, nodeTag, database);
+        }
+    }
+}

src/Raven.Server/Monitoring/Snmp/Objects/Database/5.1/5.1.11/TotalNumberOfActiveOngoingTasks.cs

@@ -0,0 +1,30 @@
+using Lextm.SharpSnmpLib;
+using Raven.Client.ServerWide;
+using Raven.Server.ServerWide;
+using Raven.Server.ServerWide.Context;
+
+namespace Raven.Server.Monitoring.Snmp.Objects.Database
+{
+    public class TotalNumberOfActiveOngoingTasks : ActiveOngoingTasksBase
+    {
+        public TotalNumberOfActiveOngoingTasks(ServerStore serverStore)
+            : base(serverStore, SnmpOids.Databases.General.TotalNumberOfActiveOngoingTasks)
+        {
+        }
+
+        protected override int GetCount(TransactionOperationContext context, RachisState rachisState, string nodeTag, RawDatabaseRecord database)
+        {
+            var count = GetNumberOfActiveElasticSearchEtls(rachisState, nodeTag, database);
+            count += GetNumberOfActiveExternalReplications(rachisState, nodeTag, database);
+            count += GetNumberOfActiveOlapEtls(rachisState, nodeTag, database);
+            count += GetNumberOfActivePeriodicBackups(rachisState, nodeTag, database);
+            count += GetNumberOfActiveQueueEtls(rachisState, nodeTag, database);
+            count += GetNumberOfActiveRavenEtls(rachisState, nodeTag, database);
+            count += GetNumberOfActiveSinkPullReplications(rachisState, nodeTag, database);
+            count += GetNumberOfActiveSqlEtls(rachisState, nodeTag, database);
+            count += GetNumberOfActiveSubscriptions(context, rachisState, nodeTag, database);
+
+            return count;
+        }
+    }
+}

src/Raven.Server/Monitoring/Snmp/Objects/Database/5.1/5.1.11/TotalNumberOfActivePullReplicationAsSinkTasks.cs

@@ -0,0 +1,19 @@
+using Raven.Client.ServerWide;
+using Raven.Server.ServerWide;
+using Raven.Server.ServerWide.Context;
+
+namespace Raven.Server.Monitoring.Snmp.Objects.Database
+{
+    public class TotalNumberOfActivePullReplicationAsSinkTasks : ActiveOngoingTasksBase
+    {
+        public TotalNumberOfActivePullReplicationAsSinkTasks(ServerStore serverStore)
+            : base(serverStore, SnmpOids.Databases.General.TotalNumberOfActivePullReplicationAsSinkTasks)
+        {
+        }
+
+        protected override int GetCount(TransactionOperationContext context, RachisState rachisState, string nodeTag, RawDatabaseRecord database)
+        {
+            return GetNumberOfActiveSinkPullReplications(rachisState, nodeTag, database);
+        }
+    }
+}