Skip to content

Commit

Permalink
Implement Add/Remove super stream feature (#357)
Browse files Browse the repository at this point in the history
* Implement Add/Remove super stream feature
* add PartitionsSuperStreamSpec to configure the Partitions 
* add BindingsSuperStreamSpec to configure the Bindings

---------
Signed-off-by: Gabriele Santomaggio <[email protected]>
  • Loading branch information
Gsantomaggio authored Feb 13, 2024
1 parent 335a0f3 commit ac23f56
Show file tree
Hide file tree
Showing 22 changed files with 585 additions and 86 deletions.
2 changes: 1 addition & 1 deletion .ci/versions.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{
"erlang": "26.1.2",
"rabbitmq": "3.13.0-rc.2"
"rabbitmq": "3.13.0-rc.4"
}
5 changes: 4 additions & 1 deletion Examples/Performances/BatchVsBatchSend.cs
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,10 @@ private static async Task<string> RecreateStream(StreamSystem system, string str

Thread.Sleep(5000);

await system.CreateStream(new StreamSpec(stream) { });
await system.CreateStream(new StreamSpec(stream)
{

});
Thread.Sleep(1000);
Console.WriteLine($"Stream: {stream} created");
return stream;
Expand Down
7 changes: 4 additions & 3 deletions RabbitMQ.Stream.Client/AvailableFeatures.cs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ internal class AvailableFeatures
public bool PublishFilter { get; private set; }

public bool Is311OrMore { get; private set; }
public bool Is313OrMore { get; private set; }

public string BrokerVersion { get; private set; }

Expand All @@ -31,9 +32,9 @@ private static string ExtractVersion(string fullVersion)

public void SetServerVersion(string brokerVersion)
{
var v = ExtractVersion(brokerVersion);
BrokerVersion = v;
Is311OrMore = new System.Version(v) >= new System.Version("3.11.0");
BrokerVersion = ExtractVersion(brokerVersion);
Is311OrMore = new System.Version(BrokerVersion) >= new System.Version("3.11.0");
Is313OrMore = new System.Version(BrokerVersion) >= new System.Version("3.13.0");
}

public void ParseCommandVersions(List<ICommandVersions> commands)
Expand Down
24 changes: 23 additions & 1 deletion RabbitMQ.Stream.Client/Client.cs
Original file line number Diff line number Diff line change
Expand Up @@ -724,6 +724,14 @@ private void HandleCorrelatedCommand(ushort tag, ref ReadOnlySequence<byte> fram
CommandVersionsResponse.Read(frame, out var commandVersionsResponse);
HandleCorrelatedResponse(commandVersionsResponse);
break;
case CreateSuperStreamResponse.Key:
CreateSuperStreamResponse.Read(frame, out var superStreamResponse);
HandleCorrelatedResponse(superStreamResponse);
break;
case DeleteSuperStreamResponse.Key:
DeleteSuperStreamResponse.Read(frame, out var deleteSuperStreamResponse);
HandleCorrelatedResponse(deleteSuperStreamResponse);
break;
default:
if (MemoryMarshal.TryGetArray(frame.First, out var segment))
{
Expand Down Expand Up @@ -863,7 +871,6 @@ public async Task<bool> StreamExists(string stream)
if (response.StreamInfos is { Count: >= 1 } &&
response.StreamInfos[stream].ResponseCode == ResponseCode.StreamNotAvailable)
{

ClientExceptions.MaybeThrowException(ResponseCode.StreamNotAvailable, stream);
}

Expand All @@ -889,6 +896,21 @@ public async ValueTask<DeleteResponse> DeleteStream(string stream)
.ConfigureAwait(false);
}

public async ValueTask<CreateSuperStreamResponse> CreateSuperStream(string superStream, List<string> partitions,
List<string> bindingKeys, IDictionary<string, string> args)
{
return await Request<CreateSuperStreamRequest, CreateSuperStreamResponse>(corr =>
new CreateSuperStreamRequest(corr, superStream, partitions, bindingKeys, args))
.ConfigureAwait(false);
}

public async ValueTask<DeleteSuperStreamResponse> DeleteSuperStream(string superStream)
{
return await Request<DeleteSuperStreamRequest, DeleteSuperStreamResponse>(corr =>
new DeleteSuperStreamRequest(corr, superStream))
.ConfigureAwait(false);
}

public async ValueTask<bool> Credit(byte subscriptionId, ushort credit)
{
return await Publish(new CreditRequest(subscriptionId, credit)).ConfigureAwait(false);
Expand Down
3 changes: 3 additions & 0 deletions RabbitMQ.Stream.Client/Consts.cs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ internal static class Consts
internal const string FilterNotSupported = "Filtering is not supported by the broker "
+ "(requires RabbitMQ 3.13+ and stream_filtering feature flag activated)";

internal const string SuperStreamCreationNotSupported = "SuperStreams creation / deleting not supported by the broker "
+ "(requires RabbitMQ 3.13+. It is possible to use the command line tool to create superstreams)";

internal static int RandomShort()
{
return Random.Shared.Next(500, 1500);
Expand Down
111 changes: 111 additions & 0 deletions RabbitMQ.Stream.Client/CreateSuperStream.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
// This source code is dual-licensed under the Apache License, version
// 2.0, and the Mozilla Public License, version 2.0.
// Copyright (c) 2017-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries.

using System;
using System.Buffers;
using System.Collections.Generic;
using System.Linq;

namespace RabbitMQ.Stream.Client;

internal readonly struct CreateSuperStreamRequest : ICommand
{
private const ushort Key = 29;
private readonly string _superStream;
private readonly uint _corrId;
private readonly IDictionary<string, string> _arguments;
private readonly List<string> _partitions;
private readonly List<string> _bindingKeys;

internal CreateSuperStreamRequest(uint corrId, string superStream,
List<string> partitions, List<string> bindingKeys, IDictionary<string, string> args)
{
_corrId = corrId;
_superStream = superStream;
_partitions = partitions;
_bindingKeys = bindingKeys;
_arguments = args;
}

public int SizeNeeded
{
get
{
var size =
2
+ 2
+ 4
+ WireFormatting.StringSize(_superStream);

size += 4 + _partitions.Sum(WireFormatting.StringSize);
size += 4 + _bindingKeys.Sum(WireFormatting.StringSize);
size += 4
+ _arguments.Sum(x => WireFormatting.StringSize(x.Key) + WireFormatting.StringSize(x.Value));
return size;
}
}

public int Write(Span<byte> span)
{
var command = (ICommand)this;
var offset = WireFormatting.WriteUInt16(span, Key);
offset += WireFormatting.WriteUInt16(span[offset..], command.Version);
offset += WireFormatting.WriteUInt32(span[offset..], _corrId);
offset += WireFormatting.WriteString(span[offset..], _superStream);
offset += WireFormatting.WriteInt32(span[offset..], _partitions.Count);
foreach (var partition in _partitions)
{
offset += WireFormatting.WriteString(span[offset..], partition);
}

offset += WireFormatting.WriteInt32(span[offset..], _bindingKeys.Count);
foreach (var bindingKey in _bindingKeys)
{
offset += WireFormatting.WriteString(span[offset..], bindingKey);
}

offset += WireFormatting.WriteInt32(span[offset..], _arguments.Count);
foreach (var (key, value) in _arguments)
{
offset += WireFormatting.WriteString(span[offset..], key);
offset += WireFormatting.WriteString(span[offset..], value);
}

return offset;
}
}

public readonly struct CreateSuperStreamResponse : ICommand
{
public const ushort Key = 29;
private readonly uint _correlationId;
private readonly ushort _responseCode;

private CreateSuperStreamResponse(uint correlationId, ushort responseCode)
{
_correlationId = correlationId;
_responseCode = responseCode;
}

public int SizeNeeded => throw new NotImplementedException();

public uint CorrelationId => _correlationId;

public ResponseCode ResponseCode => (ResponseCode)_responseCode;

public int Write(Span<byte> span)
{
throw new NotImplementedException();
}

internal static int Read(ReadOnlySequence<byte> frame, out CreateSuperStreamResponse command)
{
var offset = WireFormatting.ReadUInt16(frame, out _);
offset += WireFormatting.ReadUInt16(frame.Slice(offset), out _);
offset += WireFormatting.ReadUInt32(frame.Slice(offset), out var correlation);
offset += WireFormatting.ReadUInt16(frame.Slice(offset), out var responseCode);
command = new CreateSuperStreamResponse(correlation, responseCode);
return offset;
}
}
66 changes: 66 additions & 0 deletions RabbitMQ.Stream.Client/DeleteSuperStream.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
// This source code is dual-licensed under the Apache License, version
// 2.0, and the Mozilla Public License, version 2.0.
// Copyright (c) 2017-2023 Broadcom. All Rights Reserved. The term "Broadcom" refers to Broadcom Inc. and/or its subsidiaries.

using System;
using System.Buffers;

namespace RabbitMQ.Stream.Client;

internal readonly struct DeleteSuperStreamRequest : ICommand
{
private readonly uint _correlationId;
private readonly string _superStream;
private const ushort Key = 30;

public DeleteSuperStreamRequest(uint correlationId, string superStream)
{
_correlationId = correlationId;
_superStream = superStream;
}

public int SizeNeeded => 8 + WireFormatting.StringSize(_superStream);

public int Write(Span<byte> span)
{
var offset = WireFormatting.WriteUInt16(span, Key);
offset += WireFormatting.WriteUInt16(span[offset..], ((ICommand)this).Version);
offset += WireFormatting.WriteUInt32(span[offset..], _correlationId);
offset += WireFormatting.WriteString(span[offset..], _superStream);
return offset;
}
}

public readonly struct DeleteSuperStreamResponse : ICommand
{
public const ushort Key = 30;
private readonly uint _correlationId;
private readonly ushort _responseCode;

public DeleteSuperStreamResponse(uint correlationId, ushort responseCode)
{
_correlationId = correlationId;
_responseCode = responseCode;
}

public int SizeNeeded => throw new NotImplementedException();

public uint CorrelationId => _correlationId;

public ResponseCode ResponseCode => (ResponseCode)_responseCode;

public int Write(Span<byte> span)
{
throw new NotImplementedException();
}

internal static int Read(ReadOnlySequence<byte> frame, out DeleteSuperStreamResponse command)
{
var offset = WireFormatting.ReadUInt16(frame, out _);
offset += WireFormatting.ReadUInt16(frame.Slice(offset), out _);
offset += WireFormatting.ReadUInt32(frame.Slice(offset), out var correlation);
offset += WireFormatting.ReadUInt16(frame.Slice(offset), out var responseCode);
command = new DeleteSuperStreamResponse(correlation, responseCode);
return offset;
}
}
34 changes: 34 additions & 0 deletions RabbitMQ.Stream.Client/PublicAPI.Unshipped.txt
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ abstract RabbitMQ.Stream.Client.AbstractEntity.DeleteEntityFromTheServer(bool ig
abstract RabbitMQ.Stream.Client.AbstractEntity.DumpEntityConfiguration() -> string
abstract RabbitMQ.Stream.Client.AbstractEntity.GetStream() -> string
abstract RabbitMQ.Stream.Client.Reliable.ReliableBase.CreateNewEntity(bool boot) -> System.Threading.Tasks.Task
const RabbitMQ.Stream.Client.CreateSuperStreamResponse.Key = 29 -> ushort
const RabbitMQ.Stream.Client.DeleteSuperStreamResponse.Key = 30 -> ushort
const RabbitMQ.Stream.Client.RouteQueryResponse.Key = 24 -> ushort
const RabbitMQ.Stream.Client.StreamStatsResponse.Key = 28 -> ushort
override RabbitMQ.Stream.Client.Broker.ToString() -> string
Expand All @@ -27,14 +29,19 @@ RabbitMQ.Stream.Client.AuthMechanism.External = 1 -> RabbitMQ.Stream.Client.Auth
RabbitMQ.Stream.Client.AuthMechanism.Plain = 0 -> RabbitMQ.Stream.Client.AuthMechanism
RabbitMQ.Stream.Client.AuthMechanismNotSupportedException
RabbitMQ.Stream.Client.AuthMechanismNotSupportedException.AuthMechanismNotSupportedException(string s) -> void
RabbitMQ.Stream.Client.BindingsSuperStreamSpec
RabbitMQ.Stream.Client.BindingsSuperStreamSpec.BindingKeys.get -> string[]
RabbitMQ.Stream.Client.BindingsSuperStreamSpec.BindingsSuperStreamSpec(string Name, string[] bindingKeys) -> void
RabbitMQ.Stream.Client.Chunk.Crc.get -> uint
RabbitMQ.Stream.Client.Chunk.Data.get -> System.ReadOnlyMemory<byte>
RabbitMQ.Stream.Client.Chunk.MagicVersion.get -> byte
RabbitMQ.Stream.Client.Client.ClientId.get -> string
RabbitMQ.Stream.Client.Client.ClientId.init -> void
RabbitMQ.Stream.Client.Client.Consumers.get -> System.Collections.Generic.IDictionary<byte, (string, RabbitMQ.Stream.Client.ConsumerEvents)>
RabbitMQ.Stream.Client.Client.CreateSuperStream(string superStream, System.Collections.Generic.List<string> partitions, System.Collections.Generic.List<string> bindingKeys, System.Collections.Generic.IDictionary<string, string> args) -> System.Threading.Tasks.ValueTask<RabbitMQ.Stream.Client.CreateSuperStreamResponse>
RabbitMQ.Stream.Client.Client.DeclarePublisher(string publisherRef, string stream, System.Action<System.ReadOnlyMemory<ulong>> confirmCallback, System.Action<(ulong, RabbitMQ.Stream.Client.ResponseCode)[]> errorCallback, RabbitMQ.Stream.Client.ConnectionsPool pool = null) -> System.Threading.Tasks.Task<(byte, RabbitMQ.Stream.Client.DeclarePublisherResponse)>
RabbitMQ.Stream.Client.Client.DeletePublisher(byte publisherId, bool ignoreIfAlreadyRemoved = false) -> System.Threading.Tasks.Task<RabbitMQ.Stream.Client.DeletePublisherResponse>
RabbitMQ.Stream.Client.Client.DeleteSuperStream(string superStream) -> System.Threading.Tasks.ValueTask<RabbitMQ.Stream.Client.DeleteSuperStreamResponse>
RabbitMQ.Stream.Client.Client.ExchangeVersions() -> System.Threading.Tasks.Task<RabbitMQ.Stream.Client.CommandVersionsResponse>
RabbitMQ.Stream.Client.Client.Publishers.get -> System.Collections.Generic.IDictionary<byte, (string, (System.Action<System.ReadOnlyMemory<ulong>>, System.Action<(ulong, RabbitMQ.Stream.Client.ResponseCode)[]>))>
RabbitMQ.Stream.Client.Client.QueryRoute(string superStream, string routingKey) -> System.Threading.Tasks.Task<RabbitMQ.Stream.Client.RouteQueryResponse>
Expand Down Expand Up @@ -94,6 +101,19 @@ RabbitMQ.Stream.Client.CreateException.CreateException(string s, RabbitMQ.Stream
RabbitMQ.Stream.Client.CreateException.ResponseCode.get -> RabbitMQ.Stream.Client.ResponseCode
RabbitMQ.Stream.Client.CreateException.ResponseCode.init -> void
RabbitMQ.Stream.Client.CreateProducerException.CreateProducerException(string s, RabbitMQ.Stream.Client.ResponseCode responseCode) -> void
RabbitMQ.Stream.Client.CreateSuperStreamResponse
RabbitMQ.Stream.Client.CreateSuperStreamResponse.CorrelationId.get -> uint
RabbitMQ.Stream.Client.CreateSuperStreamResponse.CreateSuperStreamResponse() -> void
RabbitMQ.Stream.Client.CreateSuperStreamResponse.ResponseCode.get -> RabbitMQ.Stream.Client.ResponseCode
RabbitMQ.Stream.Client.CreateSuperStreamResponse.SizeNeeded.get -> int
RabbitMQ.Stream.Client.CreateSuperStreamResponse.Write(System.Span<byte> span) -> int
RabbitMQ.Stream.Client.DeleteSuperStreamResponse
RabbitMQ.Stream.Client.DeleteSuperStreamResponse.CorrelationId.get -> uint
RabbitMQ.Stream.Client.DeleteSuperStreamResponse.DeleteSuperStreamResponse() -> void
RabbitMQ.Stream.Client.DeleteSuperStreamResponse.DeleteSuperStreamResponse(uint correlationId, ushort responseCode) -> void
RabbitMQ.Stream.Client.DeleteSuperStreamResponse.ResponseCode.get -> RabbitMQ.Stream.Client.ResponseCode
RabbitMQ.Stream.Client.DeleteSuperStreamResponse.SizeNeeded.get -> int
RabbitMQ.Stream.Client.DeleteSuperStreamResponse.Write(System.Span<byte> span) -> int
RabbitMQ.Stream.Client.EntityCommonConfig
RabbitMQ.Stream.Client.EntityCommonConfig.Identifier.get -> string
RabbitMQ.Stream.Client.EntityCommonConfig.Identifier.set -> void
Expand Down Expand Up @@ -160,6 +180,9 @@ RabbitMQ.Stream.Client.MessageContext.ChunkMessagesCount.get -> uint
RabbitMQ.Stream.Client.MessageContext.MessageContext(ulong offset, System.TimeSpan timestamp, uint chunkMessagesCount, ulong chunkId) -> void
RabbitMQ.Stream.Client.OffsetTypeTimestamp.OffsetTypeTimestamp(System.DateTime dateTime) -> void
RabbitMQ.Stream.Client.OffsetTypeTimestamp.OffsetTypeTimestamp(System.DateTimeOffset dateTimeOffset) -> void
RabbitMQ.Stream.Client.PartitionsSuperStreamSpec
RabbitMQ.Stream.Client.PartitionsSuperStreamSpec.Partitions.get -> int
RabbitMQ.Stream.Client.PartitionsSuperStreamSpec.PartitionsSuperStreamSpec(string Name, int partitions) -> void
RabbitMQ.Stream.Client.ProducerFilter
RabbitMQ.Stream.Client.ProducerFilter.FilterValue.get -> System.Func<RabbitMQ.Stream.Client.Message, string>
RabbitMQ.Stream.Client.ProducerFilter.FilterValue.set -> void
Expand Down Expand Up @@ -281,14 +304,25 @@ RabbitMQ.Stream.Client.StreamStatsResponse.StreamStatsResponse() -> void
RabbitMQ.Stream.Client.StreamStatsResponse.StreamStatsResponse(uint correlationId, RabbitMQ.Stream.Client.ResponseCode responseCode, System.Collections.Generic.IDictionary<string, long> statistic) -> void
RabbitMQ.Stream.Client.StreamStatsResponse.Write(System.Span<byte> span) -> int
RabbitMQ.Stream.Client.StreamSystem.CreateRawSuperStreamProducer(RabbitMQ.Stream.Client.RawSuperStreamProducerConfig rawSuperStreamProducerConfig, Microsoft.Extensions.Logging.ILogger logger = null) -> System.Threading.Tasks.Task<RabbitMQ.Stream.Client.ISuperStreamProducer>
RabbitMQ.Stream.Client.StreamSystem.CreateSuperStream(RabbitMQ.Stream.Client.SuperStreamSpec spec) -> System.Threading.Tasks.Task
RabbitMQ.Stream.Client.StreamSystem.CreateSuperStreamConsumer(RabbitMQ.Stream.Client.RawSuperStreamConsumerConfig rawSuperStreamConsumerConfig, Microsoft.Extensions.Logging.ILogger logger = null) -> System.Threading.Tasks.Task<RabbitMQ.Stream.Client.ISuperStreamConsumer>
RabbitMQ.Stream.Client.StreamSystem.DeleteSuperStream(string superStream) -> System.Threading.Tasks.Task
RabbitMQ.Stream.Client.StreamSystem.StreamInfo(string streamName) -> System.Threading.Tasks.Task<RabbitMQ.Stream.Client.StreamInfo>
RabbitMQ.Stream.Client.StreamSystem.StreamStats(string stream) -> System.Threading.Tasks.Task<RabbitMQ.Stream.Client.StreamStats>
RabbitMQ.Stream.Client.StreamSystem.UpdateSecret(string newSecret) -> System.Threading.Tasks.Task
RabbitMQ.Stream.Client.StreamSystemConfig.AuthMechanism.get -> RabbitMQ.Stream.Client.AuthMechanism
RabbitMQ.Stream.Client.StreamSystemConfig.AuthMechanism.set -> void
RabbitMQ.Stream.Client.StreamSystemConfig.ConnectionPoolConfig.get -> RabbitMQ.Stream.Client.ConnectionPoolConfig
RabbitMQ.Stream.Client.StreamSystemConfig.ConnectionPoolConfig.set -> void
RabbitMQ.Stream.Client.SuperStreamSpec
RabbitMQ.Stream.Client.SuperStreamSpec.Args.get -> System.Collections.Generic.IDictionary<string, string>
RabbitMQ.Stream.Client.SuperStreamSpec.LeaderLocator.set -> void
RabbitMQ.Stream.Client.SuperStreamSpec.MaxAge.set -> void
RabbitMQ.Stream.Client.SuperStreamSpec.MaxLengthBytes.set -> void
RabbitMQ.Stream.Client.SuperStreamSpec.MaxSegmentSizeBytes.set -> void
RabbitMQ.Stream.Client.SuperStreamSpec.Name.get -> string
RabbitMQ.Stream.Client.SuperStreamSpec.Name.init -> void
RabbitMQ.Stream.Client.SuperStreamSpec.SuperStreamSpec(string Name) -> void
RabbitMQ.Stream.Client.TooManyConnectionsException
RabbitMQ.Stream.Client.TooManyConnectionsException.TooManyConnectionsException(string s) -> void
RabbitMQ.Stream.Client.UnknownCommandException
Expand Down
Loading

0 comments on commit ac23f56

Please sign in to comment.