From 2c0969bac7a3cdd26a0a79b6c683a2956badea9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20L=2E=20Charlier?= Date: Thu, 5 Sep 2024 18:37:03 +0200 Subject: [PATCH] feat: create specific json package and project (#58) --- Streamistry.Core/Streamistry.csproj | 4 +-- .../JsonTests.cs | 17 ++++------ .../ObjectPropertyAppenderTests.cs | 11 +++---- .../RestResponderTests.cs | 4 +-- .../Streamistry.Json.Testing.csproj | 33 +++++++++++++++++++ .../ArrayParser.cs | 6 ++-- .../ArraySplitter.cs | 6 ++-- Streamistry.Json/FodyWeavers.xml | 4 +++ Streamistry.Json/FodyWeavers.xsd | 26 +++++++++++++++ .../ObjectParser.cs | 6 ++-- .../ObjectPropertyAppender.cs | 6 ++-- .../PathPlucker.cs | 6 ++-- .../RestResponder.cs | 2 +- Streamistry.Json/Streamistry.Json.csproj | 33 +++++++++++++++++++ .../ValueMapper.cs | 6 ++-- Streamistry.sln | 19 ++++++++++- appveyor.yml | 6 +++- 17 files changed, 154 insertions(+), 41 deletions(-) rename {Streamistry.Testing => Streamistry.Json.Testing}/JsonTests.cs (82%) rename Streamistry.Testing/JsonObjectPropertyAppenderTests.cs => Streamistry.Json.Testing/ObjectPropertyAppenderTests.cs (69%) rename {Streamistry.Testing => Streamistry.Json.Testing}/RestResponderTests.cs (92%) create mode 100644 Streamistry.Json.Testing/Streamistry.Json.Testing.csproj rename Streamistry.Core/Pipes/Parsers/JsonArrayParser.cs => Streamistry.Json/ArrayParser.cs (78%) rename Streamistry.Core/Pipes/Splitters/JsonArraySplitter.cs => Streamistry.Json/ArraySplitter.cs (74%) create mode 100644 Streamistry.Json/FodyWeavers.xml create mode 100644 Streamistry.Json/FodyWeavers.xsd rename Streamistry.Core/Pipes/Parsers/JsonObjectParser.cs => Streamistry.Json/ObjectParser.cs (80%) rename Streamistry.Core/JsonObjectPropertyAppender.cs => Streamistry.Json/ObjectPropertyAppender.cs (89%) rename Streamistry.Core/Pipes/Mappers/JsonPathPlucker.cs => Streamistry.Json/PathPlucker.cs (83%) rename {Streamistry.Core/Pipes => Streamistry.Json}/RestResponder.cs (98%) create mode 100644 Streamistry.Json/Streamistry.Json.csproj rename Streamistry.Core/JsonValueMapper.cs => Streamistry.Json/ValueMapper.cs (60%) diff --git a/Streamistry.Core/Streamistry.csproj b/Streamistry.Core/Streamistry.csproj index 1566447..deb5712 100644 --- a/Streamistry.Core/Streamistry.csproj +++ b/Streamistry.Core/Streamistry.csproj @@ -3,8 +3,8 @@ Streamistry https://github.com/Seddryck/Streamistry - stream;analytics - Streamistry is a lightweight library offering streaming support with features like accumulators, windows and sinks. This allows you to process and manage continuous data streams efficiently, breaking them into time-based or count-based windows for real-time analysis. + etl;data-engineering;data-integration;data-pipeline;streaming-data + Streamistry is a lightweight library designed to support pipeline, streaming, and ETL development for data engineering and integration. Its versatility makes it an excellent tool for building robust, scalable data workflows and optimizing data processing tasks. With features such as accumulators, windows, and sinks, Streamistry efficiently handles continuous data streams, enabling real-time analysis by breaking streams into time-based or count-based windows. diff --git a/Streamistry.Testing/JsonTests.cs b/Streamistry.Json.Testing/JsonTests.cs similarity index 82% rename from Streamistry.Testing/JsonTests.cs rename to Streamistry.Json.Testing/JsonTests.cs index c766339..e061f3a 100644 --- a/Streamistry.Testing/JsonTests.cs +++ b/Streamistry.Json.Testing/JsonTests.cs @@ -7,12 +7,9 @@ using NUnit.Framework; using Streamistry.Pipes.Sinks; using Streamistry.Pipes.Sources; -using Streamistry.Pipes.Mappers; using System.Text.Json.Nodes; -using Streamistry.Pipes.Parsers; -using Streamistry.Pipes.Splitters; -namespace Streamistry.Testing; +namespace Streamistry.Json.Testing; public class JsonTests { public const string JsonFirst = @" @@ -56,7 +53,7 @@ public class JsonTests public void JsonPathPlucker_ValidPath_ExistingValue(string jsonString, string? email) { var pipeline = new Pipeline(); - var plucker = new JsonPathPlucker(pipeline, "$.user.contact.email"); + var plucker = new PathPlucker(pipeline, "$.user.contact.email"); var sink = new MemorySink(plucker); plucker.Emit((JsonObject)JsonNode.Parse(jsonString)!); @@ -69,8 +66,8 @@ public void JsonObjectParser_ValidString_ExistingValue() { var source = new EnumerableSource([JsonFirst, JsonSecond, JsonThird]); var pipeline = new Pipeline(source); - var parser = new JsonObjectParser(source); - var plucker = new JsonPathPlucker(parser, "$.user.contact.email"); + var parser = new ObjectParser(source); + var plucker = new PathPlucker(parser, "$.user.contact.email"); var sink = new MemorySink(plucker); pipeline.Start(); @@ -85,9 +82,9 @@ public void JsonArrayParser_ValidString_ExistingValue() var array = $"[{JsonFirst}, {JsonSecond}, {JsonThird}]"; var source = new EnumerableSource([array]); var pipeline = new Pipeline(source); - var parser = new JsonArrayParser(source); - var splitter = new JsonArraySplitter(parser); - var plucker = new JsonPathPlucker(splitter, "$.user.contact.email"); + var parser = new ArrayParser(source); + var splitter = new ArraySplitter(parser); + var plucker = new PathPlucker(splitter, "$.user.contact.email"); var sink = new MemorySink(plucker); pipeline.Start(); diff --git a/Streamistry.Testing/JsonObjectPropertyAppenderTests.cs b/Streamistry.Json.Testing/ObjectPropertyAppenderTests.cs similarity index 69% rename from Streamistry.Testing/JsonObjectPropertyAppenderTests.cs rename to Streamistry.Json.Testing/ObjectPropertyAppenderTests.cs index 27a4e1e..7385154 100644 --- a/Streamistry.Testing/JsonObjectPropertyAppenderTests.cs +++ b/Streamistry.Json.Testing/ObjectPropertyAppenderTests.cs @@ -5,12 +5,11 @@ using System.Text.Json.Nodes; using System.Threading.Tasks; using NUnit.Framework; -using Streamistry.Pipes.Parsers; using Streamistry.Pipes.Sinks; using Streamistry.Pipes.Sources; -namespace Streamistry.Testing; -public class JsonObjectPropertyAppenderTests +namespace Streamistry.Json.Testing; +public class ObjectPropertyAppenderTests { [Test] public void X() @@ -19,9 +18,9 @@ public void X() JsonTests.JsonFirst, JsonTests.JsonSecond, JsonTests.JsonThird]); var birthdates = new EnumerableSource([new DateOnly(1879, 3, 14), new DateOnly(1856, 7, 10), new DateOnly(1903, 12, 28)]); var pipeline = new Pipeline([persons, birthdates]); - var personObject = new JsonObjectParser(persons); - var birthdateValue = new JsonValueMapper(birthdates, date => date.ToString("yyyy-MM-dd")); - var appender = new JsonObjectPropertyAppender(personObject, birthdateValue, "$.user.birthdate"); + var personObject = new ObjectParser(persons); + var birthdateValue = new ValueMapper(birthdates, date => date.ToString("yyyy-MM-dd")); + var appender = new ObjectPropertyAppender(personObject, birthdateValue, "$.user.birthdate"); var sink = new MemorySink(appender); pipeline.Start(); diff --git a/Streamistry.Testing/RestResponderTests.cs b/Streamistry.Json.Testing/RestResponderTests.cs similarity index 92% rename from Streamistry.Testing/RestResponderTests.cs rename to Streamistry.Json.Testing/RestResponderTests.cs index 6d7bb0e..cc3165f 100644 --- a/Streamistry.Testing/RestResponderTests.cs +++ b/Streamistry.Json.Testing/RestResponderTests.cs @@ -10,7 +10,7 @@ using Streamistry.Pipes.Mappers; using Streamistry.Pipes.Sinks; -namespace Streamistry.Testing; +namespace Streamistry.Json.Testing; public class RestResponderTests { [Test] @@ -25,7 +25,7 @@ public void Emit_ValidData_CallHttpClient() var pipeline = new Pipeline(); var mapper = new RestResponder(pipeline, client, x => $"/customer/{x}"); - var plunker = new JsonPathPlucker(mapper, "$.user.name"); + var plunker = new PathPlucker(mapper, "$.user.name"); var sink = new MemorySink(plunker); pipeline.Emit(1); diff --git a/Streamistry.Json.Testing/Streamistry.Json.Testing.csproj b/Streamistry.Json.Testing/Streamistry.Json.Testing.csproj new file mode 100644 index 0000000..cfb9f03 --- /dev/null +++ b/Streamistry.Json.Testing/Streamistry.Json.Testing.csproj @@ -0,0 +1,33 @@ + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers + + + \ No newline at end of file diff --git a/Streamistry.Core/Pipes/Parsers/JsonArrayParser.cs b/Streamistry.Json/ArrayParser.cs similarity index 78% rename from Streamistry.Core/Pipes/Parsers/JsonArrayParser.cs rename to Streamistry.Json/ArrayParser.cs index 93f2366..38e3968 100644 --- a/Streamistry.Core/Pipes/Parsers/JsonArrayParser.cs +++ b/Streamistry.Json/ArrayParser.cs @@ -6,10 +6,10 @@ using System.Text.Json.Nodes; using System.Threading.Tasks; -namespace Streamistry.Pipes.Parsers; -public class JsonArrayParser : StringParser +namespace Streamistry.Json; +public class ArrayParser : StringParser { - public JsonArrayParser(IChainablePipe upstream) + public ArrayParser(IChainablePipe upstream) : base(upstream, TryParse) { } diff --git a/Streamistry.Core/Pipes/Splitters/JsonArraySplitter.cs b/Streamistry.Json/ArraySplitter.cs similarity index 74% rename from Streamistry.Core/Pipes/Splitters/JsonArraySplitter.cs rename to Streamistry.Json/ArraySplitter.cs index 69b8149..9da7ab6 100644 --- a/Streamistry.Core/Pipes/Splitters/JsonArraySplitter.cs +++ b/Streamistry.Json/ArraySplitter.cs @@ -6,10 +6,10 @@ using System.Text.Json.Nodes; using System.Threading.Tasks; -namespace Streamistry.Pipes.Splitters; -internal class JsonArraySplitter : Splitter +namespace Streamistry.Json; +internal class ArraySplitter : Splitter { - public JsonArraySplitter(IChainablePipe upstream) + public ArraySplitter(IChainablePipe upstream) : base(upstream, x => [..Split(x)]) { } diff --git a/Streamistry.Json/FodyWeavers.xml b/Streamistry.Json/FodyWeavers.xml new file mode 100644 index 0000000..f217bdf --- /dev/null +++ b/Streamistry.Json/FodyWeavers.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/Streamistry.Json/FodyWeavers.xsd b/Streamistry.Json/FodyWeavers.xsd new file mode 100644 index 0000000..f421e79 --- /dev/null +++ b/Streamistry.Json/FodyWeavers.xsd @@ -0,0 +1,26 @@ + + + + + + + + + + + 'true' to run assembly verification (PEVerify) on the target assembly after all weavers have been executed. + + + + + A comma-separated list of error codes that can be safely ignored in assembly verification. + + + + + 'false' to turn off automatic generation of the XML Schema file. + + + + + \ No newline at end of file diff --git a/Streamistry.Core/Pipes/Parsers/JsonObjectParser.cs b/Streamistry.Json/ObjectParser.cs similarity index 80% rename from Streamistry.Core/Pipes/Parsers/JsonObjectParser.cs rename to Streamistry.Json/ObjectParser.cs index 53dffb4..c00dfe0 100644 --- a/Streamistry.Core/Pipes/Parsers/JsonObjectParser.cs +++ b/Streamistry.Json/ObjectParser.cs @@ -7,10 +7,10 @@ using System.Text.Json.Nodes; using System.Threading.Tasks; -namespace Streamistry.Pipes.Parsers; -public class JsonObjectParser : StringParser +namespace Streamistry.Json; +public class ObjectParser : StringParser { - public JsonObjectParser(IChainablePipe upstream) + public ObjectParser(IChainablePipe upstream) : base(upstream, new ParserDelegate(TryParse)) { } diff --git a/Streamistry.Core/JsonObjectPropertyAppender.cs b/Streamistry.Json/ObjectPropertyAppender.cs similarity index 89% rename from Streamistry.Core/JsonObjectPropertyAppender.cs rename to Streamistry.Json/ObjectPropertyAppender.cs index b81c686..9a8136c 100644 --- a/Streamistry.Core/JsonObjectPropertyAppender.cs +++ b/Streamistry.Json/ObjectPropertyAppender.cs @@ -9,13 +9,13 @@ using Json.Path; -namespace Streamistry; -public class JsonObjectPropertyAppender : Zipper +namespace Streamistry.Json; +public class ObjectPropertyAppender : Zipper where TInputMain : JsonNode where TInputSecondary : JsonNode { - public JsonObjectPropertyAppender(IChainablePort mainUpstream, IChainablePort secondUpstream, string path) + public ObjectPropertyAppender(IChainablePort mainUpstream, IChainablePort secondUpstream, string path) : base(mainUpstream, secondUpstream, (x, y) => AppendProperty(x, y, path)) { } diff --git a/Streamistry.Core/Pipes/Mappers/JsonPathPlucker.cs b/Streamistry.Json/PathPlucker.cs similarity index 83% rename from Streamistry.Core/Pipes/Mappers/JsonPathPlucker.cs rename to Streamistry.Json/PathPlucker.cs index 793656b..ddb7e69 100644 --- a/Streamistry.Core/Pipes/Mappers/JsonPathPlucker.cs +++ b/Streamistry.Json/PathPlucker.cs @@ -6,7 +6,7 @@ using System.Threading.Tasks; using Json.Path; -namespace Streamistry.Pipes.Mappers; +namespace Streamistry.Json; public abstract class BaseJsonPathPlucker : Mapper where TJson : JsonNode { public BaseJsonPathPlucker(IChainablePipe upstream, string path) @@ -24,9 +24,9 @@ public BaseJsonPathPlucker(IChainablePipe upstream, string path) } } -public class JsonPathPlucker : BaseJsonPathPlucker +public class PathPlucker : BaseJsonPathPlucker { - public JsonPathPlucker(IChainablePipe upstream, string path) + public PathPlucker(IChainablePipe upstream, string path) : base(upstream, path) { } } diff --git a/Streamistry.Core/Pipes/RestResponder.cs b/Streamistry.Json/RestResponder.cs similarity index 98% rename from Streamistry.Core/Pipes/RestResponder.cs rename to Streamistry.Json/RestResponder.cs index 9e773d2..01e473c 100644 --- a/Streamistry.Core/Pipes/RestResponder.cs +++ b/Streamistry.Json/RestResponder.cs @@ -9,7 +9,7 @@ using Streamistry.Observability; using Streamistry.Pipes.Parsers; -namespace Streamistry.Pipes; +namespace Streamistry.Json; public class RestResponder : EscapeRouterPipe, IProcessablePipe where TOutput : JsonNode { protected HttpClient Client { get; } diff --git a/Streamistry.Json/Streamistry.Json.csproj b/Streamistry.Json/Streamistry.Json.csproj new file mode 100644 index 0000000..b35d800 --- /dev/null +++ b/Streamistry.Json/Streamistry.Json.csproj @@ -0,0 +1,33 @@ + + + + Streamistry.Json + https://github.com/Seddryck/Streamistry + etl;data-engineering;data-integration;data-pipeline;streaming-data;json + The JSON package of Streamistry is specifically designed to handle JSON data within pipeline, streaming, and ETL processes. Its powerful and flexible features make it an essential tool for managing complex JSON structures, facilitating seamless data parsing, transformation, and integration. + + + + PackageReference + true + + + + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers + + + diff --git a/Streamistry.Core/JsonValueMapper.cs b/Streamistry.Json/ValueMapper.cs similarity index 60% rename from Streamistry.Core/JsonValueMapper.cs rename to Streamistry.Json/ValueMapper.cs index f2afc92..0dfd88f 100644 --- a/Streamistry.Core/JsonValueMapper.cs +++ b/Streamistry.Json/ValueMapper.cs @@ -5,10 +5,10 @@ using System.Text.Json.Nodes; using System.Threading.Tasks; -namespace Streamistry; -public class JsonValueMapper : Mapper +namespace Streamistry.Json; +public class ValueMapper : Mapper { - public JsonValueMapper(IChainablePort upstream, Func? toString = null) + public ValueMapper(IChainablePort upstream, Func? toString = null) : base(upstream, value => toString is null ? JsonValue.Create(value) : JsonValue.Create(toString.Invoke(value))) { } diff --git a/Streamistry.sln b/Streamistry.sln index f6e0ec4..87839df 100644 --- a/Streamistry.sln +++ b/Streamistry.sln @@ -1,4 +1,5 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 + +Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.1.32328.378 MinimumVisualStudioVersion = 10.0.40219.1 @@ -34,6 +35,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Edit", "Edit", "{210C887C-6 .gitignore = .gitignore EndProjectSection EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Json", "Json", "{3AD64C0C-6DA9-4947-A225-6F68CF9A6A4D}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Streamistry.Json", "Streamistry.Json\Streamistry.Json.csproj", "{AFC0A591-FD48-481F-AB90-00E7B0926FE1}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Streamistry.Json.Testing", "Streamistry.Json.Testing\Streamistry.Json.Testing.csproj", "{84FA815D-FE05-4B57-9BB1-156E7B217576}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -48,6 +55,14 @@ Global {A0CDC412-D16C-4508-96FC-6F0BB383E962}.Debug|Any CPU.Build.0 = Debug|Any CPU {A0CDC412-D16C-4508-96FC-6F0BB383E962}.Release|Any CPU.ActiveCfg = Release|Any CPU {A0CDC412-D16C-4508-96FC-6F0BB383E962}.Release|Any CPU.Build.0 = Release|Any CPU + {AFC0A591-FD48-481F-AB90-00E7B0926FE1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AFC0A591-FD48-481F-AB90-00E7B0926FE1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AFC0A591-FD48-481F-AB90-00E7B0926FE1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AFC0A591-FD48-481F-AB90-00E7B0926FE1}.Release|Any CPU.Build.0 = Release|Any CPU + {84FA815D-FE05-4B57-9BB1-156E7B217576}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {84FA815D-FE05-4B57-9BB1-156E7B217576}.Debug|Any CPU.Build.0 = Debug|Any CPU + {84FA815D-FE05-4B57-9BB1-156E7B217576}.Release|Any CPU.ActiveCfg = Release|Any CPU + {84FA815D-FE05-4B57-9BB1-156E7B217576}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -56,6 +71,8 @@ Global {95E83BA6-B535-4D56-B06A-D00249A4EF00} = {C923A152-4091-48BE-A984-686FF5E4ACCC} {09BDFA96-353F-419D-890A-D9B9045E66DD} = {C923A152-4091-48BE-A984-686FF5E4ACCC} {210C887C-6FF3-4D1B-9A7B-894E2F286FF4} = {C923A152-4091-48BE-A984-686FF5E4ACCC} + {AFC0A591-FD48-481F-AB90-00E7B0926FE1} = {3AD64C0C-6DA9-4947-A225-6F68CF9A6A4D} + {84FA815D-FE05-4B57-9BB1-156E7B217576} = {3AD64C0C-6DA9-4947-A225-6F68CF9A6A4D} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {8EABD9FE-559A-40AF-BB4B-8026F9240FEF} diff --git a/appveyor.yml b/appveyor.yml index 67b8566..c010cd2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,6 +6,7 @@ branches: - /v\d+\.\d+\.\d+/ skip_commits: files: + - .github/ - docs/ - misc/ - README.md @@ -53,6 +54,8 @@ test_script: $ErrorActionPreference = "Stop" dotnet test Streamistry.Testing -c Release /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Threshold=10 /p:ThresholdType=line /p:CoverletOutput=../.coverage/coverage.Streamistry.xml --test-adapter-path:. --logger:Appveyor --no-build --nologo $globalTestResult = $LastExitCode + dotnet test Streamistry.Json.Testing -c Release /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Threshold=10 /p:ThresholdType=line /p:CoverletOutput=../.coverage/coverage.Streamistry.Json.xml --test-adapter-path:. --logger:Appveyor --no-build --nologo + $globalTestResult = $LastExitCode if($globalTestResult -ne 0) { $host.SetShouldExit($globalTestResult) } - pwsh: | @@ -61,7 +64,8 @@ test_script: .\codecov.exe --dir "./.coverage/" after_test: -- dotnet pack Streamistry.Core -p:version="%GitVersion_SemVer%" -c Release --include-symbols --no-build --nologo +- dotnet pack Streamistry.Core -p:version="%GitVersion_SemVer%" -c Release --include-symbols --no-build --nologo +- dotnet pack Streamistry.Json -p:version="%GitVersion_SemVer%" -c Release --include-symbols --no-build --nologo artifacts: - path: '**\*.nupkg'