Skip to content

Commit

Permalink
feat: create specific json package and project (#58)
Browse files Browse the repository at this point in the history
  • Loading branch information
Seddryck authored Sep 5, 2024
1 parent 0962720 commit 2c0969b
Show file tree
Hide file tree
Showing 17 changed files with 154 additions and 41 deletions.
4 changes: 2 additions & 2 deletions Streamistry.Core/Streamistry.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
<PropertyGroup>
<PackageId>Streamistry</PackageId>
<RepositoryUrl>https://github.com/Seddryck/Streamistry</RepositoryUrl>
<PackageTags>stream;analytics</PackageTags>
<Description>Streamistry is a lightweight library offering streaming support with features like accumulators, windows and sinks. This allows you to process and manage continuous data streams efficiently, breaking them into time-based or count-based windows for real-time analysis.</Description>
<PackageTags>etl;data-engineering;data-integration;data-pipeline;streaming-data</PackageTags>
<Description>Streamistry is a lightweight library designed to support pipeline, streaming, and ETL development for data engineering and integration. Its versatility makes it an excellent tool for building robust, scalable data workflows and optimizing data processing tasks. With features such as accumulators, windows, and sinks, Streamistry efficiently handles continuous data streams, enabling real-time analysis by breaking streams into time-based or count-based windows.</Description>
</PropertyGroup>

<PropertyGroup>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,9 @@
using NUnit.Framework;
using Streamistry.Pipes.Sinks;
using Streamistry.Pipes.Sources;
using Streamistry.Pipes.Mappers;
using System.Text.Json.Nodes;
using Streamistry.Pipes.Parsers;
using Streamistry.Pipes.Splitters;

namespace Streamistry.Testing;
namespace Streamistry.Json.Testing;
public class JsonTests
{
public const string JsonFirst = @"
Expand Down Expand Up @@ -56,7 +53,7 @@ public class JsonTests
public void JsonPathPlucker_ValidPath_ExistingValue(string jsonString, string? email)
{
var pipeline = new Pipeline<JsonObject>();
var plucker = new JsonPathPlucker<string>(pipeline, "$.user.contact.email");
var plucker = new PathPlucker<string>(pipeline, "$.user.contact.email");
var sink = new MemorySink<string>(plucker);
plucker.Emit((JsonObject)JsonNode.Parse(jsonString)!);

Expand All @@ -69,8 +66,8 @@ public void JsonObjectParser_ValidString_ExistingValue()
{
var source = new EnumerableSource<string>([JsonFirst, JsonSecond, JsonThird]);
var pipeline = new Pipeline(source);
var parser = new JsonObjectParser(source);
var plucker = new JsonPathPlucker<string>(parser, "$.user.contact.email");
var parser = new ObjectParser(source);
var plucker = new PathPlucker<string>(parser, "$.user.contact.email");
var sink = new MemorySink<string>(plucker);
pipeline.Start();

Expand All @@ -85,9 +82,9 @@ public void JsonArrayParser_ValidString_ExistingValue()
var array = $"[{JsonFirst}, {JsonSecond}, {JsonThird}]";
var source = new EnumerableSource<string>([array]);
var pipeline = new Pipeline(source);
var parser = new JsonArrayParser(source);
var splitter = new JsonArraySplitter(parser);
var plucker = new JsonPathPlucker<string>(splitter, "$.user.contact.email");
var parser = new ArrayParser(source);
var splitter = new ArraySplitter(parser);
var plucker = new PathPlucker<string>(splitter, "$.user.contact.email");
var sink = new MemorySink<string>(plucker);
pipeline.Start();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,11 @@
using System.Text.Json.Nodes;
using System.Threading.Tasks;
using NUnit.Framework;
using Streamistry.Pipes.Parsers;
using Streamistry.Pipes.Sinks;
using Streamistry.Pipes.Sources;

namespace Streamistry.Testing;
public class JsonObjectPropertyAppenderTests
namespace Streamistry.Json.Testing;
public class ObjectPropertyAppenderTests
{
[Test]
public void X()
Expand All @@ -19,9 +18,9 @@ public void X()
JsonTests.JsonFirst, JsonTests.JsonSecond, JsonTests.JsonThird]);
var birthdates = new EnumerableSource<DateOnly>([new DateOnly(1879, 3, 14), new DateOnly(1856, 7, 10), new DateOnly(1903, 12, 28)]);
var pipeline = new Pipeline([persons, birthdates]);
var personObject = new JsonObjectParser(persons);
var birthdateValue = new JsonValueMapper<DateOnly>(birthdates, date => date.ToString("yyyy-MM-dd"));
var appender = new JsonObjectPropertyAppender<JsonObject, JsonValue>(personObject, birthdateValue, "$.user.birthdate");
var personObject = new ObjectParser(persons);
var birthdateValue = new ValueMapper<DateOnly>(birthdates, date => date.ToString("yyyy-MM-dd"));
var appender = new ObjectPropertyAppender<JsonObject, JsonValue>(personObject, birthdateValue, "$.user.birthdate");
var sink = new MemorySink<JsonObject>(appender);
pipeline.Start();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
using Streamistry.Pipes.Mappers;
using Streamistry.Pipes.Sinks;

namespace Streamistry.Testing;
namespace Streamistry.Json.Testing;
public class RestResponderTests
{
[Test]
Expand All @@ -25,7 +25,7 @@ public void Emit_ValidData_CallHttpClient()

var pipeline = new Pipeline<int>();
var mapper = new RestResponder<int, JsonObject>(pipeline, client, x => $"/customer/{x}");
var plunker = new JsonPathPlucker<string>(mapper, "$.user.name");
var plunker = new PathPlucker<string>(mapper, "$.user.name");
var sink = new MemorySink<string>(plunker);

pipeline.Emit(1);
Expand Down
33 changes: 33 additions & 0 deletions Streamistry.Json.Testing/Streamistry.Json.Testing.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
<Project Sdk="Microsoft.NET.Sdk">
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.0" />
<PackageReference Include="NUnit" Version="4.2.2" />
<PackageReference Include="NUnit.Analyzers" Version="4.3.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="NUnit3TestAdapter" Version="4.6.0" />
<PackageReference Include="Appveyor.TestLogger" Version="2.0.0" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="coverlet.msbuild" Version="6.0.2">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="coverlet.collector" Version="6.0.2">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="RichardSzalay.MockHttp" Version="7.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\Streamistry\Streamistry.Core\Streamistry.csproj" />
<ProjectReference Include="..\Streamistry.Json\Streamistry.Json.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Update="DotNet.ReproducibleBuilds" Version="1.2.4">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
</PackageReference>
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@
using System.Text.Json.Nodes;
using System.Threading.Tasks;

namespace Streamistry.Pipes.Parsers;
public class JsonArrayParser : StringParser<JsonArray>
namespace Streamistry.Json;
public class ArrayParser : StringParser<JsonArray>
{
public JsonArrayParser(IChainablePipe<string> upstream)
public ArrayParser(IChainablePipe<string> upstream)
: base(upstream, TryParse)
{ }

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@
using System.Text.Json.Nodes;
using System.Threading.Tasks;

namespace Streamistry.Pipes.Splitters;
internal class JsonArraySplitter : Splitter<JsonArray, JsonObject>
namespace Streamistry.Json;
internal class ArraySplitter : Splitter<JsonArray, JsonObject>
{
public JsonArraySplitter(IChainablePipe<JsonArray> upstream)
public ArraySplitter(IChainablePipe<JsonArray> upstream)
: base(upstream, x => [..Split(x)])
{ }

Expand Down
4 changes: 4 additions & 0 deletions Streamistry.Json/FodyWeavers.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<Weavers xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="FodyWeavers.xsd">
<MethodBoundaryAspect />
</Weavers>
26 changes: 26 additions & 0 deletions Streamistry.Json/FodyWeavers.xsd
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<?xml version="1.0" encoding="utf-8"?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
<!-- This file was generated by Fody. Manual changes to this file will be lost when your project is rebuilt. -->
<xs:element name="Weavers">
<xs:complexType>
<xs:all>
<xs:element name="MethodBoundaryAspect" minOccurs="0" maxOccurs="1" type="xs:anyType" />
</xs:all>
<xs:attribute name="VerifyAssembly" type="xs:boolean">
<xs:annotation>
<xs:documentation>'true' to run assembly verification (PEVerify) on the target assembly after all weavers have been executed.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="VerifyIgnoreCodes" type="xs:string">
<xs:annotation>
<xs:documentation>A comma-separated list of error codes that can be safely ignored in assembly verification.</xs:documentation>
</xs:annotation>
</xs:attribute>
<xs:attribute name="GenerateXsd" type="xs:boolean">
<xs:annotation>
<xs:documentation>'false' to turn off automatic generation of the XML Schema file.</xs:documentation>
</xs:annotation>
</xs:attribute>
</xs:complexType>
</xs:element>
</xs:schema>
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@
using System.Text.Json.Nodes;
using System.Threading.Tasks;

namespace Streamistry.Pipes.Parsers;
public class JsonObjectParser : StringParser<JsonObject>
namespace Streamistry.Json;
public class ObjectParser : StringParser<JsonObject>
{
public JsonObjectParser(IChainablePipe<string> upstream)
public ObjectParser(IChainablePipe<string> upstream)
: base(upstream, new ParserDelegate<string, JsonObject>(TryParse))
{ }

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,13 @@
using Json.Path;


namespace Streamistry;
public class JsonObjectPropertyAppender<TInputMain, TInputSecondary> : Zipper<TInputMain, TInputSecondary, TInputMain>
namespace Streamistry.Json;
public class ObjectPropertyAppender<TInputMain, TInputSecondary> : Zipper<TInputMain, TInputSecondary, TInputMain>
where TInputMain : JsonNode
where TInputSecondary : JsonNode
{

public JsonObjectPropertyAppender(IChainablePort<TInputMain> mainUpstream, IChainablePort<TInputSecondary> secondUpstream, string path)
public ObjectPropertyAppender(IChainablePort<TInputMain> mainUpstream, IChainablePort<TInputSecondary> secondUpstream, string path)
: base(mainUpstream, secondUpstream, (x, y) => AppendProperty(x, y, path))
{ }

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
using System.Threading.Tasks;
using Json.Path;

namespace Streamistry.Pipes.Mappers;
namespace Streamistry.Json;
public abstract class BaseJsonPathPlucker<TJson, T> : Mapper<TJson, T> where TJson : JsonNode
{
public BaseJsonPathPlucker(IChainablePipe<TJson> upstream, string path)
Expand All @@ -24,9 +24,9 @@ public BaseJsonPathPlucker(IChainablePipe<TJson> upstream, string path)
}
}

public class JsonPathPlucker<T> : BaseJsonPathPlucker<JsonObject, T>
public class PathPlucker<T> : BaseJsonPathPlucker<JsonObject, T>
{
public JsonPathPlucker(IChainablePipe<JsonObject> upstream, string path)
public PathPlucker(IChainablePipe<JsonObject> upstream, string path)
: base(upstream, path)
{ }
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
using Streamistry.Observability;
using Streamistry.Pipes.Parsers;

namespace Streamistry.Pipes;
namespace Streamistry.Json;
public class RestResponder<TInput, TOutput> : EscapeRouterPipe<TInput, TOutput>, IProcessablePipe<TInput?> where TOutput : JsonNode
{
protected HttpClient Client { get; }
Expand Down
33 changes: 33 additions & 0 deletions Streamistry.Json/Streamistry.Json.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<PackageId>Streamistry.Json</PackageId>
<RepositoryUrl>https://github.com/Seddryck/Streamistry</RepositoryUrl>
<PackageTags>etl;data-engineering;data-integration;data-pipeline;streaming-data;json</PackageTags>
<Description>The JSON package of Streamistry is specifically designed to handle JSON data within pipeline, streaming, and ETL processes. Its powerful and flexible features make it an essential tool for managing complex JSON structures, facilitating seamless data parsing, transformation, and integration.</Description>
</PropertyGroup>

<PropertyGroup>
<RestoreProjectStyle>PackageReference</RestoreProjectStyle>
<CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="MethodBoundaryAspect.Fody" Version="2.0.149" />
</ItemGroup>

<ItemGroup>
<PackageReference Include="JsonPath.Net" Version="1.1.4" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\Streamistry.Core\Streamistry.csproj" />
</ItemGroup>

<ItemGroup>
<PackageReference Update="DotNet.ReproducibleBuilds" Version="1.2.4">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
</PackageReference>
</ItemGroup>
</Project>
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@
using System.Text.Json.Nodes;
using System.Threading.Tasks;

namespace Streamistry;
public class JsonValueMapper<TInput> : Mapper<TInput, JsonValue>
namespace Streamistry.Json;
public class ValueMapper<TInput> : Mapper<TInput, JsonValue>
{
public JsonValueMapper(IChainablePort<TInput> upstream, Func<TInput?, string>? toString = null)
public ValueMapper(IChainablePort<TInput> upstream, Func<TInput?, string>? toString = null)
: base(upstream, value => toString is null ? JsonValue.Create(value) : JsonValue.Create(toString.Invoke(value)))
{
}
Expand Down
19 changes: 18 additions & 1 deletion Streamistry.sln
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
Microsoft Visual Studio Solution File, Format Version 12.00

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.1.32328.378
MinimumVisualStudioVersion = 10.0.40219.1
Expand Down Expand Up @@ -34,6 +35,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Edit", "Edit", "{210C887C-6
.gitignore = .gitignore
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Json", "Json", "{3AD64C0C-6DA9-4947-A225-6F68CF9A6A4D}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Streamistry.Json", "Streamistry.Json\Streamistry.Json.csproj", "{AFC0A591-FD48-481F-AB90-00E7B0926FE1}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Streamistry.Json.Testing", "Streamistry.Json.Testing\Streamistry.Json.Testing.csproj", "{84FA815D-FE05-4B57-9BB1-156E7B217576}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand All @@ -48,6 +55,14 @@ Global
{A0CDC412-D16C-4508-96FC-6F0BB383E962}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A0CDC412-D16C-4508-96FC-6F0BB383E962}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A0CDC412-D16C-4508-96FC-6F0BB383E962}.Release|Any CPU.Build.0 = Release|Any CPU
{AFC0A591-FD48-481F-AB90-00E7B0926FE1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{AFC0A591-FD48-481F-AB90-00E7B0926FE1}.Debug|Any CPU.Build.0 = Debug|Any CPU
{AFC0A591-FD48-481F-AB90-00E7B0926FE1}.Release|Any CPU.ActiveCfg = Release|Any CPU
{AFC0A591-FD48-481F-AB90-00E7B0926FE1}.Release|Any CPU.Build.0 = Release|Any CPU
{84FA815D-FE05-4B57-9BB1-156E7B217576}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{84FA815D-FE05-4B57-9BB1-156E7B217576}.Debug|Any CPU.Build.0 = Debug|Any CPU
{84FA815D-FE05-4B57-9BB1-156E7B217576}.Release|Any CPU.ActiveCfg = Release|Any CPU
{84FA815D-FE05-4B57-9BB1-156E7B217576}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand All @@ -56,6 +71,8 @@ Global
{95E83BA6-B535-4D56-B06A-D00249A4EF00} = {C923A152-4091-48BE-A984-686FF5E4ACCC}
{09BDFA96-353F-419D-890A-D9B9045E66DD} = {C923A152-4091-48BE-A984-686FF5E4ACCC}
{210C887C-6FF3-4D1B-9A7B-894E2F286FF4} = {C923A152-4091-48BE-A984-686FF5E4ACCC}
{AFC0A591-FD48-481F-AB90-00E7B0926FE1} = {3AD64C0C-6DA9-4947-A225-6F68CF9A6A4D}
{84FA815D-FE05-4B57-9BB1-156E7B217576} = {3AD64C0C-6DA9-4947-A225-6F68CF9A6A4D}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {8EABD9FE-559A-40AF-BB4B-8026F9240FEF}
Expand Down
6 changes: 5 additions & 1 deletion appveyor.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ branches:
- /v\d+\.\d+\.\d+/
skip_commits:
files:
- .github/
- docs/
- misc/
- README.md
Expand Down Expand Up @@ -53,6 +54,8 @@ test_script:
$ErrorActionPreference = "Stop"
dotnet test Streamistry.Testing -c Release /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Threshold=10 /p:ThresholdType=line /p:CoverletOutput=../.coverage/coverage.Streamistry.xml --test-adapter-path:. --logger:Appveyor --no-build --nologo
$globalTestResult = $LastExitCode
dotnet test Streamistry.Json.Testing -c Release /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Threshold=10 /p:ThresholdType=line /p:CoverletOutput=../.coverage/coverage.Streamistry.Json.xml --test-adapter-path:. --logger:Appveyor --no-build --nologo
$globalTestResult = $LastExitCode
if($globalTestResult -ne 0) { $host.SetShouldExit($globalTestResult) }
- pwsh: |
Expand All @@ -61,7 +64,8 @@ test_script:
.\codecov.exe --dir "./.coverage/"
after_test:
- dotnet pack Streamistry.Core -p:version="%GitVersion_SemVer%" -c Release --include-symbols --no-build --nologo
- dotnet pack Streamistry.Core -p:version="%GitVersion_SemVer%" -c Release --include-symbols --no-build --nologo
- dotnet pack Streamistry.Json -p:version="%GitVersion_SemVer%" -c Release --include-symbols --no-build --nologo

artifacts:
- path: '**\*.nupkg'
Expand Down

0 comments on commit 2c0969b

Please sign in to comment.