JsonL Attacher [wip]

Rdmp.Core.Tests/DataLoad/Engine/Integration/JsonLAttacherTests.cs

@@ -0,0 +1,107 @@
+using FAnsi;
+using FAnsi.Discovery;
+using NUnit.Framework;
+using Rdmp.Core.Curation;
+using Rdmp.Core.Curation.Data;
+using Rdmp.Core.DataFlowPipeline;
+using Rdmp.Core.DataLoad;
+using Rdmp.Core.DataLoad.Engine.Job;
+using Rdmp.Core.DataLoad.Modules.Attachers;
+using ReusableLibraryCode.Progress;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Tests.Common;
+using TypeGuesser;
+
+namespace Rdmp.Core.Tests.DataLoad.Engine.Integration
+{
+    class JsonLAttacherTests : DatabaseTests
+    {
+        private LoadDirectory LoadDirectory;
+        DirectoryInfo parentDir;
+        private DiscoveredDatabase _database;
+
+        [SetUp]
+        protected override void SetUp()
+        {
+            base.SetUp();
+
+            var workingDir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory);
+            parentDir = workingDir.CreateSubdirectory("FlatFileAttacherTests");
+
+            DirectoryInfo toCleanup = parentDir.GetDirectories().SingleOrDefault(d => d.Name.Equals("Test_CSV_Attachment"));
+            if (toCleanup != null)
+                toCleanup.Delete(true);
+
+            LoadDirectory = LoadDirectory.CreateDirectoryStructure(parentDir, "JsonLAttacherTests",true);
+
+            // create a separate builder for setting an initial catalog on (need to figure out how best to stop child classes changing ServerICan... as this then causes TearDown to fail)
+            _database = GetCleanedServer(DatabaseType.MicrosoftSQLServer);
+
+            using (var con = _database.Server.GetConnection())
+            {
+                con.Open();
+
+                var cmdCreateTable = _database.Server.GetCommand("CREATE Table " + _database.GetRuntimeName() + "..Bob([name] [varchar](500),[name2] [varchar](500))", con);
+                cmdCreateTable.ExecuteNonQuery();
+            }
+        }
+
+        [Test]
+        public void SimpleJsonl_OneTable()
+        {
+            string json = @"
+        {""name"": ""Gilbert"", ""wins"": [[""straight"", ""7♣""], [""one pair"", ""10♥""]]}
+{""name"": ""Alexa"", ""wins"": [[""two pair"", ""4♠""], [""two pair"", ""9♠""]]}
+{ ""name"": ""May"", ""wins"": []}
+{ ""name"": ""Deloise"", ""wins"": [[""three of a kind"", ""5♣""]]}
+";
+
+            string filename = Path.Combine(LoadDirectory.ForLoading.FullName, "some.jsonl");
+            File.WriteAllText(filename, json);
+
+            var attacher = new JsonLAttacher();
+            attacher.Initialize(LoadDirectory, _database);
+            attacher.FilePattern = "some.jsonl";
+
+            var player = _database.CreateTable("Player", new[] { 
+                new DatabaseColumnRequest("name", new DatabaseTypeRequest(typeof(string), 10)){IsPrimaryKey = true},
+                new DatabaseColumnRequest("wins", new DatabaseTypeRequest(typeof(string), int.MaxValue))
+            });
+
+            Import(player, out ITableInfo playerTi,out _);
+
+            attacher.RootTable = playerTi;
+
+            //other cases (i.e. correct separator)
+            attacher.Attach(new ThrowImmediatelyDataLoadJob(), new GracefulCancellationToken());
+
+            Assert.AreEqual(4, player.GetRowCount());
+
+            using (var con = _database.Server.GetConnection())
+            {
+                con.Open();
+                using (var r = _database.Server.GetCommand("Select * from Player order by name", con).ExecuteReader())
+                {
+                    Assert.IsTrue(r.Read());
+                    Assert.AreEqual("Alexa", r["name"]);
+                    Assert.IsTrue(r.Read());
+                    Assert.AreEqual("Deloise", r["name"]);
+                    Assert.IsTrue(r.Read());
+                    Assert.AreEqual("Gilbert", r["name"]);
+                    Assert.IsTrue(r.Read());
+                    Assert.AreEqual("May", r["name"]);
+                }
+            }
+
+            attacher.LoadCompletedSoDispose(ExitCodeType.Success, new ThrowImmediatelyDataLoadEventListener());
+
+            File.Delete(filename);
+        }
+
+    }
+}

Rdmp.Core/DataLoad/Modules/Attachers/JSONLAttacher.cs

@@ -0,0 +1,88 @@
+using FAnsi.Discovery;
+using Newtonsoft.Json;
+using Rdmp.Core.Curation.Data;
+using Rdmp.Core.Curation.Data.DataLoad;
+using Rdmp.Core.DataFlowPipeline;
+using Rdmp.Core.DataLoad.Engine.Attachers;
+using Rdmp.Core.DataLoad.Engine.Job;
+using ReusableLibraryCode.Checks;
+using ReusableLibraryCode.Progress;
+using System;
+using System.Collections.Generic;
+using System.IO;
+
+namespace Rdmp.Core.DataLoad.Modules.Attachers
+{
+    public class JsonLAttacher : Attacher
+    {
+        [DemandsInitialization("The root table in RAW that will be loaded by this Attacher", Mandatory = true)]
+        public ITableInfo RootTable { get; set; }
+
+        [DemandsInitialization("Pattern to match files in forLoading in.  Defaults to *.json", DefaultValue = "*.json", Mandatory = true)]
+        public string FilePattern { get; set; } = "*.json";
+
+        [DemandsInitialization(@"Map for sub attributes to tables.  Format is MyProp=MyTable,MyProp2=MyTable2.  
+If not specified then properties must exactly match table names")]
+        public string AttributeTableMap { get; set; }
+
+        public JsonLAttacher():base(true)
+        {
+
+        }
+
+        public override ExitCodeType Attach(IDataLoadJob job, GracefulCancellationToken cancellationToken)
+        {
+            foreach(var file in LoadDirectory.ForLoading.GetFiles(FilePattern))
+            {
+                using (var sr = new StreamReader(file.FullName))
+                {
+                    var jsonReader = new JsonTextReader(sr)
+                    {
+                        SupportMultipleContent = true // This is important!
+                    };
+
+                    Dictionary<string, object> vals = new Dictionary<string, object>();
+
+                    var tblName = RootTable.GetRuntimeName(LoadStage.AdjustRaw, job?.Configuration?.DatabaseNamer);
+                    var tbl = _dbInfo.ExpectTable(tblName);
+
+                    if(!tbl.Exists())
+                    {
+                        throw new Exception($"Expected table {tbl.GetFullyQualifiedName()} was not found in RAW databse");
+                    }
+
+                    while (jsonReader.Read())
+                    {
+                        if(jsonReader.TokenType == JsonToken.PropertyName)
+                        {
+                            vals.Add(jsonReader.Value.ToString(), null);
+
+                        }
+                        if(jsonReader.TokenType == JsonToken.String)
+                        {
+                            if(vals.ContainsKey(jsonReader.Path))
+                                vals[jsonReader.Path] = jsonReader.Value;
+                        }
+                        if (jsonReader.TokenType == JsonToken.EndObject)
+                        {
+                            // load tables
+                            tbl.Insert(vals);
+                            vals.Clear();
+                        }
+                    }
+                }
+            }
+
+
+            return ExitCodeType.Success;
+        }
+
+        public override void Check(ICheckNotifier notifier)
+        {
+        }
+
+        public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
+        {
+        }
+    }
+}