From 804d6b5c042e7cade9d42847a1113af651b4a737 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Jan 2024 00:30:56 +0000 Subject: [PATCH 01/27] Bump actions/cache from 3 to 4 Bumps [actions/cache](https://github.com/actions/cache) from 3 to 4. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- .github/workflows/docker.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b01fd67b56..4f170f114b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -261,7 +261,7 @@ jobs: dist - name: Cache Build id: cache-build - uses: actions/cache/save@v3 + uses: actions/cache/save@v4 with: path: ${{ github.workspace }}/ key: ${{ github.sha }}-your-cache-key-bundled @@ -272,7 +272,7 @@ jobs: needs: ['bundle','tests_db','tests_file_system'] if: contains(github.ref, 'refs/tags/v') steps: - - uses: actions/cache/restore@v3 + - uses: actions/cache/restore@v4 id: restore-build with: path: ${{ github.workspace }}/ diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5e3ef508dc..b3cb3dbfa6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,7 +21,7 @@ jobs: with: dotnet-version: 6.0.x - name: Cache Nuget - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj') }} From 5d56ce979cb16002e62d54ddc3cb11dcf12af413 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jan 2024 00:32:33 +0000 Subject: [PATCH 02/27] Bump Terminal.Gui from 1.14.1 to 1.15.1 Bumps [Terminal.Gui](https://github.com/gui-cs/Terminal.Gui) from 1.14.1 to 1.15.1. - [Release notes](https://github.com/gui-cs/Terminal.Gui/releases) - [Commits](https://github.com/gui-cs/Terminal.Gui/compare/v1.14.1...v1.15.1) --- updated-dependencies: - dependency-name: Terminal.Gui dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Rdmp.Core/Rdmp.Core.csproj | 2 +- Tools/rdmp/rdmp.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index cea418e93f..d03d73ff29 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -316,7 +316,7 @@ - + diff --git a/Tools/rdmp/rdmp.csproj b/Tools/rdmp/rdmp.csproj index e71929d7b2..6ce5c59730 100644 --- a/Tools/rdmp/rdmp.csproj +++ b/Tools/rdmp/rdmp.csproj @@ -39,7 +39,7 @@ - + From 17f45f345ff636f4cdbd841f184cf251cb45c3aa Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 23 Jan 2024 10:14:50 +0000 Subject: [PATCH 03/27] Spike/rdmp-132 migrate to log file (#1727) * update tableLoadInfo table * update loggin to trim logs * add basic logger * add user settings * bump to 8.1.4 * additional logs to file * tidy up code * update log generator * tidy up * tidy up form pr * fix build --- CHANGELOG.md | 6 ++ Rdmp.Core/Logging/DataLoadInfo.cs | 55 +++++++++++------- Rdmp.Core/Logging/FileSystemLogger.cs | 56 +++++++++++++++++++ Rdmp.Core/Logging/LogManager.cs | 7 ++- Rdmp.Core/Logging/TableLoadInfo.cs | 56 ++++++++++++------- .../Settings/UserSettings.cs | 18 ++++++ .../SimpleDialogs/UserSettingsUI.Designer.cs | 49 ++++++++++++++-- Rdmp.UI/SimpleDialogs/UserSettingsUI.cs | 10 ++++ Rdmp.UI/SimpleDialogs/UserSettingsUI.resx | 3 + SharedAssemblyInfo.cs | 6 +- rdmp-client.xml | 4 +- 11 files changed, 216 insertions(+), 54 deletions(-) create mode 100644 Rdmp.Core/Logging/FileSystemLogger.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a7178a541..6dc054034a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [8.1.4] - Unreleased + +## Changed + +- Add ability to log to file instead of database + ## [8.1.3] - 2024-01-15 ### Changed diff --git a/Rdmp.Core/Logging/DataLoadInfo.cs b/Rdmp.Core/Logging/DataLoadInfo.cs index 6d2dbb91d8..f21a7be030 100644 --- a/Rdmp.Core/Logging/DataLoadInfo.cs +++ b/Rdmp.Core/Logging/DataLoadInfo.cs @@ -10,6 +10,7 @@ using System.Threading; using FAnsi; using FAnsi.Discovery; +using Rdmp.Core.ReusableLibraryCode.Settings; namespace Rdmp.Core.Logging; @@ -183,8 +184,8 @@ private void RecordNewDataLoadInDatabase(string dataLoadTaskName) DatabaseSettings.AddParameterWithValueToCommand("@startTime", cmd, _startTime); DatabaseSettings.AddParameterWithValueToCommand("@dataLoadTaskID", cmd, parentTaskID); DatabaseSettings.AddParameterWithValueToCommand("@isTest", cmd, _isTest); - DatabaseSettings.AddParameterWithValueToCommand("@packageName", cmd, _packageName); - DatabaseSettings.AddParameterWithValueToCommand("@userAccount", cmd, _userAccount); + DatabaseSettings.AddParameterWithValueToCommand("@packageName", cmd, _packageName.Substring(Math.Max(0, _packageName.Length - 750))); + DatabaseSettings.AddParameterWithValueToCommand("@userAccount", cmd, _userAccount.Substring(Math.Max(0, _packageName.Length - 500))); DatabaseSettings.AddParameterWithValueToCommand("@suggestedRollbackCommand", cmd, _suggestedRollbackCommand ?? string.Empty); @@ -295,16 +296,24 @@ public void LogFatalError(string errorSource, string errorDescription) var statusID = int.Parse(cmdLookupStatusID.ExecuteScalar().ToString()); - var cmdRecordFatalError = DatabaseSettings.GetCommand( - @"INSERT INTO FatalError (time,source,description,statusID,dataLoadRunID) VALUES (@time,@source,@description,@statusID,@dataLoadRunID);", - con); - DatabaseSettings.AddParameterWithValueToCommand("@time", cmdRecordFatalError, DateTime.Now); - DatabaseSettings.AddParameterWithValueToCommand("@source", cmdRecordFatalError, errorSource); - DatabaseSettings.AddParameterWithValueToCommand("@description", cmdRecordFatalError, errorDescription); - DatabaseSettings.AddParameterWithValueToCommand("@statusID", cmdRecordFatalError, statusID); - DatabaseSettings.AddParameterWithValueToCommand("@dataLoadRunID", cmdRecordFatalError, ID); + if (UserSettings.LogToFileSystem && !string.IsNullOrWhiteSpace(UserSettings.FileSystemLogLocation)) + { + var logger = FileSystemLogger.Instance; + logger.LogEventToFile(FileSystemLogger.AvailableLoggers.FatalError, [errorSource,errorDescription, statusID, ID]); + } + else + { + var cmdRecordFatalError = DatabaseSettings.GetCommand( + @"INSERT INTO FatalError (time,source,description,statusID,dataLoadRunID) VALUES (@time,@source,@description,@statusID,@dataLoadRunID);", + con); + DatabaseSettings.AddParameterWithValueToCommand("@time", cmdRecordFatalError, DateTime.Now); + DatabaseSettings.AddParameterWithValueToCommand("@source", cmdRecordFatalError, errorSource); + DatabaseSettings.AddParameterWithValueToCommand("@description", cmdRecordFatalError, errorDescription); + DatabaseSettings.AddParameterWithValueToCommand("@statusID", cmdRecordFatalError, statusID); + DatabaseSettings.AddParameterWithValueToCommand("@dataLoadRunID", cmdRecordFatalError, ID); - cmdRecordFatalError.ExecuteNonQuery(); + cmdRecordFatalError.ExecuteNonQuery(); + } //this might get called multiple times (many errors in rapid succession as the program crashes) but only close the dataLoadInfo once if (!IsClosed) @@ -322,16 +331,24 @@ public enum ProgressEventType public void LogProgress(ProgressEventType pevent, string Source, string Description) { - lock (_oLock) + if (UserSettings.LogToFileSystem && !string.IsNullOrWhiteSpace(UserSettings.FileSystemLogLocation)) + { + var logger = FileSystemLogger.Instance; + logger.LogEventToFile(FileSystemLogger.AvailableLoggers.ProgressLog, [ID, pevent, Source, Description]); + } + else { - if (_logQueue is null || _logQueue.IsAddingCompleted) - LogInit(); - if (_logQueue is null) - throw new InvalidOperationException("LogInit failed to create new worker"); + lock (_oLock) + { + if (_logQueue is null || _logQueue.IsAddingCompleted) + LogInit(); + if (_logQueue is null) + throw new InvalidOperationException("LogInit failed to create new worker"); - _logQueue.Add(new LogEntry(pevent.ToString(), Description, Source, DateTime.Now)); + _logQueue.Add(new LogEntry(pevent.ToString(), Description, Source, DateTime.Now)); + } + lock (_logWaiter) + Monitor.Pulse(_logWaiter); } - lock (_logWaiter) - Monitor.Pulse(_logWaiter); } } diff --git a/Rdmp.Core/Logging/FileSystemLogger.cs b/Rdmp.Core/Logging/FileSystemLogger.cs new file mode 100644 index 0000000000..4521d2693a --- /dev/null +++ b/Rdmp.Core/Logging/FileSystemLogger.cs @@ -0,0 +1,56 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using NLog; +using Rdmp.Core.ReusableLibraryCode.Settings; +using System; +using System.IO; + +namespace Rdmp.Core.Logging; + + +/// +/// Singleton logger for writing logs to file +/// +public class FileSystemLogger +{ + + private static readonly Lazy lazy = + new Lazy(() => new FileSystemLogger()); + + public static FileSystemLogger Instance { get { return lazy.Value; } } + + + public enum AvailableLoggers + { + ProgressLog, + DataSource, + FatalError + } + + + + private FileSystemLogger() + { + var location = UserSettings.FileSystemLogLocation; + var config = new NLog.Config.LoggingConfiguration(); + string[] logs = System.Enum.GetNames(typeof(AvailableLoggers)); + foreach (var log in logs){ + using var nLogEntry = new NLog.Targets.FileTarget(log) { FileName = Path.Combine(location, $"{log}.log"), ArchiveAboveSize = UserSettings.LogFileSizeLimit }; + config.AddRule(LogLevel.Info, LogLevel.Info, nLogEntry); + } + NLog.LogManager.Configuration = config; + } + + public void LogEventToFile(AvailableLoggers logType, object[] logItems) + { + var logMessage = $"{string.Join("|", Array.ConvertAll(logItems, item => item.ToString()))}"; + var Logger = NLog.LogManager.GetLogger(Enum.GetName(typeof(AvailableLoggers), logType)); + Logger.Info(logMessage); + } + + +} \ No newline at end of file diff --git a/Rdmp.Core/Logging/LogManager.cs b/Rdmp.Core/Logging/LogManager.cs index 0777065d35..dbba1afde6 100644 --- a/Rdmp.Core/Logging/LogManager.cs +++ b/Rdmp.Core/Logging/LogManager.cs @@ -13,6 +13,7 @@ using System.Threading; using FAnsi.Discovery; using FAnsi.Discovery.QuerySyntax; +using Rdmp.Core.Curation.Data; using Rdmp.Core.Logging.PastEvents; using Rdmp.Core.ReusableLibraryCode; using Rdmp.Core.ReusableLibraryCode.DataAccess; @@ -242,8 +243,8 @@ public void CreateNewLoggingTask(int id, string dataSetID) using var cmd = Server.GetCommand(sql, conn); Server.AddParameterWithValueToCommand("@date", cmd, DateTime.Now); - Server.AddParameterWithValueToCommand("@dataSetID", cmd, dataSetID); - Server.AddParameterWithValueToCommand("@username", cmd, Environment.UserName); + Server.AddParameterWithValueToCommand("@dataSetID", cmd, dataSetID.Substring(Math.Max(0, dataSetID.Length - 1000))); + Server.AddParameterWithValueToCommand("@username", cmd, Environment.UserName.Substring(Math.Max(0, Environment.UserName.Length - 500))); cmd.ExecuteNonQuery(); } @@ -256,7 +257,7 @@ private void CreateNewDataSet(string datasetName) const string sql = "INSERT INTO DataSet (dataSetID,name) VALUES (@datasetName,@datasetName)"; using var cmd = Server.GetCommand(sql, conn); - Server.AddParameterWithValueToCommand("@datasetName", cmd, datasetName); + Server.AddParameterWithValueToCommand("@datasetName", cmd, datasetName.Substring(Math.Max(0, datasetName.Length - 150))); cmd.ExecuteNonQuery(); } } diff --git a/Rdmp.Core/Logging/TableLoadInfo.cs b/Rdmp.Core/Logging/TableLoadInfo.cs index 2d9c1543ae..40c20e41eb 100644 --- a/Rdmp.Core/Logging/TableLoadInfo.cs +++ b/Rdmp.Core/Logging/TableLoadInfo.cs @@ -7,8 +7,11 @@ using System; using System.Data; using System.Threading; +using Amazon.Auth.AccessControlPolicy; +using BadMedicine; using FAnsi.Connections; using FAnsi.Discovery; +using Rdmp.Core.ReusableLibraryCode.Settings; namespace Rdmp.Core.Logging; @@ -85,7 +88,7 @@ private void RecordNewTableLoadInDatabase(DataLoadInfo parent, string destinatio DatabaseSettings.AddParameterWithValueToCommand("@startTime", cmd, DateTime.Now); DatabaseSettings.AddParameterWithValueToCommand("@dataLoadRunID", cmd, parent.ID); - DatabaseSettings.AddParameterWithValueToCommand("@targetTable", cmd, destinationTable); + DatabaseSettings.AddParameterWithValueToCommand("@targetTable", cmd, destinationTable.Substring(Math.Max(0,destinationTable.Length - 200))); //200 char limit on this table, just pull the rightmost 200 chars DatabaseSettings.AddParameterWithValueToCommand("@expectedInserts", cmd, expectedInserts); DatabaseSettings.AddParameterWithValueToCommand("@suggestedRollbackCommand", cmd, _suggestedRollbackCommand); @@ -93,38 +96,49 @@ private void RecordNewTableLoadInDatabase(DataLoadInfo parent, string destinatio //get the ID, can come back as a decimal or an Int32 or an Int64 so whatever, just turn it into a string and then parse it _id = int.Parse(cmd.ExecuteScalar().ToString()); + //keep a record of all data sources DataSources = sources; //for each of the sources, create them in the DataSource table foreach (var s in DataSources) { - using var cmdInsertDs = DatabaseSettings.GetCommand( - "INSERT INTO DataSource (source,tableLoadRunID,originDate,MD5) " + - "VALUES (@source,@tableLoadRunID,@originDate,@MD5); SELECT @@IDENTITY;", con); - DatabaseSettings.AddParameterWithValueToCommand("@source", cmdInsertDs, s.Source); - DatabaseSettings.AddParameterWithValueToCommand("@tableLoadRunID", cmdInsertDs, _id); - DatabaseSettings.AddParameterWithValueToCommand("@originDate", cmdInsertDs, - s.UnknownOriginDate ? DBNull.Value : s.OriginDate); - - // old logging schema used binary[128] for the MD5 column - if (IsLegacyLoggingSchema) + if (UserSettings.LogToFileSystem && !string.IsNullOrWhiteSpace(UserSettings.FileSystemLogLocation)) { - var p = cmdInsertDs.CreateParameter(); - p.DbType = DbType.Binary; - p.Size = 128; - p.Value = s.MD5 != null ? s.MD5 : DBNull.Value; - p.ParameterName = "@MD5"; - cmdInsertDs.Parameters.Add(p); + var logger = FileSystemLogger.Instance; + logger.LogEventToFile(FileSystemLogger.AvailableLoggers.DataSource, [s.Source,_id ,s.UnknownOriginDate ? "" : s.OriginDate]); } else { - // now logging schema uses string for easier usability and FAnsiSql compatibility - DatabaseSettings.AddParameterWithValueToCommand("@MD5", cmdInsertDs, - s.MD5 != null ? s.MD5 : DBNull.Value); + + using var cmdInsertDs = DatabaseSettings.GetCommand( + "INSERT INTO DataSource (source,tableLoadRunID,originDate,MD5) " + + "VALUES (@source,@tableLoadRunID,@originDate,@MD5); SELECT @@IDENTITY;", con); + DatabaseSettings.AddParameterWithValueToCommand("@source", cmdInsertDs, s.Source); + DatabaseSettings.AddParameterWithValueToCommand("@tableLoadRunID", cmdInsertDs, _id); + DatabaseSettings.AddParameterWithValueToCommand("@originDate", cmdInsertDs, + s.UnknownOriginDate ? DBNull.Value : s.OriginDate); + + // old logging schema used binary[128] for the MD5 column + if (IsLegacyLoggingSchema) + { + var p = cmdInsertDs.CreateParameter(); + p.DbType = DbType.Binary; + p.Size = 128; + p.Value = s.MD5 != null ? s.MD5 : DBNull.Value; + p.ParameterName = "@MD5"; + cmdInsertDs.Parameters.Add(p); + } + else + { + // now logging schema uses string for easier usability and FAnsiSql compatibility + DatabaseSettings.AddParameterWithValueToCommand("@MD5", cmdInsertDs, + s.MD5 != null ? s.MD5 : DBNull.Value); + } + + s.ID = int.Parse(cmdInsertDs.ExecuteScalar().ToString()); } - s.ID = int.Parse(cmdInsertDs.ExecuteScalar().ToString()); } } diff --git a/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs b/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs index 94e5bd34a0..0b9a8223df 100644 --- a/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs +++ b/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs @@ -34,6 +34,24 @@ public static bool UseLocalFileSystem set => AppSettings.AddOrUpdateValue("UseLocalFileSystem", value); } + public static bool LogToFileSystem + { + get => AppSettings.GetValueOrDefault("LogToFileSystem", false); + set => AppSettings.AddOrUpdateValue("LogToFileSystem", value); + } + + public static long LogFileSizeLimit + { + get => AppSettings.GetValueOrDefault("LogFileSizeLimit", 1073741824); + set => AppSettings.AddOrUpdateValue("LogFileSizeLimit", value); + } + + public static string FileSystemLogLocation + { + get => AppSettings.GetValueOrDefault("FileSystemLogLocation", null); + set => AppSettings.AddOrUpdateValue("FileSystemLogLocation", value); + } + public static string LocalFileSystemLocation { get => AppSettings.GetValueOrDefault("LocalFileSystemLocation", Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "rdmp")); diff --git a/Rdmp.UI/SimpleDialogs/UserSettingsUI.Designer.cs b/Rdmp.UI/SimpleDialogs/UserSettingsUI.Designer.cs index 22c4c212ac..200a9fd928 100644 --- a/Rdmp.UI/SimpleDialogs/UserSettingsUI.Designer.cs +++ b/Rdmp.UI/SimpleDialogs/UserSettingsUI.Designer.cs @@ -75,6 +75,7 @@ private void InitializeComponent() groupBox6 = new System.Windows.Forms.GroupBox(); cbSkipCohortBuilderValidationOnCommit = new System.Windows.Forms.CheckBox(); groupBox7 = new System.Windows.Forms.GroupBox(); + cbUseLogFiles = new System.Windows.Forms.CheckBox(); label15 = new System.Windows.Forms.Label(); tbLocalFileSystemLocation = new System.Windows.Forms.TextBox(); cbUseLocalFileSystem = new System.Windows.Forms.CheckBox(); @@ -91,6 +92,8 @@ private void InitializeComponent() userSettingsToolTips = new System.Windows.Forms.ToolTip(components); tbFind = new System.Windows.Forms.TextBox(); label14 = new System.Windows.Forms.Label(); + tbLogLocation = new System.Windows.Forms.TextBox(); + label16 = new System.Windows.Forms.Label(); groupBox1.SuspendLayout(); ((System.ComponentModel.ISupportInitialize)olvErrorCodes).BeginInit(); groupBox2.SuspendLayout(); @@ -331,7 +334,7 @@ private void InitializeComponent() // groupBox1.Anchor = System.Windows.Forms.AnchorStyles.None; groupBox1.Controls.Add(olvErrorCodes); - groupBox1.Location = new System.Drawing.Point(3, 445); + groupBox1.Location = new System.Drawing.Point(3, 478); groupBox1.Name = "groupBox1"; groupBox1.Size = new System.Drawing.Size(978, 249); groupBox1.TabIndex = 16; @@ -514,7 +517,7 @@ private void InitializeComponent() flowLayoutPanel1.Controls.Add(groupBox1); flowLayoutPanel1.Location = new System.Drawing.Point(13, 69); flowLayoutPanel1.Name = "flowLayoutPanel1"; - flowLayoutPanel1.Size = new System.Drawing.Size(987, 671); + flowLayoutPanel1.Size = new System.Drawing.Size(1012, 671); flowLayoutPanel1.TabIndex = 24; // // groupBox8 @@ -573,6 +576,9 @@ private void InitializeComponent() // // groupBox7 // + groupBox7.Controls.Add(label16); + groupBox7.Controls.Add(tbLogLocation); + groupBox7.Controls.Add(cbUseLogFiles); groupBox7.Controls.Add(label15); groupBox7.Controls.Add(tbLocalFileSystemLocation); groupBox7.Controls.Add(cbUseLocalFileSystem); @@ -593,11 +599,21 @@ private void InitializeComponent() groupBox7.Controls.Add(cbAlwaysJoinEverything); groupBox7.Location = new System.Drawing.Point(615, 174); groupBox7.Name = "groupBox7"; - groupBox7.Size = new System.Drawing.Size(360, 265); + groupBox7.Size = new System.Drawing.Size(360, 298); groupBox7.TabIndex = 25; groupBox7.TabStop = false; groupBox7.Text = "Miscellaneous"; // + // cbUseLogFiles + // + cbUseLogFiles.AutoSize = true; + cbUseLogFiles.Location = new System.Drawing.Point(7, 257); + cbUseLogFiles.Name = "cbUseLogFiles"; + cbUseLogFiles.Size = new System.Drawing.Size(94, 19); + cbUseLogFiles.TabIndex = 29; + cbUseLogFiles.Text = "Use Log Files"; + cbUseLogFiles.UseVisualStyleBackColor = true; + // // label15 // label15.AutoSize = true; @@ -732,7 +748,7 @@ private void InitializeComponent() // tbFind // tbFind.Anchor = System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right; - tbFind.Location = new System.Drawing.Point(900, 13); + tbFind.Location = new System.Drawing.Point(925, 13); tbFind.Name = "tbFind"; tbFind.Size = new System.Drawing.Size(100, 23); tbFind.TabIndex = 25; @@ -742,17 +758,35 @@ private void InitializeComponent() // label14.Anchor = System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right; label14.AutoSize = true; - label14.Location = new System.Drawing.Point(821, 16); + label14.Location = new System.Drawing.Point(846, 16); label14.Name = "label14"; label14.Size = new System.Drawing.Size(73, 15); label14.TabIndex = 26; label14.Text = "Find Setting:"; // + // tbLogLocation + // + tbLogLocation.Location = new System.Drawing.Point(156, 257); + tbLogLocation.Name = "tbLogLocation"; + tbLogLocation.Size = new System.Drawing.Size(110, 23); + tbLogLocation.TabIndex = 30; + tbLogLocation.TextChanged += tbLogLocation_TextChanged; + // + // label16 + // + label16.AutoSize = true; + label16.Location = new System.Drawing.Point(273, 261); + label16.Name = "label16"; + label16.Size = new System.Drawing.Size(53, 15); + label16.TabIndex = 31; + label16.Text = "Location"; + label16.Click += label16_Click; + // // UserSettingsFileUI // AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; - ClientSize = new System.Drawing.Size(1012, 758); + ClientSize = new System.Drawing.Size(1037, 758); Controls.Add(label14); Controls.Add(btnClearUserSettings); Controls.Add(tbFind); @@ -850,5 +884,8 @@ private void InitializeComponent() private System.Windows.Forms.Label label15; private System.Windows.Forms.TextBox tbLocalFileSystemLocation; private System.Windows.Forms.CheckBox cbUseLocalFileSystem; + private System.Windows.Forms.CheckBox cbUseLogFiles; + private System.Windows.Forms.Label label16; + private System.Windows.Forms.TextBox tbLogLocation; } } \ No newline at end of file diff --git a/Rdmp.UI/SimpleDialogs/UserSettingsUI.cs b/Rdmp.UI/SimpleDialogs/UserSettingsUI.cs index d69e474228..f962ef3b99 100644 --- a/Rdmp.UI/SimpleDialogs/UserSettingsUI.cs +++ b/Rdmp.UI/SimpleDialogs/UserSettingsUI.cs @@ -106,6 +106,7 @@ public UserSettingsFileUI(IActivateItems activator) RegisterCheckbox(cbUseAliasInsteadOfTransformInGroupByAggregateGraphs, nameof(UserSettings.UseAliasInsteadOfTransformInGroupByAggregateGraphs)); RegisterCheckbox(cbUseLocalFileSystem, nameof(UserSettings.UseLocalFileSystem)); + RegisterCheckbox(cbUseLogFiles, nameof(UserSettings.LogToFileSystem)); AddTooltip(label7, nameof(UserSettings.CreateDatabaseTimeout)); AddTooltip(tbCreateDatabaseTimeout, nameof(UserSettings.CreateDatabaseTimeout)); AddTooltip(label13, nameof(UserSettings.ArchiveTriggerTimeout)); @@ -139,6 +140,8 @@ public UserSettingsFileUI(IActivateItems activator) ddWordWrap.SelectedItem = (WrapMode)UserSettings.WrapMode; tbHeatmapColours.Text = UserSettings.HeatMapColours; + tbLocalFileSystemLocation.Text = UserSettings.LocalFileSystemLocation; + tbLogLocation.Text = UserSettings.FileSystemLogLocation; _bLoaded = true; @@ -250,6 +253,11 @@ private void tbLocalFileSystemLocation_TextChanged(object sender, EventArgs e) UserSettings.LocalFileSystemLocation = tbLocalFileSystemLocation.Text; } + private void tbLogLocation_TextChanged(object sender, EventArgs e) + { + UserSettings.FileSystemLogLocation = tbLogLocation.Text; + } + private void tbTooltipAppearDelay_TextChanged(object sender, EventArgs e) { if (int.TryParse(tbTooltipAppearDelay.Text, out var result)) UserSettings.TooltipAppearDelay = result; @@ -260,6 +268,8 @@ private void tbFind_TextChanged(object sender, EventArgs e) Find(tbFind.Text); } + private void label16_Click(object sender, EventArgs e) { } + private void Find(string text) { foreach (var cb in checkboxDictionary) diff --git a/Rdmp.UI/SimpleDialogs/UserSettingsUI.resx b/Rdmp.UI/SimpleDialogs/UserSettingsUI.resx index e96c315530..3654ecf8fb 100644 --- a/Rdmp.UI/SimpleDialogs/UserSettingsUI.resx +++ b/Rdmp.UI/SimpleDialogs/UserSettingsUI.resx @@ -120,4 +120,7 @@ 17, 17 + + 25 + \ No newline at end of file diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index 79feed5afe..abcc7124ff 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -10,6 +10,6 @@ [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] -[assembly: AssemblyVersion("8.1.3")] -[assembly: AssemblyFileVersion("8.1.3")] -[assembly: AssemblyInformationalVersion("8.1.3")] \ No newline at end of file +[assembly: AssemblyVersion("8.1.4")] +[assembly: AssemblyFileVersion("8.1.4")] +[assembly: AssemblyInformationalVersion("8.1.4-rc1")] \ No newline at end of file diff --git a/rdmp-client.xml b/rdmp-client.xml index 95bda60872..683046fb35 100644 --- a/rdmp-client.xml +++ b/rdmp-client.xml @@ -1,7 +1,7 @@ - 8.1.3.0 - https://github.com/HicServices/RDMP/releases/download/v8.1.3/rdmp-8.1.3-client.zip + 8.1.3.1 + https://github.com/HicServices/RDMP/releases/download/v8.1.4-rc1/rdmp-8.1.4-rc1-client.zip https://github.com/HicServices/RDMP/blob/main/CHANGELOG.md#7 true From e55d30ad4a4ee552a8a307d4c6bff0e350ced127 Mon Sep 17 00:00:00 2001 From: James Friel Date: Tue, 23 Jan 2024 14:49:12 +0000 Subject: [PATCH 04/27] allow project specific extraction categories to change --- .../ExecuteCommandChangeExtractionCategory.cs | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractionCategory.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractionCategory.cs index a2a0eafb01..ba2a60339c 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractionCategory.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractionCategory.cs @@ -39,16 +39,6 @@ public ExecuteCommandChangeExtractionCategory(IBasicActivateItems activator, Ext if (cata.Length == 1) _isProjectSpecific = cata[0].IsProjectSpecific(BasicActivator.RepositoryLocator.DataExportRepository); - // if project specific only let them set to project specific - if (_category != null && _isProjectSpecific && _category != ExtractionCategory.ProjectSpecific) - { - // user is trying to set to Core - if (_category == ExtractionCategory.Core) - // surely they meant project specific! - _category = ExtractionCategory.ProjectSpecific; - else - SetImpossible("CatalogueItems can only be ProjectSpecific extraction category"); - } } public override string GetCommandName() => @@ -72,10 +62,13 @@ public override void Execute() if (c == null) return; - // if project specific only let them set to project specific - if (_isProjectSpecific && c != ExtractionCategory.ProjectSpecific) - throw new Exception( - "All CatalogueItems in ProjectSpecific Catalogues must have ExtractionCategory of 'ProjectSpecific'"); + if (_isProjectSpecific && c == ExtractionCategory.Core) + { + // Don't allow project specific catalogue items to become core + c = ExtractionCategory.ProjectSpecific; + Show("Cannot set the Extraction Category to 'Core' for a Project Specific Catalogue item. It will be saved as 'Project Specific'."); + } + if(c == _category) return;//no commit needed if (ExecuteWithCommit(() => ExecuteImpl(c.Value), $"Set ExtractionCategory to '{c}'", _extractionInformations)) //publish the root Catalogue From 449842964fcf2de60a8876c047e50c5837758eeb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Jan 2024 00:58:59 +0000 Subject: [PATCH 05/27] Bump CsvHelper from 30.0.1 to 30.0.3 Bumps [CsvHelper](https://github.com/JoshClose/CsvHelper) from 30.0.1 to 30.0.3. - [Commits](https://github.com/JoshClose/CsvHelper/compare/30.0.1...30.0.3) --- updated-dependencies: - dependency-name: CsvHelper dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Rdmp.Core/Rdmp.Core.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index d03d73ff29..2042cf0feb 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -295,7 +295,7 @@ - + From a540c9722cca57732d7396455c83d80c8a853462 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Jan 2024 00:59:32 +0000 Subject: [PATCH 06/27] Bump SixLabors.ImageSharp.Drawing from 2.1.0 to 2.1.1 Bumps [SixLabors.ImageSharp.Drawing](https://github.com/SixLabors/ImageSharp.Drawing) from 2.1.0 to 2.1.1. - [Release notes](https://github.com/SixLabors/ImageSharp.Drawing/releases) - [Commits](https://github.com/SixLabors/ImageSharp.Drawing/compare/v2.1.0...v2.1.1) --- updated-dependencies: - dependency-name: SixLabors.ImageSharp.Drawing dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Rdmp.Core/Rdmp.Core.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index d03d73ff29..98f0731fbb 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -312,7 +312,7 @@ - + From 340fb5997fbd82eb14b35c0e8b03600ec283854f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Jan 2024 01:00:15 +0000 Subject: [PATCH 07/27] Bump YamlDotNet from 13.7.1 to 15.1.0 Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 13.7.1 to 15.1.0. - [Release notes](https://github.com/aaubry/YamlDotNet/releases) - [Commits](https://github.com/aaubry/YamlDotNet/compare/v13.7.1...v15.1.0) --- updated-dependencies: - dependency-name: YamlDotNet dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- Rdmp.Core/Rdmp.Core.csproj | 2 +- Tools/rdmp/rdmp.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index d03d73ff29..98c82a272e 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -317,7 +317,7 @@ - + diff --git a/Tools/rdmp/rdmp.csproj b/Tools/rdmp/rdmp.csproj index 6ce5c59730..39e4789a18 100644 --- a/Tools/rdmp/rdmp.csproj +++ b/Tools/rdmp/rdmp.csproj @@ -41,7 +41,7 @@ - + From e95fdaa60121e9e35a97653f012608f00bd6ae50 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 24 Jan 2024 09:51:20 +0000 Subject: [PATCH 08/27] add tests --- CHANGELOG.md | 1 + ...uteCommandChangeExtractionCategoryTests.cs | 87 +++++++++++++++++++ .../ExecuteCommandChangeExtractionCategory.cs | 6 +- 3 files changed, 91 insertions(+), 3 deletions(-) create mode 100644 Rdmp.Core.Tests/CommandExecution/ExecuteCommandChangeExtractionCategoryTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dc054034a..f01a5c3722 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Changed - Add ability to log to file instead of database +- Add ability to use Extraction Category with Project Specific Catalogues ## [8.1.3] - 2024-01-15 diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandChangeExtractionCategoryTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandChangeExtractionCategoryTests.cs new file mode 100644 index 0000000000..5ed73f0cce --- /dev/null +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandChangeExtractionCategoryTests.cs @@ -0,0 +1,87 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using NUnit.Framework; +using Rdmp.Core.CommandExecution; +using Rdmp.Core.CommandExecution.AtomicCommands; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Repositories; +using System.Linq; +using Tests.Common; +using NSubstitute; +using NSubstitute.Extensions; +using Rdmp.Core.DataExport.Data; +using Rdmp.Core.Curation.Data.Aggregation; +using static Org.BouncyCastle.Math.EC.ECCurve; +namespace Rdmp.Core.Tests.CommandExecution; + +public class ExecuteCommandChangeExtractionCategoryTests : DatabaseTests +{ + private Catalogue _cata1; + private Catalogue _cata2; + private TableInfo _t1; + private TableInfo _t2; + private ColumnInfo _c1; + private ColumnInfo _c2; + private CatalogueItem _ci1; + private CatalogueItem _ci2; + private Project _project; + + private ExtractionInformation _extractionInfo1; + private ExtractionInformation _extractionInfo2; + + [Test] + public void TestProjectSpecificCatalogueChangeToSuplemental() + { + //change project specific to supplemental + _cata1 =new Catalogue(CatalogueRepository, "Dataset1"); + _t1 = new TableInfo(CatalogueRepository, "T1"); + + _c1 = new ColumnInfo(CatalogueRepository, "PrivateIdentifierA", "varchar(10)", _t1); + + _ci1 = new CatalogueItem(CatalogueRepository, _cata1, "PrivateIdentifierA"); + + _extractionInfo1 = new ExtractionInformation(CatalogueRepository, _ci1, _c1, _c1.ToString()) + { + Order = 123, + ExtractionCategory = ExtractionCategory.ProjectSpecific, + IsExtractionIdentifier = true + }; + _extractionInfo1.CatalogueItem.Catalogue.InjectKnown(new CatalogueExtractabilityStatus(true, true)); + + ExtractionInformation[] eid = { _extractionInfo1 }; + var cmd = new ExecuteCommandChangeExtractionCategory(new ThrowImmediatelyActivator(RepositoryLocator), eid, ExtractionCategory.Supplemental); + Assert.DoesNotThrow(() => cmd.Execute()); + Assert.That(_extractionInfo1.ExtractionCategory, Is.EqualTo(ExtractionCategory.Supplemental)); + } + + [Test] + public void TestExtractionCategoryCatalogueChangeFromSupplementalToCore() + { + //change a project specific column to core + _cata1 = new Catalogue(CatalogueRepository, "Dataset1"); + _t1 = new TableInfo(CatalogueRepository, "T1"); + + _c1 = new ColumnInfo(CatalogueRepository, "PrivateIdentifierA", "varchar(10)", _t1); + + _ci1 = new CatalogueItem(CatalogueRepository, _cata1, "PrivateIdentifierA"); + + + _extractionInfo1 = new ExtractionInformation(CatalogueRepository, _ci1, _c1, _c1.ToString()) + { + Order = 123, + ExtractionCategory = ExtractionCategory.Supplemental, + IsExtractionIdentifier = true + }; + _extractionInfo1.CatalogueItem.Catalogue.InjectKnown(new CatalogueExtractabilityStatus(true, true)); + + ExtractionInformation[] eid = { _extractionInfo1 }; + var cmd = new ExecuteCommandChangeExtractionCategory(new ThrowImmediatelyActivator(RepositoryLocator), eid, ExtractionCategory.Core); + Assert.DoesNotThrow(() => cmd.Execute()); + var x = CatalogueRepository.GetAllObjects(); + Assert.That(_extractionInfo1.ExtractionCategory, Is.EqualTo(ExtractionCategory.ProjectSpecific)); + } +} diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractionCategory.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractionCategory.cs index ba2a60339c..700004f96c 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractionCategory.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractionCategory.cs @@ -54,21 +54,21 @@ public override void Execute() base.Execute(); var c = _category; - if (c == null && BasicActivator.SelectValueType("New Extraction Category", typeof(ExtractionCategory), ExtractionCategory.Core, out var category)) + { c = (ExtractionCategory)category; + } if (c == null) return; - if (_isProjectSpecific && c == ExtractionCategory.Core) { // Don't allow project specific catalogue items to become core c = ExtractionCategory.ProjectSpecific; Show("Cannot set the Extraction Category to 'Core' for a Project Specific Catalogue item. It will be saved as 'Project Specific'."); } - if(c == _category) return;//no commit needed + //if (c == _category) return;//no commit needed if (ExecuteWithCommit(() => ExecuteImpl(c.Value), $"Set ExtractionCategory to '{c}'", _extractionInformations)) //publish the root Catalogue From a9e40e2203997542dfe261f21f605d4d2176a29a Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 24 Jan 2024 10:13:02 +0000 Subject: [PATCH 09/27] tidy up code --- .../ExecuteCommandChangeExtractionCategoryTests.cs | 8 -------- .../ExecuteCommandChangeExtractionCategory.cs | 1 - 2 files changed, 9 deletions(-) diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandChangeExtractionCategoryTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandChangeExtractionCategoryTests.cs index 5ed73f0cce..6abab54986 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandChangeExtractionCategoryTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandChangeExtractionCategoryTests.cs @@ -21,17 +21,11 @@ namespace Rdmp.Core.Tests.CommandExecution; public class ExecuteCommandChangeExtractionCategoryTests : DatabaseTests { private Catalogue _cata1; - private Catalogue _cata2; private TableInfo _t1; - private TableInfo _t2; private ColumnInfo _c1; - private ColumnInfo _c2; private CatalogueItem _ci1; - private CatalogueItem _ci2; - private Project _project; private ExtractionInformation _extractionInfo1; - private ExtractionInformation _extractionInfo2; [Test] public void TestProjectSpecificCatalogueChangeToSuplemental() @@ -69,7 +63,6 @@ public void TestExtractionCategoryCatalogueChangeFromSupplementalToCore() _ci1 = new CatalogueItem(CatalogueRepository, _cata1, "PrivateIdentifierA"); - _extractionInfo1 = new ExtractionInformation(CatalogueRepository, _ci1, _c1, _c1.ToString()) { Order = 123, @@ -81,7 +74,6 @@ public void TestExtractionCategoryCatalogueChangeFromSupplementalToCore() ExtractionInformation[] eid = { _extractionInfo1 }; var cmd = new ExecuteCommandChangeExtractionCategory(new ThrowImmediatelyActivator(RepositoryLocator), eid, ExtractionCategory.Core); Assert.DoesNotThrow(() => cmd.Execute()); - var x = CatalogueRepository.GetAllObjects(); Assert.That(_extractionInfo1.ExtractionCategory, Is.EqualTo(ExtractionCategory.ProjectSpecific)); } } diff --git a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractionCategory.cs b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractionCategory.cs index 700004f96c..f8a80e4f55 100644 --- a/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractionCategory.cs +++ b/Rdmp.Core/CommandExecution/AtomicCommands/ExecuteCommandChangeExtractionCategory.cs @@ -68,7 +68,6 @@ public override void Execute() c = ExtractionCategory.ProjectSpecific; Show("Cannot set the Extraction Category to 'Core' for a Project Specific Catalogue item. It will be saved as 'Project Specific'."); } - //if (c == _category) return;//no commit needed if (ExecuteWithCommit(() => ExecuteImpl(c.Value), $"Set ExtractionCategory to '{c}'", _extractionInformations)) //publish the root Catalogue From 0d50e9cbd4fbb51f46d6c0c40ad442c968db7666 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 25 Jan 2024 16:36:04 +0000 Subject: [PATCH 10/27] backout logging to file (#1734) --- Rdmp.Core/Logging/DataLoadInfo.cs | 52 ++++++----------- Rdmp.Core/Logging/FileSystemLogger.cs | 56 ------------------- Rdmp.Core/Logging/TableLoadInfo.cs | 52 +++++++---------- .../Settings/UserSettings.cs | 18 ------ .../SimpleDialogs/UserSettingsUI.Designer.cs | 45 +++++---------- Rdmp.UI/SimpleDialogs/UserSettingsUI.cs | 10 ---- 6 files changed, 53 insertions(+), 180 deletions(-) delete mode 100644 Rdmp.Core/Logging/FileSystemLogger.cs diff --git a/Rdmp.Core/Logging/DataLoadInfo.cs b/Rdmp.Core/Logging/DataLoadInfo.cs index f21a7be030..5fb52e84a7 100644 --- a/Rdmp.Core/Logging/DataLoadInfo.cs +++ b/Rdmp.Core/Logging/DataLoadInfo.cs @@ -201,7 +201,7 @@ public void CloseAndMarkComplete() { lock (_oLock) { - if (_logQueue?.IsAddingCompleted==false) + if (_logQueue?.IsAddingCompleted == false) _logQueue.CompleteAdding(); _logThread?.Join(); @@ -296,24 +296,16 @@ public void LogFatalError(string errorSource, string errorDescription) var statusID = int.Parse(cmdLookupStatusID.ExecuteScalar().ToString()); - if (UserSettings.LogToFileSystem && !string.IsNullOrWhiteSpace(UserSettings.FileSystemLogLocation)) - { - var logger = FileSystemLogger.Instance; - logger.LogEventToFile(FileSystemLogger.AvailableLoggers.FatalError, [errorSource,errorDescription, statusID, ID]); - } - else - { - var cmdRecordFatalError = DatabaseSettings.GetCommand( - @"INSERT INTO FatalError (time,source,description,statusID,dataLoadRunID) VALUES (@time,@source,@description,@statusID,@dataLoadRunID);", - con); - DatabaseSettings.AddParameterWithValueToCommand("@time", cmdRecordFatalError, DateTime.Now); - DatabaseSettings.AddParameterWithValueToCommand("@source", cmdRecordFatalError, errorSource); - DatabaseSettings.AddParameterWithValueToCommand("@description", cmdRecordFatalError, errorDescription); - DatabaseSettings.AddParameterWithValueToCommand("@statusID", cmdRecordFatalError, statusID); - DatabaseSettings.AddParameterWithValueToCommand("@dataLoadRunID", cmdRecordFatalError, ID); + var cmdRecordFatalError = DatabaseSettings.GetCommand( + @"INSERT INTO FatalError (time,source,description,statusID,dataLoadRunID) VALUES (@time,@source,@description,@statusID,@dataLoadRunID);", + con); + DatabaseSettings.AddParameterWithValueToCommand("@time", cmdRecordFatalError, DateTime.Now); + DatabaseSettings.AddParameterWithValueToCommand("@source", cmdRecordFatalError, errorSource); + DatabaseSettings.AddParameterWithValueToCommand("@description", cmdRecordFatalError, errorDescription); + DatabaseSettings.AddParameterWithValueToCommand("@statusID", cmdRecordFatalError, statusID); + DatabaseSettings.AddParameterWithValueToCommand("@dataLoadRunID", cmdRecordFatalError, ID); - cmdRecordFatalError.ExecuteNonQuery(); - } + cmdRecordFatalError.ExecuteNonQuery(); //this might get called multiple times (many errors in rapid succession as the program crashes) but only close the dataLoadInfo once if (!IsClosed) @@ -331,24 +323,16 @@ public enum ProgressEventType public void LogProgress(ProgressEventType pevent, string Source, string Description) { - if (UserSettings.LogToFileSystem && !string.IsNullOrWhiteSpace(UserSettings.FileSystemLogLocation)) - { - var logger = FileSystemLogger.Instance; - logger.LogEventToFile(FileSystemLogger.AvailableLoggers.ProgressLog, [ID, pevent, Source, Description]); - } - else + lock (_oLock) { - lock (_oLock) - { - if (_logQueue is null || _logQueue.IsAddingCompleted) - LogInit(); - if (_logQueue is null) - throw new InvalidOperationException("LogInit failed to create new worker"); + if (_logQueue is null || _logQueue.IsAddingCompleted) + LogInit(); + if (_logQueue is null) + throw new InvalidOperationException("LogInit failed to create new worker"); - _logQueue.Add(new LogEntry(pevent.ToString(), Description, Source, DateTime.Now)); - } - lock (_logWaiter) - Monitor.Pulse(_logWaiter); + _logQueue.Add(new LogEntry(pevent.ToString(), Description, Source, DateTime.Now)); } + lock (_logWaiter) + Monitor.Pulse(_logWaiter); } } diff --git a/Rdmp.Core/Logging/FileSystemLogger.cs b/Rdmp.Core/Logging/FileSystemLogger.cs deleted file mode 100644 index 4521d2693a..0000000000 --- a/Rdmp.Core/Logging/FileSystemLogger.cs +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright (c) The University of Dundee 2024-2024 -// This file is part of the Research Data Management Platform (RDMP). -// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. -// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -// You should have received a copy of the GNU General Public License along with RDMP. If not, see . - -using NLog; -using Rdmp.Core.ReusableLibraryCode.Settings; -using System; -using System.IO; - -namespace Rdmp.Core.Logging; - - -/// -/// Singleton logger for writing logs to file -/// -public class FileSystemLogger -{ - - private static readonly Lazy lazy = - new Lazy(() => new FileSystemLogger()); - - public static FileSystemLogger Instance { get { return lazy.Value; } } - - - public enum AvailableLoggers - { - ProgressLog, - DataSource, - FatalError - } - - - - private FileSystemLogger() - { - var location = UserSettings.FileSystemLogLocation; - var config = new NLog.Config.LoggingConfiguration(); - string[] logs = System.Enum.GetNames(typeof(AvailableLoggers)); - foreach (var log in logs){ - using var nLogEntry = new NLog.Targets.FileTarget(log) { FileName = Path.Combine(location, $"{log}.log"), ArchiveAboveSize = UserSettings.LogFileSizeLimit }; - config.AddRule(LogLevel.Info, LogLevel.Info, nLogEntry); - } - NLog.LogManager.Configuration = config; - } - - public void LogEventToFile(AvailableLoggers logType, object[] logItems) - { - var logMessage = $"{string.Join("|", Array.ConvertAll(logItems, item => item.ToString()))}"; - var Logger = NLog.LogManager.GetLogger(Enum.GetName(typeof(AvailableLoggers), logType)); - Logger.Info(logMessage); - } - - -} \ No newline at end of file diff --git a/Rdmp.Core/Logging/TableLoadInfo.cs b/Rdmp.Core/Logging/TableLoadInfo.cs index 40c20e41eb..4cc6485969 100644 --- a/Rdmp.Core/Logging/TableLoadInfo.cs +++ b/Rdmp.Core/Logging/TableLoadInfo.cs @@ -88,7 +88,7 @@ private void RecordNewTableLoadInDatabase(DataLoadInfo parent, string destinatio DatabaseSettings.AddParameterWithValueToCommand("@startTime", cmd, DateTime.Now); DatabaseSettings.AddParameterWithValueToCommand("@dataLoadRunID", cmd, parent.ID); - DatabaseSettings.AddParameterWithValueToCommand("@targetTable", cmd, destinationTable.Substring(Math.Max(0,destinationTable.Length - 200))); //200 char limit on this table, just pull the rightmost 200 chars + DatabaseSettings.AddParameterWithValueToCommand("@targetTable", cmd, destinationTable.Substring(Math.Max(0, destinationTable.Length - 200))); //200 char limit on this table, just pull the rightmost 200 chars DatabaseSettings.AddParameterWithValueToCommand("@expectedInserts", cmd, expectedInserts); DatabaseSettings.AddParameterWithValueToCommand("@suggestedRollbackCommand", cmd, _suggestedRollbackCommand); @@ -103,42 +103,32 @@ private void RecordNewTableLoadInDatabase(DataLoadInfo parent, string destinatio //for each of the sources, create them in the DataSource table foreach (var s in DataSources) { - if (UserSettings.LogToFileSystem && !string.IsNullOrWhiteSpace(UserSettings.FileSystemLogLocation)) + using var cmdInsertDs = DatabaseSettings.GetCommand( + "INSERT INTO DataSource (source,tableLoadRunID,originDate,MD5) " + + "VALUES (@source,@tableLoadRunID,@originDate,@MD5); SELECT @@IDENTITY;", con); + DatabaseSettings.AddParameterWithValueToCommand("@source", cmdInsertDs, s.Source); + DatabaseSettings.AddParameterWithValueToCommand("@tableLoadRunID", cmdInsertDs, _id); + DatabaseSettings.AddParameterWithValueToCommand("@originDate", cmdInsertDs, + s.UnknownOriginDate ? DBNull.Value : s.OriginDate); + + // old logging schema used binary[128] for the MD5 column + if (IsLegacyLoggingSchema) { - var logger = FileSystemLogger.Instance; - logger.LogEventToFile(FileSystemLogger.AvailableLoggers.DataSource, [s.Source,_id ,s.UnknownOriginDate ? "" : s.OriginDate]); + var p = cmdInsertDs.CreateParameter(); + p.DbType = DbType.Binary; + p.Size = 128; + p.Value = s.MD5 != null ? s.MD5 : DBNull.Value; + p.ParameterName = "@MD5"; + cmdInsertDs.Parameters.Add(p); } else { - - using var cmdInsertDs = DatabaseSettings.GetCommand( - "INSERT INTO DataSource (source,tableLoadRunID,originDate,MD5) " + - "VALUES (@source,@tableLoadRunID,@originDate,@MD5); SELECT @@IDENTITY;", con); - DatabaseSettings.AddParameterWithValueToCommand("@source", cmdInsertDs, s.Source); - DatabaseSettings.AddParameterWithValueToCommand("@tableLoadRunID", cmdInsertDs, _id); - DatabaseSettings.AddParameterWithValueToCommand("@originDate", cmdInsertDs, - s.UnknownOriginDate ? DBNull.Value : s.OriginDate); - - // old logging schema used binary[128] for the MD5 column - if (IsLegacyLoggingSchema) - { - var p = cmdInsertDs.CreateParameter(); - p.DbType = DbType.Binary; - p.Size = 128; - p.Value = s.MD5 != null ? s.MD5 : DBNull.Value; - p.ParameterName = "@MD5"; - cmdInsertDs.Parameters.Add(p); - } - else - { - // now logging schema uses string for easier usability and FAnsiSql compatibility - DatabaseSettings.AddParameterWithValueToCommand("@MD5", cmdInsertDs, - s.MD5 != null ? s.MD5 : DBNull.Value); - } - - s.ID = int.Parse(cmdInsertDs.ExecuteScalar().ToString()); + // now logging schema uses string for easier usability and FAnsiSql compatibility + DatabaseSettings.AddParameterWithValueToCommand("@MD5", cmdInsertDs, + s.MD5 != null ? s.MD5 : DBNull.Value); } + s.ID = int.Parse(cmdInsertDs.ExecuteScalar().ToString()); } } diff --git a/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs b/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs index 0b9a8223df..94e5bd34a0 100644 --- a/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs +++ b/Rdmp.Core/ReusableLibraryCode/Settings/UserSettings.cs @@ -34,24 +34,6 @@ public static bool UseLocalFileSystem set => AppSettings.AddOrUpdateValue("UseLocalFileSystem", value); } - public static bool LogToFileSystem - { - get => AppSettings.GetValueOrDefault("LogToFileSystem", false); - set => AppSettings.AddOrUpdateValue("LogToFileSystem", value); - } - - public static long LogFileSizeLimit - { - get => AppSettings.GetValueOrDefault("LogFileSizeLimit", 1073741824); - set => AppSettings.AddOrUpdateValue("LogFileSizeLimit", value); - } - - public static string FileSystemLogLocation - { - get => AppSettings.GetValueOrDefault("FileSystemLogLocation", null); - set => AppSettings.AddOrUpdateValue("FileSystemLogLocation", value); - } - public static string LocalFileSystemLocation { get => AppSettings.GetValueOrDefault("LocalFileSystemLocation", Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "rdmp")); diff --git a/Rdmp.UI/SimpleDialogs/UserSettingsUI.Designer.cs b/Rdmp.UI/SimpleDialogs/UserSettingsUI.Designer.cs index 200a9fd928..2de66db057 100644 --- a/Rdmp.UI/SimpleDialogs/UserSettingsUI.Designer.cs +++ b/Rdmp.UI/SimpleDialogs/UserSettingsUI.Designer.cs @@ -75,7 +75,8 @@ private void InitializeComponent() groupBox6 = new System.Windows.Forms.GroupBox(); cbSkipCohortBuilderValidationOnCommit = new System.Windows.Forms.CheckBox(); groupBox7 = new System.Windows.Forms.GroupBox(); - cbUseLogFiles = new System.Windows.Forms.CheckBox(); + label16 = new System.Windows.Forms.Label(); + tbLogLocation = new System.Windows.Forms.TextBox(); label15 = new System.Windows.Forms.Label(); tbLocalFileSystemLocation = new System.Windows.Forms.TextBox(); cbUseLocalFileSystem = new System.Windows.Forms.CheckBox(); @@ -92,8 +93,6 @@ private void InitializeComponent() userSettingsToolTips = new System.Windows.Forms.ToolTip(components); tbFind = new System.Windows.Forms.TextBox(); label14 = new System.Windows.Forms.Label(); - tbLogLocation = new System.Windows.Forms.TextBox(); - label16 = new System.Windows.Forms.Label(); groupBox1.SuspendLayout(); ((System.ComponentModel.ISupportInitialize)olvErrorCodes).BeginInit(); groupBox2.SuspendLayout(); @@ -578,7 +577,6 @@ private void InitializeComponent() // groupBox7.Controls.Add(label16); groupBox7.Controls.Add(tbLogLocation); - groupBox7.Controls.Add(cbUseLogFiles); groupBox7.Controls.Add(label15); groupBox7.Controls.Add(tbLocalFileSystemLocation); groupBox7.Controls.Add(cbUseLocalFileSystem); @@ -604,15 +602,19 @@ private void InitializeComponent() groupBox7.TabStop = false; groupBox7.Text = "Miscellaneous"; // - // cbUseLogFiles + // label16 + // + label16.Location = new System.Drawing.Point(0, 0); + label16.Name = "label16"; + label16.Size = new System.Drawing.Size(100, 23); + label16.TabIndex = 0; + // + // tbLogLocation // - cbUseLogFiles.AutoSize = true; - cbUseLogFiles.Location = new System.Drawing.Point(7, 257); - cbUseLogFiles.Name = "cbUseLogFiles"; - cbUseLogFiles.Size = new System.Drawing.Size(94, 19); - cbUseLogFiles.TabIndex = 29; - cbUseLogFiles.Text = "Use Log Files"; - cbUseLogFiles.UseVisualStyleBackColor = true; + tbLogLocation.Location = new System.Drawing.Point(0, 0); + tbLogLocation.Name = "tbLogLocation"; + tbLogLocation.Size = new System.Drawing.Size(100, 23); + tbLogLocation.TabIndex = 1; // // label15 // @@ -764,24 +766,6 @@ private void InitializeComponent() label14.TabIndex = 26; label14.Text = "Find Setting:"; // - // tbLogLocation - // - tbLogLocation.Location = new System.Drawing.Point(156, 257); - tbLogLocation.Name = "tbLogLocation"; - tbLogLocation.Size = new System.Drawing.Size(110, 23); - tbLogLocation.TabIndex = 30; - tbLogLocation.TextChanged += tbLogLocation_TextChanged; - // - // label16 - // - label16.AutoSize = true; - label16.Location = new System.Drawing.Point(273, 261); - label16.Name = "label16"; - label16.Size = new System.Drawing.Size(53, 15); - label16.TabIndex = 31; - label16.Text = "Location"; - label16.Click += label16_Click; - // // UserSettingsFileUI // AutoScaleDimensions = new System.Drawing.SizeF(7F, 15F); @@ -884,7 +868,6 @@ private void InitializeComponent() private System.Windows.Forms.Label label15; private System.Windows.Forms.TextBox tbLocalFileSystemLocation; private System.Windows.Forms.CheckBox cbUseLocalFileSystem; - private System.Windows.Forms.CheckBox cbUseLogFiles; private System.Windows.Forms.Label label16; private System.Windows.Forms.TextBox tbLogLocation; } diff --git a/Rdmp.UI/SimpleDialogs/UserSettingsUI.cs b/Rdmp.UI/SimpleDialogs/UserSettingsUI.cs index f962ef3b99..d69e474228 100644 --- a/Rdmp.UI/SimpleDialogs/UserSettingsUI.cs +++ b/Rdmp.UI/SimpleDialogs/UserSettingsUI.cs @@ -106,7 +106,6 @@ public UserSettingsFileUI(IActivateItems activator) RegisterCheckbox(cbUseAliasInsteadOfTransformInGroupByAggregateGraphs, nameof(UserSettings.UseAliasInsteadOfTransformInGroupByAggregateGraphs)); RegisterCheckbox(cbUseLocalFileSystem, nameof(UserSettings.UseLocalFileSystem)); - RegisterCheckbox(cbUseLogFiles, nameof(UserSettings.LogToFileSystem)); AddTooltip(label7, nameof(UserSettings.CreateDatabaseTimeout)); AddTooltip(tbCreateDatabaseTimeout, nameof(UserSettings.CreateDatabaseTimeout)); AddTooltip(label13, nameof(UserSettings.ArchiveTriggerTimeout)); @@ -140,8 +139,6 @@ public UserSettingsFileUI(IActivateItems activator) ddWordWrap.SelectedItem = (WrapMode)UserSettings.WrapMode; tbHeatmapColours.Text = UserSettings.HeatMapColours; - tbLocalFileSystemLocation.Text = UserSettings.LocalFileSystemLocation; - tbLogLocation.Text = UserSettings.FileSystemLogLocation; _bLoaded = true; @@ -253,11 +250,6 @@ private void tbLocalFileSystemLocation_TextChanged(object sender, EventArgs e) UserSettings.LocalFileSystemLocation = tbLocalFileSystemLocation.Text; } - private void tbLogLocation_TextChanged(object sender, EventArgs e) - { - UserSettings.FileSystemLogLocation = tbLogLocation.Text; - } - private void tbTooltipAppearDelay_TextChanged(object sender, EventArgs e) { if (int.TryParse(tbTooltipAppearDelay.Text, out var result)) UserSettings.TooltipAppearDelay = result; @@ -268,8 +260,6 @@ private void tbFind_TextChanged(object sender, EventArgs e) Find(tbFind.Text); } - private void label16_Click(object sender, EventArgs e) { } - private void Find(string text) { foreach (var cb in checkboxDictionary) From dacd140307321a5c2558d0b12c101af753eefd7a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jan 2024 01:15:54 +0000 Subject: [PATCH 11/27] Bump NUnit.Analyzers from 3.10.0 to 4.0.0 Bumps [NUnit.Analyzers](https://github.com/nunit/nunit.analyzers) from 3.10.0 to 4.0.0. - [Release notes](https://github.com/nunit/nunit.analyzers/releases) - [Changelog](https://github.com/nunit/nunit.analyzers/blob/master/CHANGES.txt) - [Commits](https://github.com/nunit/nunit.analyzers/compare/3.10.0...4.0.0) --- updated-dependencies: - dependency-name: NUnit.Analyzers dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- Rdmp.Core.Tests/Rdmp.Core.Tests.csproj | 2 +- Rdmp.UI.Tests/Rdmp.UI.Tests.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj b/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj index aa3f251a8f..8c8e33a4fb 100644 --- a/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj +++ b/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj @@ -72,7 +72,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Rdmp.UI.Tests/Rdmp.UI.Tests.csproj b/Rdmp.UI.Tests/Rdmp.UI.Tests.csproj index b0dd70cb99..7c2be20304 100644 --- a/Rdmp.UI.Tests/Rdmp.UI.Tests.csproj +++ b/Rdmp.UI.Tests/Rdmp.UI.Tests.csproj @@ -19,7 +19,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive From 9a62f2c99a28ae9f300482e74c95de3002e3793b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jan 2024 01:13:13 +0000 Subject: [PATCH 12/27] Bump FluentFTP from 49.0.1 to 49.0.2 Bumps [FluentFTP](https://github.com/robinrodricks/FluentFTP) from 49.0.1 to 49.0.2. - [Release notes](https://github.com/robinrodricks/FluentFTP/releases) - [Changelog](https://github.com/robinrodricks/FluentFTP/blob/master/RELEASES.md) - [Commits](https://github.com/robinrodricks/FluentFTP/commits) --- updated-dependencies: - dependency-name: FluentFTP dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Rdmp.Core/Rdmp.Core.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index b0832d1edc..22f1e5afdd 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -298,7 +298,7 @@ - + From e461ffe29b729056cd017fa6be3ed98ca0a3712b Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 29 Jan 2024 15:10:53 +0000 Subject: [PATCH 13/27] Task/rdmp 118 Allow Excel Attacher To Read From Arbitrary Start Point (#1735) * add excel attacher tests * add excel attacher documentation --- CHANGELOG.md | 2 +- Documentation/DataLoadEngine/ExcelAttacher.md | 28 ++ .../Engine/Integration/ExcelAttacherTests.cs | 475 ++++++++++++++++++ .../Modules/Attachers/ExcelAttacher.cs | 29 +- .../DataFlowSources/ExcelDataFlowSource.cs | 31 +- 5 files changed, 554 insertions(+), 11 deletions(-) create mode 100644 Documentation/DataLoadEngine/ExcelAttacher.md create mode 100644 Rdmp.Core.Tests/DataLoad/Engine/Integration/ExcelAttacherTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dc054034a..f71f0f9803 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Changed -- Add ability to log to file instead of database +- Update Excel Attacher to read data from arbitrary start points within sheets ## [8.1.3] - 2024-01-15 diff --git a/Documentation/DataLoadEngine/ExcelAttacher.md b/Documentation/DataLoadEngine/ExcelAttacher.md new file mode 100644 index 0000000000..3437333e1e --- /dev/null +++ b/Documentation/DataLoadEngine/ExcelAttacher.md @@ -0,0 +1,28 @@ +## Excel Attacher +The Excel attacher is a convenient way to load your data from .xlsx files into RDMP. +It works out of the box with a single-sheet workbook. + +### Simple Example +For example, the following Excel sheet would be transformed into a single database table with columns "COLUMN_1" and "COLUMN_2" +With a single entry ("some data","some other data"). + +| | A | B | +|---|-----------|-----------------| +| 1 | COLUMN_1 | COLUMN_2 | +| 2 | some data | some other data | + + +## Configurable options +* Work Sheet Name - the name of the sheet you wish to import, only required if your Excel workbook has multiple sheets +* Add Filename Column Named - (Optional) If you want to store the source file location in the database, add the name of a new column to store this information in +* Force Replacement Headers - A comma separated list of headers to replace those found in your sheet +* Allow Extra Columns In Target Without Complaining of Column Mismatch - Allows for missing columns in the source file +* File Pattern - the file you wish to load +* Table To Load / Table Name - Which Table you wish to populate +* Send Load Not Required If File Not Found - If there is no file, we can skip the load +* Delay Load Failures - Wait till load complete to fail about missing files +* Culture / Explicit Date Time Format - Set the time format of your data +* Row Offset - If your data doesn't start on the first row, set the Row Offset to the row that contains your data's headers. The first row is 1. +* Column Offset - If your data doesn't start on the first column, set the Column Offset to the column that contains your data's headers. The first Column is 'A' or '0'. + + diff --git a/Rdmp.Core.Tests/DataLoad/Engine/Integration/ExcelAttacherTests.cs b/Rdmp.Core.Tests/DataLoad/Engine/Integration/ExcelAttacherTests.cs new file mode 100644 index 0000000000..b58e4500f2 --- /dev/null +++ b/Rdmp.Core.Tests/DataLoad/Engine/Integration/ExcelAttacherTests.cs @@ -0,0 +1,475 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using FAnsi; +using FAnsi.Discovery; +using NPOI.SS.Formula.Functions; +using NPOI.SS.UserModel; +using NPOI.XSSF.UserModel; +using NUnit.Framework; +using Rdmp.Core.Curation; +using Rdmp.Core.DataFlowPipeline; +using Rdmp.Core.DataLoad; +using Rdmp.Core.DataLoad.Engine.Job; +using Rdmp.Core.DataLoad.Modules.Attachers; +using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Collections.Generic; +using System.Drawing; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Tests.Common; + +namespace Rdmp.Core.Tests.DataLoad.Engine.Integration; + +public class ExcelAttacherTests : DatabaseTests +{ + private IWorkbook _workbook; + private LoadDirectory _loadDirectory; + private DirectoryInfo _parentDir; + private DiscoveredDatabase _database; + private DiscoveredTable _table; + private string _filename; + + + [OneTimeSetUp] + protected void Setup() + { + base.SetUp(); + + var workingDir = new DirectoryInfo(TestContext.CurrentContext.TestDirectory); + _parentDir = workingDir.CreateSubdirectory("ExcelAttacherTests"); + _database = GetCleanedServer(DatabaseType.MicrosoftSQLServer); + + var toCleanup = _parentDir.GetDirectories().SingleOrDefault(d => d.Name.Equals("EXCEL_ATTACHER")); + toCleanup?.Delete(true); + + _loadDirectory = LoadDirectory.CreateDirectoryStructure(_parentDir, "EXCEL_ATTACHER"); + + using var con = _database.Server.GetConnection(); + con.Open(); + + var cmdCreateTable = _database.Server.GetCommand( + $"CREATE Table {_database.GetRuntimeName()}..ExcelAttacher([chi] [varchar](500),[value] [varchar](500))", con); + cmdCreateTable.ExecuteNonQuery(); + + + _table = _database.ExpectTable("ExcelAttacher"); + } + + [TearDown] + public void CleanUp() + { + _workbook.Dispose(); + File.Delete(_filename); + } + + [Test] + public void Test_BasicExcelAttacher() + { + _workbook = new XSSFWorkbook(); + ISheet sheet = _workbook.CreateSheet("Sheet1"); + IRow row = sheet.CreateRow(0); + row.CreateCell(0).SetCellValue("chi"); + row.CreateCell(1).SetCellValue("value"); + row = sheet.CreateRow(1); + row.CreateCell(0).SetCellValue("1111111111"); + row.CreateCell(1).SetCellValue("some value"); + row = sheet.CreateRow(2); + row.CreateCell(0).SetCellValue("2222222222"); + row.CreateCell(1).SetCellValue("some other value"); + _filename = Path.Combine(_loadDirectory.ForLoading.FullName, "ExcelAttacher.xlsx"); + using var fs = new FileStream(_filename, FileMode.Create, FileAccess.Write); + _workbook.Write(fs); + + var attacher = new ExcelAttacher(); + attacher.Initialize(_loadDirectory, _database); + attacher.FilePattern = "ExcelAttacher*"; + attacher.TableName = "ExcelAttacher"; + Assert.DoesNotThrow(() => attacher.Attach(new ThrowImmediatelyDataLoadJob(), new GracefulCancellationToken())); + var table = _database.ExpectTable("ExcelAttacher"); + Assert.That(table.Exists()); + using var con = _database.Server.GetConnection(); + con.Open(); + var reader = _database.Server.GetCommand("Select * from ExcelAttacher", con).ExecuteReader(); + Assert.Multiple(() => + { + Assert.That(reader.Read()); + Assert.That(reader["chi"], Is.EqualTo("1111111111")); + Assert.That(reader["value"], Is.EqualTo("some value")); + }); + Assert.Multiple(() => + { + Assert.That(reader.Read()); + Assert.That(reader["chi"], Is.EqualTo("2222222222")); + Assert.That(reader["value"], Is.EqualTo("some other value")); + }); + attacher.LoadCompletedSoDispose(ExitCodeType.Success, ThrowImmediatelyDataLoadEventListener.Quiet); + } + + [Test] + public void Test_BasicExcelAttacherWithBlankFirstColumn() + { + _workbook = new XSSFWorkbook(); + ISheet sheet = _workbook.CreateSheet("Sheet1"); + IRow row = sheet.CreateRow(0); + row.CreateCell(1).SetCellValue("chi"); + row.CreateCell(2).SetCellValue("value"); + row = sheet.CreateRow(1); + row.CreateCell(1).SetCellValue("1111111111"); + row.CreateCell(2).SetCellValue("some value"); + row = sheet.CreateRow(2); + row.CreateCell(1).SetCellValue("2222222222"); + row.CreateCell(2).SetCellValue("some other value"); + _filename = Path.Combine(_loadDirectory.ForLoading.FullName, "ExcelAttacher.xlsx"); + using var fs = new FileStream(_filename, FileMode.Create, FileAccess.Write); + _workbook.Write(fs); + + var attacher = new ExcelAttacher(); + attacher.Initialize(_loadDirectory, _database); + attacher.FilePattern = "ExcelAttacher*"; + attacher.TableName = "ExcelAttacher"; + Assert.DoesNotThrow(() => attacher.Attach(new ThrowImmediatelyDataLoadJob(), new GracefulCancellationToken())); + var table = _database.ExpectTable("ExcelAttacher"); + Assert.That(table.Exists()); + using var con = _database.Server.GetConnection(); + con.Open(); + var reader = _database.Server.GetCommand("Select * from ExcelAttacher", con).ExecuteReader(); + Assert.Multiple(() => + { + Assert.That(reader.Read()); + Assert.That(reader["chi"], Is.EqualTo("1111111111")); + Assert.That(reader["value"], Is.EqualTo("some value")); + }); + Assert.Multiple(() => + { + Assert.That(reader.Read()); + Assert.That(reader["chi"], Is.EqualTo("2222222222")); + Assert.That(reader["value"], Is.EqualTo("some other value")); + }); + attacher.LoadCompletedSoDispose(ExitCodeType.Success, ThrowImmediatelyDataLoadEventListener.Quiet); + } + + [Test] + public void Test_BasicExcelAttacherWithBlankFirstColumnAndRow() + { + _workbook = new XSSFWorkbook(); + ISheet sheet = _workbook.CreateSheet("Sheet1"); + IRow row = sheet.CreateRow(1); + row.CreateCell(1).SetCellValue("chi"); + row.CreateCell(2).SetCellValue("value"); + row = sheet.CreateRow(2); + row.CreateCell(1).SetCellValue("1111111111"); + row.CreateCell(2).SetCellValue("some value"); + row = sheet.CreateRow(3); + row.CreateCell(1).SetCellValue("2222222222"); + row.CreateCell(2).SetCellValue("some other value"); + _filename = Path.Combine(_loadDirectory.ForLoading.FullName, "ExcelAttacher.xlsx"); + using var fs = new FileStream(_filename, FileMode.Create, FileAccess.Write); + _workbook.Write(fs); + + var attacher = new ExcelAttacher(); + attacher.Initialize(_loadDirectory, _database); + attacher.FilePattern = "ExcelAttacher*"; + attacher.TableName = "ExcelAttacher"; + Assert.DoesNotThrow(() => attacher.Attach(new ThrowImmediatelyDataLoadJob(), new GracefulCancellationToken())); + var table = _database.ExpectTable("ExcelAttacher"); + Assert.That(table.Exists()); + using var con = _database.Server.GetConnection(); + con.Open(); + var reader = _database.Server.GetCommand("Select * from ExcelAttacher", con).ExecuteReader(); + Assert.Multiple(() => + { + Assert.That(reader.Read()); + Assert.That(reader["chi"], Is.EqualTo("1111111111")); + Assert.That(reader["value"], Is.EqualTo("some value")); + }); + Assert.Multiple(() => + { + Assert.That(reader.Read()); + Assert.That(reader["chi"], Is.EqualTo("2222222222")); + Assert.That(reader["value"], Is.EqualTo("some other value")); + }); + attacher.LoadCompletedSoDispose(ExitCodeType.Success, ThrowImmediatelyDataLoadEventListener.Quiet); + } + + [Test] + public void Test_BasicExcelAttacherWithBlankFirstColumnAndRowAndUnnecessaryOverwrite_ColumnOnly_Number() + { + _workbook = new XSSFWorkbook(); + ISheet sheet = _workbook.CreateSheet("Sheet1"); + IRow row = sheet.CreateRow(1); + row.CreateCell(1).SetCellValue("chi"); + row.CreateCell(2).SetCellValue("value"); + row = sheet.CreateRow(2); + row.CreateCell(1).SetCellValue("1111111111"); + row.CreateCell(2).SetCellValue("some value"); + row = sheet.CreateRow(3); + row.CreateCell(1).SetCellValue("2222222222"); + row.CreateCell(2).SetCellValue("some other value"); + _filename = Path.Combine(_loadDirectory.ForLoading.FullName, "ExcelAttacher.xlsx"); + using var fs = new FileStream(_filename, FileMode.Create, FileAccess.Write); + _workbook.Write(fs); + + var attacher = new ExcelAttacher(); + attacher.Initialize(_loadDirectory, _database); + attacher.FilePattern = "ExcelAttacher*"; + attacher.TableName = "ExcelAttacher"; + attacher.ColumnOffset = "1"; + Assert.DoesNotThrow(() => attacher.Attach(new ThrowImmediatelyDataLoadJob(), new GracefulCancellationToken())); + var table = _database.ExpectTable("ExcelAttacher"); + Assert.That(table.Exists()); + using var con = _database.Server.GetConnection(); + con.Open(); + var reader = _database.Server.GetCommand("Select * from ExcelAttacher", con).ExecuteReader(); + Assert.Multiple(() => + { + Assert.That(reader.Read()); + Assert.That(reader["chi"], Is.EqualTo("1111111111")); + Assert.That(reader["value"], Is.EqualTo("some value")); + }); + Assert.Multiple(() => + { + Assert.That(reader.Read()); + Assert.That(reader["chi"], Is.EqualTo("2222222222")); + Assert.That(reader["value"], Is.EqualTo("some other value")); + }); + attacher.LoadCompletedSoDispose(ExitCodeType.Success, ThrowImmediatelyDataLoadEventListener.Quiet); + } + + + [Test] + public void Test_BasicExcelAttacherWithBlankFirstColumnAndRowAndUnnecessaryOverwrite_ColumnOnly_Letter() + { + _workbook = new XSSFWorkbook(); + ISheet sheet = _workbook.CreateSheet("Sheet1"); + IRow row = sheet.CreateRow(1); + row.CreateCell(1).SetCellValue("chi"); + row.CreateCell(2).SetCellValue("value"); + row = sheet.CreateRow(2); + row.CreateCell(1).SetCellValue("1111111111"); + row.CreateCell(2).SetCellValue("some value"); + row = sheet.CreateRow(3); + row.CreateCell(1).SetCellValue("2222222222"); + row.CreateCell(2).SetCellValue("some other value"); + _filename = Path.Combine(_loadDirectory.ForLoading.FullName, "ExcelAttacher.xlsx"); + using var fs = new FileStream(_filename, FileMode.Create, FileAccess.Write); + _workbook.Write(fs); + + var attacher = new ExcelAttacher(); + attacher.Initialize(_loadDirectory, _database); + attacher.FilePattern = "ExcelAttacher*"; + attacher.TableName = "ExcelAttacher"; + attacher.ColumnOffset = "B"; + Assert.DoesNotThrow(() => attacher.Attach(new ThrowImmediatelyDataLoadJob(), new GracefulCancellationToken())); + var table = _database.ExpectTable("ExcelAttacher"); + Assert.That(table.Exists()); + using var con = _database.Server.GetConnection(); + con.Open(); + var reader = _database.Server.GetCommand("Select * from ExcelAttacher", con).ExecuteReader(); + Assert.Multiple(() => + { + Assert.That(reader.Read()); + Assert.That(reader["chi"], Is.EqualTo("1111111111")); + Assert.That(reader["value"], Is.EqualTo("some value")); + }); + Assert.Multiple(() => + { + Assert.That(reader.Read()); + Assert.That(reader["chi"], Is.EqualTo("2222222222")); + Assert.That(reader["value"], Is.EqualTo("some other value")); + }); + attacher.LoadCompletedSoDispose(ExitCodeType.Success, ThrowImmediatelyDataLoadEventListener.Quiet); + } + + [Test] + public void Test_ExcelAttacherSkipFirstColumn_Letter() + { + _workbook = new XSSFWorkbook(); + ISheet sheet = _workbook.CreateSheet("Sheet1"); + IRow row = sheet.CreateRow(0); + row.CreateCell(1).SetCellValue("junk_data"); + row.CreateCell(2).SetCellValue("chi"); + row.CreateCell(3).SetCellValue("value"); + row = sheet.CreateRow(1); + row.CreateCell(1).SetCellValue("junk_data"); + row.CreateCell(2).SetCellValue("1111111111"); + row.CreateCell(3).SetCellValue("some value"); + row = sheet.CreateRow(2); + row.CreateCell(1).SetCellValue("junk_data"); + row.CreateCell(2).SetCellValue("2222222222"); + row.CreateCell(3).SetCellValue("some other value"); + _filename = Path.Combine(_loadDirectory.ForLoading.FullName, "ExcelAttacher.xlsx"); + using var fs = new FileStream(_filename, FileMode.Create, FileAccess.Write); + _workbook.Write(fs); + + var attacher = new ExcelAttacher(); + attacher.Initialize(_loadDirectory, _database); + attacher.FilePattern = "ExcelAttacher*"; + attacher.TableName = "ExcelAttacher"; + attacher.ColumnOffset = "C"; + Assert.DoesNotThrow(() => attacher.Attach(new ThrowImmediatelyDataLoadJob(), new GracefulCancellationToken())); + var table = _database.ExpectTable("ExcelAttacher"); + Assert.That(table.Exists()); + using var con = _database.Server.GetConnection(); + con.Open(); + var reader = _database.Server.GetCommand("Select * from ExcelAttacher", con).ExecuteReader(); + Assert.Multiple(() => + { + Assert.That(reader.Read()); + Assert.That(reader["chi"], Is.EqualTo("1111111111")); + Assert.That(reader["value"], Is.EqualTo("some value")); + }); + Assert.Multiple(() => + { + Assert.That(reader.Read()); + Assert.That(reader["chi"], Is.EqualTo("2222222222")); + Assert.That(reader["value"], Is.EqualTo("some other value")); + }); + attacher.LoadCompletedSoDispose(ExitCodeType.Success, ThrowImmediatelyDataLoadEventListener.Quiet); + } + + + [Test] + public void Test_ExcelAttacherSkipFirstColumn_Number() + { + _workbook = new XSSFWorkbook(); + ISheet sheet = _workbook.CreateSheet("Sheet1"); + IRow row = sheet.CreateRow(0); + row.CreateCell(1).SetCellValue("junk_data"); + row.CreateCell(2).SetCellValue("chi"); + row.CreateCell(3).SetCellValue("value"); + row = sheet.CreateRow(1); + row.CreateCell(1).SetCellValue("junk_data"); + row.CreateCell(2).SetCellValue("1111111111"); + row.CreateCell(3).SetCellValue("some value"); + row = sheet.CreateRow(2); + row.CreateCell(1).SetCellValue("junk_data"); + row.CreateCell(2).SetCellValue("2222222222"); + row.CreateCell(3).SetCellValue("some other value"); + _filename = Path.Combine(_loadDirectory.ForLoading.FullName, "ExcelAttacher.xlsx"); + using var fs = new FileStream(_filename, FileMode.Create, FileAccess.Write); + _workbook.Write(fs); + + var attacher = new ExcelAttacher(); + attacher.Initialize(_loadDirectory, _database); + attacher.FilePattern = "ExcelAttacher*"; + attacher.TableName = "ExcelAttacher"; + attacher.ColumnOffset = "2"; + Assert.DoesNotThrow(() => attacher.Attach(new ThrowImmediatelyDataLoadJob(), new GracefulCancellationToken())); + var table = _database.ExpectTable("ExcelAttacher"); + Assert.That(table.Exists()); + using var con = _database.Server.GetConnection(); + con.Open(); + var reader = _database.Server.GetCommand("Select * from ExcelAttacher", con).ExecuteReader(); + Assert.Multiple(() => + { + Assert.That(reader.Read()); + Assert.That(reader["chi"], Is.EqualTo("1111111111")); + Assert.That(reader["value"], Is.EqualTo("some value")); + }); + Assert.Multiple(() => + { + Assert.That(reader.Read()); + Assert.That(reader["chi"], Is.EqualTo("2222222222")); + Assert.That(reader["value"], Is.EqualTo("some other value")); + }); + attacher.LoadCompletedSoDispose(ExitCodeType.Success, ThrowImmediatelyDataLoadEventListener.Quiet); + } + + + [Test] + public void Test_ExcelAttacherSkipFirstColumnAndRow() + { + _workbook = new XSSFWorkbook(); + ISheet sheet = _workbook.CreateSheet("Sheet1"); + IRow row = sheet.CreateRow(0); + row.CreateCell(1).SetCellValue("junk_data"); + row.CreateCell(2).SetCellValue("junk_data"); + row.CreateCell(3).SetCellValue("junk_data"); + row = sheet.CreateRow(1); + + row.CreateCell(1).SetCellValue("junk_data"); + row.CreateCell(2).SetCellValue("chi"); + row.CreateCell(3).SetCellValue("value"); + row = sheet.CreateRow(2); + row.CreateCell(1).SetCellValue("junk_data"); + row.CreateCell(2).SetCellValue("1111111111"); + row.CreateCell(3).SetCellValue("some value"); + row = sheet.CreateRow(3); + row.CreateCell(1).SetCellValue("junk_data"); + row.CreateCell(2).SetCellValue("2222222222"); + row.CreateCell(3).SetCellValue("some other value"); + _filename = Path.Combine(_loadDirectory.ForLoading.FullName, "ExcelAttacher.xlsx"); + using var fs = new FileStream(_filename, FileMode.Create, FileAccess.Write); + _workbook.Write(fs); + + var attacher = new ExcelAttacher(); + attacher.Initialize(_loadDirectory, _database); + attacher.FilePattern = "ExcelAttacher*"; + attacher.TableName = "ExcelAttacher"; + attacher.ColumnOffset = "2"; + attacher.RowOffset = 2; + Assert.DoesNotThrow(() => attacher.Attach(new ThrowImmediatelyDataLoadJob(), new GracefulCancellationToken())); + var table = _database.ExpectTable("ExcelAttacher"); + Assert.That(table.Exists()); + using var con = _database.Server.GetConnection(); + con.Open(); + var reader = _database.Server.GetCommand("Select * from ExcelAttacher", con).ExecuteReader(); + Assert.Multiple(() => + { + Assert.That(reader.Read()); + Assert.That(reader["chi"], Is.EqualTo("1111111111")); + Assert.That(reader["value"], Is.EqualTo("some value")); + }); + Assert.Multiple(() => + { + Assert.That(reader.Read()); + Assert.That(reader["chi"], Is.EqualTo("2222222222")); + Assert.That(reader["value"], Is.EqualTo("some other value")); + }); + attacher.LoadCompletedSoDispose(ExitCodeType.Success, ThrowImmediatelyDataLoadEventListener.Quiet); + } + + [Test] + public void Test_ExcelAttacherBadColumnOffset() + { + _workbook = new XSSFWorkbook(); + ISheet sheet = _workbook.CreateSheet("Sheet1"); + IRow row = sheet.CreateRow(0); + row.CreateCell(1).SetCellValue("junk_data"); + row.CreateCell(2).SetCellValue("junk_data"); + row.CreateCell(3).SetCellValue("junk_data"); + row = sheet.CreateRow(1); + + row.CreateCell(1).SetCellValue("junk_data"); + row.CreateCell(2).SetCellValue("chi"); + row.CreateCell(3).SetCellValue("value"); + row = sheet.CreateRow(2); + row.CreateCell(1).SetCellValue("junk_data"); + row.CreateCell(2).SetCellValue("1111111111"); + row.CreateCell(3).SetCellValue("some value"); + row = sheet.CreateRow(3); + row.CreateCell(1).SetCellValue("junk_data"); + row.CreateCell(2).SetCellValue("2222222222"); + row.CreateCell(3).SetCellValue("some other value"); + _filename = Path.Combine(_loadDirectory.ForLoading.FullName, "ExcelAttacher.xlsx"); + using var fs = new FileStream(_filename, FileMode.Create, FileAccess.Write); + _workbook.Write(fs); + + var attacher = new ExcelAttacher(); + attacher.Initialize(_loadDirectory, _database); + attacher.FilePattern = "ExcelAttacher*"; + attacher.TableName = "ExcelAttacher"; + attacher.ColumnOffset = "2.987"; + attacher.RowOffset = 2; + Assert.Throws(() => attacher.Attach(new ThrowImmediatelyDataLoadJob(), new GracefulCancellationToken()),"test"); + } + + +} diff --git a/Rdmp.Core/DataLoad/Modules/Attachers/ExcelAttacher.cs b/Rdmp.Core/DataLoad/Modules/Attachers/ExcelAttacher.cs index 31e7c4e444..b3d3dea382 100644 --- a/Rdmp.Core/DataLoad/Modules/Attachers/ExcelAttacher.cs +++ b/Rdmp.Core/DataLoad/Modules/Attachers/ExcelAttacher.cs @@ -15,6 +15,7 @@ using Rdmp.Core.DataFlowPipeline.Requirements; using Rdmp.Core.DataLoad.Engine.Job; using Rdmp.Core.DataLoad.Modules.DataFlowSources; +using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; namespace Rdmp.Core.DataLoad.Modules.Attachers; @@ -43,8 +44,29 @@ public class ExcelAttacher : FlatFileAttacher "By default ALL columns in the source MUST match exactly (by name) the set of all columns in the destination table. If you enable this option then it is allowable for there to be extra columns in the destination that are not populated (because they are not found in the flat file). This does not let you discard columns from the source! (all source columns must have mappings but destination columns with no matching source are left null)")] public bool AllowExtraColumnsInTargetWithoutComplainingOfColumnMismatch { get; set; } + [DemandsInitialization("Which Row the data you want to read starts on. Set this value to row containing the header information. The First row is 1.")] + public int RowOffset { get; set; } = 0;//First row 1; + + [DemandsInitialization("Which column the data you want to read starts on. Accepts both letters & numbers, starts at 'A'/0. ")] + public string ColumnOffset { get; set; } = "A";//A=0,B=1... + private bool _haveServedData = false; + + private int ConvertColumnOffsetToInt() + { + if(int.TryParse(ColumnOffset,out var result)) return result; + if (ColumnOffset.Length == 1 && char.IsLetter(ColumnOffset[0])) return char.ToUpper(ColumnOffset[0]) - 65;//would be 64, but we index from zero here + throw new Exception("Column offset is not a valid number or letter"); + } + + + public override void Check(ICheckNotifier notifier) + { + ConvertColumnOffsetToInt(); + base.Check(notifier); + } + protected override void OpenFile(FileInfo fileToLoad, IDataLoadEventListener listener, GracefulCancellationToken cancellationToken) { @@ -60,7 +82,8 @@ protected override void OpenFile(FileInfo fileToLoad, IDataLoadEventListener lis listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"About to start processing {fileToLoad.FullName}")); - _dataTable = _hostedSource.GetChunk(listener, cancellationToken); + var columnTranslation = ConvertColumnOffsetToInt(); + _dataTable = _hostedSource.GetChunk(listener, cancellationToken, RowOffset, columnTranslation); if (!string.IsNullOrEmpty(ForceReplacementHeaders)) { @@ -78,11 +101,11 @@ protected override void OpenFile(FileInfo fileToLoad, IDataLoadEventListener lis else for (var i = 0; i < replacementHeadersSplit.Length; i++) _dataTable.Columns[i].ColumnName = - replacementHeadersSplit[i]; //rename the columns to match the forced replacments + replacementHeadersSplit[i]; //rename the columns to match the forced replacements } //all data should now be exhausted - if (_hostedSource.GetChunk(listener, cancellationToken) != null) + if (_hostedSource.GetChunk(listener, cancellationToken, RowOffset, columnTranslation) != null) throw new Exception( "Hosted source served more than 1 chunk, expected all the data to be read from the Excel file in one go"); } diff --git a/Rdmp.Core/DataLoad/Modules/DataFlowSources/ExcelDataFlowSource.cs b/Rdmp.Core/DataLoad/Modules/DataFlowSources/ExcelDataFlowSource.cs index 53d458e17b..7afdd5fcdf 100644 --- a/Rdmp.Core/DataLoad/Modules/DataFlowSources/ExcelDataFlowSource.cs +++ b/Rdmp.Core/DataLoad/Modules/DataFlowSources/ExcelDataFlowSource.cs @@ -54,9 +54,9 @@ public class ExcelDataFlowSource : IPluginDataFlowSource, IPipelineRe private DataTable dataReadFromFile; private bool haveDispatchedDataTable; - public DataTable GetChunk(IDataLoadEventListener listener, GracefulCancellationToken cancellationToken) + public DataTable GetChunk(IDataLoadEventListener listener, GracefulCancellationToken cancellationToken, int rowOffset = 0, int columnOffset = 0) { - dataReadFromFile ??= GetAllData(listener, cancellationToken); + dataReadFromFile ??= GetAllData(listener, cancellationToken,rowOffset,columnOffset); if (haveDispatchedDataTable) return null; @@ -66,7 +66,7 @@ public DataTable GetChunk(IDataLoadEventListener listener, GracefulCancellationT return dataReadFromFile; } - private DataTable GetAllData(IDataLoadEventListener listener, GracefulCancellationToken cancellationToken) + private DataTable GetAllData(IDataLoadEventListener listener, GracefulCancellationToken cancellationToken, int rowOffset=0, int columnOffset=0) { var sw = new Stopwatch(); sw.Start(); @@ -88,7 +88,7 @@ private DataTable GetAllData(IDataLoadEventListener listener, GracefulCancellati (string.IsNullOrWhiteSpace(WorkSheetName) ? wb.GetSheetAt(0) : wb.GetSheet(WorkSheetName)) ?? throw new FlatFileLoadException( $"The Excel sheet '{WorkSheetName}' was not found in workbook '{_fileToLoad.File.Name}'"); - toReturn = GetAllData(worksheet, listener); + toReturn = GetAllData(worksheet, listener, rowOffset,columnOffset); //set the table name the file name toReturn.TableName = @@ -120,8 +120,10 @@ private DataTable GetAllData(IDataLoadEventListener listener, GracefulCancellati /// /// /// + /// + /// /// - public DataTable GetAllData(ISheet worksheet, IDataLoadEventListener listener) + public DataTable GetAllData(ISheet worksheet, IDataLoadEventListener listener, int rowOffset=0, int columnOffset =0) { var toReturn = new DataTable(); toReturn.BeginLoadData(); @@ -134,7 +136,8 @@ public DataTable GetAllData(ISheet worksheet, IDataLoadEventListener listener) while (rowEnumerator.MoveNext()) { var row = (IRow)rowEnumerator.Current; - + if (rowOffset - 1 > row.RowNum) continue;// .RowNumber is 0 indexed + //if all the cells in the current row are blank skip it (eliminates top of file whitespace) if (row.Cells.All(c => string.IsNullOrWhiteSpace(c.ToString()))) continue; @@ -150,7 +153,16 @@ public DataTable GetAllData(ISheet worksheet, IDataLoadEventListener listener) { //if the cell header is blank var cell = row.Cells[i]; - var h = cell.StringCellValue; + if (cell.ColumnIndex < columnOffset) continue; + string h; + try + { + h = cell.StringCellValue; + } + catch (Exception) + { + h = cell.NumericCellValue.ToString(); + } if (string.IsNullOrWhiteSpace(h)) continue; @@ -366,4 +378,9 @@ public void PreInitialize(FlatFileToLoad value, IDataLoadEventListener listener) { _fileToLoad = value; } + + public DataTable GetChunk(IDataLoadEventListener listener, GracefulCancellationToken cancellationToken) + { + return GetChunk(listener, cancellationToken, 0, 0); + } } \ No newline at end of file From 4dbf779960295a367f3a8c62eee566272b1b28fe Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 29 Jan 2024 15:28:52 +0000 Subject: [PATCH 14/27] add form focus (#1739) --- Rdmp.UI/SimpleDialogs/WideMessageBox.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Rdmp.UI/SimpleDialogs/WideMessageBox.cs b/Rdmp.UI/SimpleDialogs/WideMessageBox.cs index d02aab349b..770aed96a3 100644 --- a/Rdmp.UI/SimpleDialogs/WideMessageBox.cs +++ b/Rdmp.UI/SimpleDialogs/WideMessageBox.cs @@ -202,6 +202,7 @@ public static void Show(string title, string message, string environmentDotStack wmb.ShowDialog(); else wmb.Show(); + wmb.Focus(); } public static void Show(string title, string message, WideMessageBoxTheme theme) From 6e43a79e7205374d159f3f5b5f5015718201ec67 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 08:22:55 +0000 Subject: [PATCH 15/27] Bump shogo82148/actions-setup-perl from 1.27.0 to 1.28.0 (#1740) Bumps [shogo82148/actions-setup-perl](https://github.com/shogo82148/actions-setup-perl) from 1.27.0 to 1.28.0. - [Release notes](https://github.com/shogo82148/actions-setup-perl/releases) - [Commits](https://github.com/shogo82148/actions-setup-perl/compare/v1.27.0...v1.28.0) --- updated-dependencies: - dependency-name: shogo82148/actions-setup-perl dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4f170f114b..e17d35fca5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -204,7 +204,7 @@ jobs: - name: Install Perl dependencies - uses: shogo82148/actions-setup-perl@v1.27.0 + uses: shogo82148/actions-setup-perl@v1.28.0 with: install-modules-with: cpanm install-modules: Archive::Zip Archive::Tar From 22f4a0986a001ee26fee845043956f4655c4a527 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 08:38:30 +0000 Subject: [PATCH 16/27] Bump NUnit.Analyzers from 4.0.0 to 4.0.1 (#1741) Bumps [NUnit.Analyzers](https://github.com/nunit/nunit.analyzers) from 4.0.0 to 4.0.1. - [Release notes](https://github.com/nunit/nunit.analyzers/releases) - [Changelog](https://github.com/nunit/nunit.analyzers/blob/master/CHANGES.txt) - [Commits](https://github.com/nunit/nunit.analyzers/compare/4.0.0...4.0.1) --- updated-dependencies: - dependency-name: NUnit.Analyzers dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Rdmp.Core.Tests/Rdmp.Core.Tests.csproj | 2 +- Rdmp.UI.Tests/Rdmp.UI.Tests.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj b/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj index 8c8e33a4fb..457640b807 100644 --- a/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj +++ b/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj @@ -72,7 +72,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Rdmp.UI.Tests/Rdmp.UI.Tests.csproj b/Rdmp.UI.Tests/Rdmp.UI.Tests.csproj index 7c2be20304..45425c90ab 100644 --- a/Rdmp.UI.Tests/Rdmp.UI.Tests.csproj +++ b/Rdmp.UI.Tests/Rdmp.UI.Tests.csproj @@ -19,7 +19,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive From e0edd678612d017a15c8a5cd09c04dbc15c05852 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Feb 2024 00:50:30 +0000 Subject: [PATCH 17/27] Bump YamlDotNet from 15.1.0 to 15.1.1 Bumps [YamlDotNet](https://github.com/aaubry/YamlDotNet) from 15.1.0 to 15.1.1. - [Release notes](https://github.com/aaubry/YamlDotNet/releases) - [Commits](https://github.com/aaubry/YamlDotNet/compare/v15.1.0...v15.1.1) --- updated-dependencies: - dependency-name: YamlDotNet dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Tools/rdmp/rdmp.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/rdmp/rdmp.csproj b/Tools/rdmp/rdmp.csproj index 39e4789a18..a7d23a30e5 100644 --- a/Tools/rdmp/rdmp.csproj +++ b/Tools/rdmp/rdmp.csproj @@ -41,7 +41,7 @@ - + From ef0afe83e100c76e446324ed38847f0ddc3a4671 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 5 Feb 2024 14:54:12 +0000 Subject: [PATCH 18/27] Feature/rdmp 119 data load deltas (#1726) --- CHANGELOG.md | 1 + .../Remote_Attacher_Attacher_Location.PNG | Bin 0 -> 35328 bytes .../Remote_Attacher_External_Database.PNG | Bin 0 -> 38839 bytes .../DataLoadEngine/RemoteAttachers.md | 71 ++ .../Engine/Integration/RemoteAttacherTests.cs | 141 ++++ .../RemoteDatabaseAttacherTests.cs | 160 +++- .../Attachers/RemoteTableAttacherTests.cs | 167 ++++ Rdmp.Core/CommandLine/Runners/DleRunner.cs | 46 +- .../Curation/Data/DataLoad/ILoadMetadata.cs | 8 +- .../Curation/Data/DataLoad/LoadMetadata.cs | 18 +- .../Attachers/AttacherHistoricalDurations.cs | 19 + .../Modules/Attachers/RemoteAttacher.cs | 208 +++++ .../Attachers/RemoteDatabaseAttacher.cs | 29 +- .../Modules/Attachers/RemoteTableAttacher.cs | 17 +- .../CreateCatalogue.sql | 3 +- .../up/078_AddLastLoadTimeToLoadMetadata.sql | 7 + Rdmp.Core/Rdmp.Core.csproj | 764 +++++++++--------- Rdmp.UI/Menus/LoadStageNodeMenu.cs | 3 +- SharedAssemblyInfo.cs | 2 +- Tests.Common/TestDatabases.txt | 2 +- Tools/rdmp/Databases.yaml | 4 +- rdmp-client.xml | 2 +- 22 files changed, 1260 insertions(+), 412 deletions(-) create mode 100644 Documentation/DataLoadEngine/Images/Remote_Attacher_Attacher_Location.PNG create mode 100644 Documentation/DataLoadEngine/Images/Remote_Attacher_External_Database.PNG create mode 100644 Documentation/DataLoadEngine/RemoteAttachers.md create mode 100644 Rdmp.Core.Tests/DataLoad/Engine/Integration/RemoteAttacherTests.cs create mode 100644 Rdmp.Core/DataLoad/Modules/Attachers/AttacherHistoricalDurations.cs create mode 100644 Rdmp.Core/DataLoad/Modules/Attachers/RemoteAttacher.cs create mode 100644 Rdmp.Core/Databases/CatalogueDatabase/up/078_AddLastLoadTimeToLoadMetadata.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index f71f0f9803..37a2e52292 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Changed - Update Excel Attacher to read data from arbitrary start points within sheets +- Add Time based filtering of remote table and database attachers ## [8.1.3] - 2024-01-15 diff --git a/Documentation/DataLoadEngine/Images/Remote_Attacher_Attacher_Location.PNG b/Documentation/DataLoadEngine/Images/Remote_Attacher_Attacher_Location.PNG new file mode 100644 index 0000000000000000000000000000000000000000..1ac3a693936650a8b14fcbcef9c79ba49b9c9095 GIT binary patch literal 35328 zcmcG$byQnl*EL$jonXa^lj2a^+CXp(rMPQxhoUV`(BPB;#VJ~xqQ$jHi)(Rr*L;UQ z&+q-7_q}(FJMMSK{euyboU>)Ewbz_$?tK!bq9l!t@eJd^g9q5MGLo1TrTq{e3G76WrtqoC4nRzhALXt`TX^<--5fj+d@j3 z7ArC44*S`q?+Kdlabk)sk3sYPgUSc=>>Ingov{q^qsjy`FJHb)j+SBlaZXqB=Ney! ze(Cz~j5C%e(!LQ$|-#!@Ev0qHovCAaw6cE|yCiWIL#%!i8f0`Q# z18HbNs;np2t99SLmEwXsdJIRiUHEqVy+jT_m)Xt%1M#$zxn}XT_}@+rZbR9aq48OD zujPhDvZa4A2*V;wWp-Kvd~I6)Jq{H(Gc<7E#sBC?vZuK1|KTELQ?MNLmw%8S0-gP;GP`Wz=`GyJ(t zk$(|e?vUQHpw>byyM!EHCiS#m182&rK8+=@+V5;LEtewbUH&~wxxJ(SCpKP$lyP4M zD^x2l?zuDXw(X|U6THUdW`9Hv?fo5gk&?=#mJw)jdYYE8e|wT6<9Fwz_4#pQ3oAWV zUOX+sR+LG+W>Rq?-6dIC!;sgktST%PkIo`fD1|=g+rL^gKjx)54*G&h=K7%==ZfGm zHwsk&#AKeH_S+m5>-Q-BV(I~SIRpc0+ zW>ru++nAZoyZP#Ouu5)HGBG`MCNZA0I$ja_{np_O{O#wleUEOJ0t z3^};^VgE5AkOlg2KcpWW((E_>#(&+BYx3G$BoT>sLfvkfa;nq)E7s(cPAn(X-FW@%a&~7gi9lq(y1s01G4dL&k8T(wh z%r(g{xc3C^cbWyd!>01Bp8b%C>-Fo#XZuEfEG7!~J)$WWzRTcgfnf%24T94+J6q6l z98UA=4jM&^=YM;p|Xkfu}bCa zT}VD>%B|!P^#f-nXbtynJIjQ!2E6#pNpo;w{Lm{}-2mcjT zm2{F=UW4-MnZRHCvZi^7wA;sdKE4uH1A;gzTi2Nr;X5sPYnVo|s^9i&boG-1>G8Kq zzUli-EHB0Hr#AceOqHRn%eNRejn5WB z4A<>0aWQI#zVA}g@))APOsTJqb|bFZbhb}PIy@?@nuIJZ&DJojPk6?H#@Dj8hTa1X zvAw1-Zni#pddd+}sXT8B*-NEo&6GhLPK}9k=w~o=Tj*a8pxp4~b)VCWpv%MkeQSSm zuWxU^*9Cbr+Yj!}B<*&Sw~BZ~=QBa4on3u0dKA!aJ#w2!#>GkVE^a*i^mq45%C+sS zeY;oOPWGd=(SDzxtHIG|7V&<)4jv%;cM62kHdDvbLe}u8PE7u$#D(Zdf0!hg*7WWF z$au66bE5zME(*kh$oNCRw7@GGz6vl*;GZYX&!KUomi-XrUAvRfU1%BI9Ir6vsJihu z!~JDpIF;D`C{Zlf_JPEG_-3V5KgXq28ru;RXwQPuM#s4sF44RTb7ay536G_BGR+3W1E#P!^U zB~b9Li!aM>PbCe%Z6w$=+y7FwS+~_4-DqNGx)&;cKTz6~M3C$u872AgeQP}mx4knX zK1NK8D1*iYc0?#e3tfi0sI)Vj<7)ngh_?~AyrH8e-rsOc0(h4!s`2vKPsVp%AE)F? z@|&24pnR3ZjFDuyw}VVzGC5d?J*NPMp~8+*s1Nlsjdt=gee3nt2E2no!uU$%t>=jmy>7PrgHu|;S1g} zXf-?~zYDTt})U=uZIC7RPC=;^&p{k5Wn7*tg?;k4Ymu^la)JMEdECS!5e;+ z^YU8I;Xf_ClU1Gc%eS;3sK?|lvJRB`v_MpG{4CfiDdo4Bd%Xr0`&6WtIuyQPv@c!B!8!lAXz1Q#vuK!5@KxVrC;Pq%OGl9?O=;)uX`ROombQ)7W zQgR#6S5(ICpS*!FmtBo9Ss~QEq?qjWq|qWr1>b{#c$e?rY#5IBeat8&H1N+w;q3Kj z8psu%i#`o@t_fg#E2PwR?U$5Dvei_v5X7Ljf>Tv!%zN2c`H`fS8#7kTZx5oy&$M)F z4)_mA{0>RX0Ru?}mpZekmS5JYp4~mFS``Zn8?mbd%$5;q4X$@sd?a?9GV}*|of>}F z?RO-2Q!9SUDt5c6TyzRd44sjwTP5zw&%tOyL#1iE%)xivuietGT2uVp`18O4k~Lj; zTfN^WRPnpRXvfW3UviEXZ}euQGN!|yANwY){^1U*2^ybx^Js}=G0nG|-)S!#?uv9ykEQi8=K2VLKsPzMJ^04YHz6O`xn|eG+VR+ z2GGCCE@>vwN2vr3P&u`e(v*d(%UU zvC6lMcN@`2G-{s{gcc8}Gd&c)d!&{iQihV%TnnB@KRv?~HrQdw(RF3bHJSMF?&{4b z<&`u8gidswQ|#AQsR%0=l1g-aq1~|GG2%?!7*b5a7KtMhh~U2^MqbFv5p)BoCPi=Q zE&OeyK5>REqGYy4G^a;zvpLv*tND-8Bd?x%ejGP9a;g}wZKgbStf0WC*|HM;GZhkL z(z5!k!d{%h>~(s`zJXy4cco8ZGxgr}I&~k84c&F2jK1ag@9W|7q3=FgYX*GjajN2A z*|TaqOsLO{bIg#BQN>1B%t6X$(%065qRhxYYRq0p+LvZ`6luA=8zu7*AG>ZzsclZ? z{2aaOC@KMzZoby-`iomWe@TC9MWAgBRqN<3js3^A^hmpJw zJ*O4AXWJ2jlDmR~ zhN(8p+b1O^pFjIB88AH3KP^5z;|s>yFgx=U3oHK_hq^7%z0*!|`wG8~?R?z`Elkk^ zWWJDS`D7=fMm%f4a(`TEd!`8Y`+Ti4ag=Sc&&<`YX_?p98~!nzF~7~z7|OmFFI%Wq z9}jU(ev0Zne7y+C&U({Q;0Y^9rJ(MTaYx_6@g*lp^(8FH-|w_ENMK%nIlc2d`e{5Z zcY~xNypfWaTQQ6{U>x(dR&d$5b z^jfkptr$KGAHi?p4aq^jk{C597-;b&^+o2qjObp}-buglImzF!zmOCdPS&!rs5{28 zIyWfHn-K~(sEA}lOua(x5poMlLg8CmTOvFOpxajJn~$IPu1^?97I`wXKFEtTJTSPz zO$sGEu{fN`Xn8_Mmm%Riqv6mk3W2lhe-OWE7k{Z!zbLZ!DUg-WsR}CGDlHEm_0{~f zw!x9Jp2^dBr&t>aF>p)lwTpaL^Wj#bZX$`+?3SKBRv`-4=*wnIY}WA%@=3cM^kD^$ z*l5JGk3svTtzSfXpR1LeZ7k$ve~-z>nZuBn9()a0d{R=#!tET#pMJkfvSQCu=gEeB zadh;nm%a|2`-`XxNKx;)A0-+m;MhF3>6*Fk_qLrJHFQZp%Yu7{>H0e#}oW-)O)v6=^3_=^x*j2FEQC%>3Buq+fiN~_z7blZwFnKyrAbdgT+!x_j zX?cd|esrHpfQ)K>gcR{KSI5nYat#zSh~<1fP0GqSC92U$8em?ixauy^6PVFPRD;ykw|UM!i(Jkb7Qydey7ZATasfcsy?`uW@*K zBEw{=5|?t_7j<*!_`S@|ZQx6o$+og2@29LtTP-h})Zi29ofXSO_XU)~sG&2oFp)8@ zJ0`0$pSkmiq7ZFm<1d?>j9nq4PA1bWi*3?=Q6uwm)e7e1?^(Zq5L0G@xy-sX_6t!Q zcXxi0*8R{64&)~P!F5Z^1LU$F>yJ(nBub6%4YD11W6%;5?RLkdEsVrV<-g_kK29YG zcO_XYyW9GGfs0(keREyc&Q~Gf;{E$W;J_o3$EO9eM^9g7)DzN)5?TKIu8cOEv2;@- zuG*5$kw3aZ?Hu0{EI#Zaspp}H7Us7Efv;rz_B$OjwVCz^^YBL0(>3$94gWqC{5*Jt zwO4=a9;0)hb5XL;^aRM$zJvxBP4X%#&$pDlF;e+`iOWi5x;iAie_j@i;OWaygnklR z-iDY*H`hn2G#@$_h#yJlPTulwMf3CJ-GUUE{DS*R`~oo>nj?aK?HSFh#z^(SYE4=NHAqk#V&R!km;F{K9_hb0UN)y<@ z;^>^oOA|PNO-M>j%>8olPMUmI!O*x)4{fBN{tSUfurPoL#&8#(_Ucxh;DQ$>;=keR z$I5PeM`{)+Auz!i3PUW_KLm~8{IO9K$0)jXDAmWwCcLk~ni0b=mnB{CzHQGDhsb&%?^f))9%nbz! zz?2aBD$p*+22>9k6qA|%Veb5a*B*-7+(T~{mT$KtKN2$@Qs0mt3`TQao(-wK7eck) zB_V+RElt-b(b=HaN1iGgT49GiLnbcEddaiiBvNX2-5!0$5lu;fVW<2iD`}~mt3-zCTTkWgOE7L+SyJ#P(%tq+!%DUW&?)R8#F+(OVBnM zRrg)?QHzDh;NQ5RA_cy7!oI}Swi1^6lu#=>CQp2ooKh7Xu*0hZh7?9j-x_Q|A~h-mKZgSK&k?voTy z^i$0ZE2=ZE6f$>Lekg-_6eroXt?gJ(T>qQTHNF7RViZ<797)Re_`H*Gc07%GU2`8A zm;FF7(9gkt#-LeK>WdptB1pO`@Zn=&(ul$LN94F^otJsQfazZ67G#>BH7N7H0Be^w zjjlqhTW&~~=(xt8_Qiq-R41P4!6xXmjA=`CqFWg4ZJRGZ{=z3aC#Zo@bx!q@B;v&ep4o zT6BGi3*>u85>JTLExgP{-ONXgF5T8;5W#|NS%DByw}uVvB*L&78*&D2I^no&TQwJ@ z)0cHj+iilm9C^x9OumL}lHE z)}@ZiT_Z{gQB_7xb?RuUMR;GhBdToM9ojLl#@O5c1W9wMT*_m`FZh?2KDO-j#h4QI z3u5aYTu!j3SSwmBzVw9!;WySys7#(l6%(!P?~&iF6p)dpdSa+q!yVZELI__2a~*x-opgy{A` zHs#Dn>>>SGNek+bEqllGPZa4{n-BnOn$i zb0@OOF62&#d@-9Y4ccXNFtn9KKCYQ4n;0959PW9o31S1eI&gr4KpO5eVf2Ol@$xMo zpH$$qM;39uIj0CNq;54_O#{RL5X6P|3f>$t?{A;QbZ$QqcT7a1yglr+1SRAp!@jfR$kQfhTmt<52QLskSe`va+1aD0k7r61F6dAM`juBwJ5edDkm>DrR za;AY!WYr#Y1L^>k=TS3TjwRivXp!Kx0A+?y&?U0*1AY_%(3&h3ffT$R(>*KS64g_V zFMD}Zext_|g)&7si*6a;B}e?jS)6;e=&n)~bqls_H*wf(U`_T!wy-!^WW)aJ?L*T= z$z^kUAjHv9?CB#w*yF2Nmp`~iZR${ndXuueJ10~R3lqwwV0LDm8qhiWFN~{OUm^Mx2eDeQ99p@UtjaPCh1?hT1J;1mfLA<-f6OrKje$XQ?Yt+ZtE=nA z_ICTmM!~W@-n}X4Rb?!}FIZS4-o7P0zQEBj`Z}P1&4%;DfbcF z`?W>Z5*!P0Ir#H>Wo<1hJNxn2*cdZA`&%cMy;;GknwqT2O0`JZUMy_v6t=%A$AXL| z7~tLzIXQUjb3q?QX*keT!t>pQ7V4>~DN8%Mx2Gs5D8s|UU$&YK+At}2|1NbJphQH) zp=#?(U=gM3Pg95hvgx;}q^m7j8iAoAd>X-Lrja;lLarG1B{IQtLD=ZrTuC2?Zrz8r zHi*dith1FT8(ufHXvCz!N4NR>dpprozSfmPfHDpb9fx|HnC7Y*R5E<{pDBqyKYCtM z;zz1e>l=D>H*PwW$fgs6>_z=Z2VGOtwLSMI z+Kvsl18=C9h5XID15b}3@aCi4Yny(%|B=!)>s`=aym&Ec|3&r^LHGLU)29O77o2kP z@*|_8IZ~lHpx$>czuO(J{j@A@dhYoNo$vD>0&cL0)}2y6oC@# ze;7(XWDJqxnofuF&wEHHCaJ5dgEtZWE^7m1Ik~yu*A;n#D^JHO(m{E2(otVpT84H; z=mQ&iic$-;e_QpX=WGZ}d>>P@-i4V)$!VQ_&Q^On)wW_ZF!ALorPSij;amk`3 zXyS?A8cW>=tREmr2nkL5An1U(&d8`rZ+f2a=b98;KJb)JTHYARmJ3{|mKZ;k+}zxh zi^yT_1@j8Z(fhF|Dy8*S{LWWK)i*8aFog^E%C~Y#>o!86oa&-Uw!I?^tLzV*h$X!r zT~*b*lX9~x)$qSxAoLF$S)fO??ZqtG!`2sw<7CppA02-LBXB{F3I8U09_qQ^WFc419TSizO6Ba)Y)J>qx3YV;OvXQ0`*W8GO6hr zqdKFffHrA#&_+jEaKnJoXWyzh@vzENJSc8|;lowE_IZn{Y1F}F$LpTpt6d%~^IkXs z9PkV;(-I})yPwUq_mp$2^%4a3H z&Yr;Kj{TdBNS0vm&w&-xX_wnv!!bc&t;P{Gei4Xq5RHVrQQaL8)?1F&KbKxyD!x2d z@?}ti!-Nf%8)DDnmS98;sOq!FBL?BU)|HOBDIE$!a_D`U=y?NR>1t{FD?OC1Yh|Mx zGs<9c4WoiRK`eD~e_$p-cyr8w8&)SoKDbR!7bKPU7P6SuSs>}-nX8k_mSQ(A1Bp_a zI>oF&eQuHeVq*vaSbXKf$JI=n<7po=uc}E4^&(rhk~lT1pFAeQ&1%_kJ4w3Hl^Lnt zV7uW0h8B|EXpae=4qc;*&b<7i#>EbuJ|WLp_u)9|JwD$~CF#J2kOt+)E~$PH8~+pV3!8QuF_!#svD~OK{ys$>Q_3M^ z7c@E8=T6i`K3&U;c(1YFIP?=STlRJ5>_%Pug{U$W_{$uSD8ikCs=tW@d9!#9Y{LUo z;}KA5ocv}(`fs}-SxRS2Oc%Ra+Ls}4Dp6;C-?~-m!*=Y2_Azq52w``Qt|w*xBKo`l z(bvfFmX{r_XbUN)M?MrmxKmRNHQ`3D&IX`X?)~sdsa;IkLDzSGllz7M3BOZoe7 zJ{L>Kl~lA&|MvZ)RIL|aJ{hXYo8j2 zjGhYEV_46sj3K27DbMM`X}r(1o#{_fn}qh-|0W?iAWm~GBBCPSOC<_#md*uT?KF%y zTs=gfjt$utUy5HGN-wmx>|Wvfrj(;;P+zc@-mEy?7%qta@$zZ``|3yK5Q?|gF0_D< z-_Eu#x*GgK$oX%2xDw1sp1QhUCATz5)j3Jr&z{>4C{(tJCqJauD|1d8M2|E-i#U60 za-3qYTpc&`XL?ushu&uRLAG*p*QoKb$CR#tSshY14&A~ zSCik?i(av``qyA)-NOf z97Q+7x1(>!o16AvH|N4gmOGb2zb^G6q+kJPDt`#e@xY+cg!Q^fqAzM}954LGw%5)r zW&qb#Xol}cE=Qepc(yY2wC6-~v6G2`PbZCi(ml560oc~K#P@$wP_XNqyUse@Sk&!y zU|*cmA^lUMk>ax+C(m^KE*2c-T8j>{0W)1mO#GRNzEd7G<7mCOV6I1L7@%Uq8Y!2O z-Eq%%?+J;0em%Fn${cxwp2QtwVk!&e;Dd}o2v`VAipcG9xXlFAE=*?7!rx4mO|ZMJO?^oW|HGQ(ld>4)LofmJp@V|@CFheKX6L>*zouuo z<%;i9j8|(0iF{hB!gELf0rMHyX}3Q&4OAFYd%t_!alh-J1euB+{6DWs^#2mQn9El{ z?XV|=HEL$qhy+P#j~mrF5(RIix2v4ekrp?qp^ZdBLQAC#M$U(I4itf%Nj6WXX4;i^ zl-g0EQETmgH``3b;{Lxt&6f$@`<#oGf(z6Ot|WVs$|E=@@itQ3Uz3Irr&vPV8Isan zbQ>~$4|bXMrM#xoEX94vr=!>izN6bpmr1+(azArl2pC`PZ42{I(*!cVX)_>1ywpqz zUwhoIQn~o-4ZQ*Uh{!;?v*>NJx%~G8A_F4VmXDv}Qv+3Y7iIopNi>kX+E#Kd>>-h7 z3Q^IDEONojlE9#P8W|mPEHT=4O}vc>BK6uY3LKPQ{puG&_?DtF$u&4O)?s$Tg%FHR zg^e>df45gPKf+82?5Jb|^b61}v0GjTY1ckS37Z2hgo zl*cXJj!3-L@b}cYV?7O~+3Br^gI?Iwr+d^Zf0Wx}$_P7Y4U?K}I>r%sf1%+)cdjl{ zb#8aFljaAt>Xa5xsYcECybv_EI-@%5jnc{MQ)i)w_}{nF&x+Nfc9k!BJhoeKVxgMz zqt+jK|Mc9)8`aKx+oHg3SHgLsjzjb@p@7a@fUuvQVGZLwWEQ^ybED@*hmD9uVDnvb zbInGa4Vo7VdQ;W9bS@m57uOVt0(Uss`+!Znh#{q2;sr&JvcS@-$g5V)A0OK{`o@czO^M3sv1o?%w&a!B{I-YXE&0WiH4=`jT{Cbz&Lcnk zJtp23o#>6Z-&9L_%A^m~7=PhnQ6^=*X|ywB+r`UanykZ%FwW8we}iZo1`*&QWEHtf zi7;ZGf8R+WYZ>;CMM$?S<N+DWK6%y4F@K}1joJ zYW~1{2AlJ?CZcCKqDb$Bm?z9o#>|ru)jFI2#rz<`F_T7q%w4|D-)-&mV7XO$3lC%=cmG2W!81G+)aA{_QtlbWM@tjjuJ=}uCb0A;zX(%98{WyGmv3bG# znYGDDtoQpB4b3E;T-5WR=!XQch%NmiG5HL8X^s<_7De7nK zpOM@1#naCk4h!Axe~?X%*h=XvQl9S_q7u~&rR~_M25mA)x$Y;6#h979@uO8j^Se{7 zh3WUo$VN+T48$`AA+3*bo0|fzTOe+w{N?d;-;V~l^N~dP`akq$>^qh9o1US8X5X4fxw?dP%VoFnz zO_?e;w8#g@sa|p;oG-NJBs)NPS_vSN>-gvQ4Xe^+f>JK~YWFMILTI6zP%_P{L&A%x zPu33oUE~Lot*GTYD+II0hr1W8{R^E$%1FYy+sIB2q%O`lCb(W!W<)taM%4;|mrbXB zR`IjxmFhmLJZ5ev;3C-_`bJP>ROBUXu8aW=BeVz#+%fHZ!}&J9IXt7$m#9?ln2;BC za?;D>aNZq$*hJDCU@vl`DB|ovr!OZSwa&xiSQ3-vheHeP?LD&Ki1pqsiYbsU;N~e8p(AWz2e)ojID0fTQVbD;mgcJnJ&M0PVEL4K_+yIJTTy0YK>-Y< zwSvpFxOIcUcik)^*_MEW^<^3UFazTXQMMZ2FxH)#PFN69wfn~HKGOYQVoHmHwN-69l`y8 zO|x@0nkiU4p5tQrx)Nhjy&ztwj{UGt)y{d$0yJTCW+WS}F*_{@kiOtBn|KgI*sLQ6 zb5GPZ6d&kj%?B8%@#=R4pu4|c-=0ipI<%WhMC=XmNQck11YorfL<9aUl)_77<)W5M z0f0i`%Hzz&c6flXoE18=*MenGkIb=75oapD=np@TTXg9+wL0iiHzqgWa#qJsY8X#_ zp=Qez?{P{qVSUZ))LHmUI!>yo5}gNg6}V0Xn&_ek19dKVGaAAnJAYrMMWYsi`SB zIhnJ?5s;&>0L=O%6VOIlP&| ziPjNvzSaI+pB>XE1;OgsIlp#N2<5@+++_TG8~L+m|BaIqca7~#@Y0g;$;pYKxA*K# z7d~VSn0?$PV5vc1KSoY|CVjo%JcDBTP#5ytB96?CnrP)+f< zEmCwh?$Z}MQlU1Gm)Vj_fqP>i{hkai{oj7R9ZGRH>z!gG7-AV6Sgpw$^nZ$}I}t)~ znWkNC{Itey4xW`o3s8J?fZ>=5`Di-*o|p*v^hukyksbGQT3VW=wY9XfGp5zEXR2>? zLi`Ic1(H)kA_@zi#L|M<*3e;*q=Hf`1FBReFTkC?+}Ll;t*LcijE<)9Mw*f*lq-KH zGd4>fcHf>EmsFa`=h&RVHZ#yOYh`!+c>Q8VVB)El(%a87zmg)m!}kqV{BW+03AoO> zp#tt4Jkp;jZc)ANO3PCE#RgzGpdc10pj03r0d&g(A5CMrr_B8P8n>fn1xh_C-(xb{ zFvkScwZcx0RJeDqv*uBd8l=DPDX{Ff*&rIqX?|f`ZzY{9*Xsf=ua!K9XpzN&oT&S1(_B(lh zvyTdaNln6#K^!~SL`t;D&+Gcfx$2B~w>9y_m9$tQD2VRA#>wonL88e2^5q{>Pa&l= zp`e=^@6j>-FR&gJ)v>25Xk>lG1s&ziW|N=NA|q+Hc`6;WWmHQo$rU1CaUCB;Wap1~ zNx#Zyf5Yn~DhET4o_x)?7)Adr)Qf@^>GlZw?V1i#meL=!h?hNt4T6uBgtfeCob9^H zm6s&Ht zmiu(^FUJeO-l*#YnA1SjOKh2~EIYn#J)3043_hELTa6I{lqWnn)}7$w*kMZ-anNIifXBud$DUOIp#!m|m%C3wZj(M}^6< z2-igXQNI0WCTpLmNv3nY%$*{gg=t@}y`t%IC)O zF?XZtY5i~OrVPn0b@7`+{njnT>xD}jbd$&KK0p0;4NM73@p^$BNLgKg!6zBmoe;8W zH0pHela%2#FOwdV8}u6TF!UpM&xyd2Q`uc77^DBXx~H~~f&tl-Xp zXOx+l8^4!goEOswHoydH1B`CUA2OTj_9&VNWC(TUp%TyuoCpLp(M+QN9F>4rpo|8l zW;q^DmOn&lUJ5*m^EWsG0Y;}QCwbP3$SQoXv%S?8HO}WfJRE9Kp}g2?((Mr469@56 zix;q9QE6*hE_lZ^E zq>L4r|4u@AGa>O5DT+&n*R=caMurNx6?H@sG@Y<~>nHJl8?M9kx14U4%6q1A&v^b0 z04Oc{UH(#%f7awc4(?^qRV#O@%71p!wn;Y%u9*5PmwiuP{<&767FXU$N=iya+jCF> zEN3bwr^*U@qwD{uyxGSRjdJOW5VrXvVu%AF?0`h9bITS8Hh&C||622E}(6RrUEYP=4F@6?n`>AVZ?F*g9>9+M@{>LMbO^FuE`pnFS zOuv6SZjFM0bI94H%?}eG!SF)%=h8hY=1_*HG?w=99UB{vRz^w?5M3*55dSblBkret zUyJmPE4?G@12``vM=FAXjz!L}_+1(3I!s(Mg~P7mdF^{L;(flvF@QX*=1RqwlCKNZ zdU|GXx90SCG+x|6FJc)?r|f5`eNGhn(2zCW_uP@sw+0jp(FI4OeGC3w6Y#%%_%6X z#)e`?3aIM^EGHxg=)R{4v9@xo`-(0-J>#e;GvY}DMGXQ`;@bjG63r7C%(E?AZ7SNx zBb@sb(q`FczM;oOxK-c+G+@1Yp2h$M85nt{Tz*iT-yMVFN9psjYa|2MnX-)3^Q{`8 zOw~;BQ8=C6xTaYqf9O23-x0+Q{Y^yo@5N#&1bYc~dMz{ZF^eh^=1^c%}6|zG+)o z0burhtlQ^CGgg{pHt???Nk5x72I>Do`SA?=7SN^H2z#V9b!&Qy{K7-mM|}kRix`N1 z)``>Qs+4%EQ|ERyDWNMWR4xdV_O(<>%bkB-+zo46@-4MMDA6_UABGe<>BF1yWe>jH zCvKray}9e()MZ9+M#k0QHur=N5{1R~!SySGLRK5E2)5GjR?47E*|s;%;~SW^j9XL* z|66V4Za^%0lV8bygZB87o`s&@t5W;z^Ygpa=${##_a4*zCq`zT&`86tI);=GA~euV z2>hs_K4xS(SOPn867VG(u-LweM8|*9&UwjeWa1?Z0}$bP&V~;SlpLzV4|Y;Y0UsXf z#F(rwyX6n9^n}iwvTZ%FSIm5;_VytOAu7vpC*G4H{VxJ!1bKZFe#AfS@tb7d^X^F; zP7*f*#5_a)I#*8zG%WpXz3iN=eUoSCKPgfHZ1jaMAKVlJJpCy(oU*f1fHubx*Z{x{ z0@(sU8Woe}4%Cl;H{Z&PY)3|VJZD0+4g@~1eGD8%La(@U-BynMWEai^Hcs}y{deLb zCh)XcPr3i2VoSTO#}?rS zpJKB3C1)OhjPsmhC~2-KWU3FA=Get72$tx5LaI{U?k7D%pVA zu?PVL0*eQ5y731~SyeqfQJC6!W?ml3`cQ`SyLaHDV@yPXD@J#8#*e#O)WgH>hQqdS z4wLo4Xme-;a3^63fY|11stXzO`dBzP#x5=(LPEmi&!1loX_n&xT%+D6K0fhJZj!-U z+WWJKG!jpH2G9D3OZB`C7DHyfy<2{plVj6`$vNjMC189s>f0olRA~}%E)urN~3(Uqd}!gs22D| z` zhQrj#wI2sRV)4lctS#@d@Rxj5xy2Ck|0~1&@va+%udhQ@y_zi82-4k+$ z32%2B*vscQdt9+cNq5f{JT@OMDAx)}yVqC67_WSxT!HS6K? z|1F+ORaJVCHJ=cTWDF!F8Kvr^B@fB6-c?*u4Cs~@qLc)I>z3}Nl}z<+PZ3wzt7l&3 zp4T8{#6kwI^*A$EES;3oi%8hZZb^B#*c37>Hfwg)o7c9+eP4`MuF7@W&HJeLVNSk9 z*0*oNFA3W!gDXukSe;B#MHe{>9V$xxO_K9{fO+ z>EQX6bL*WQlROyD@3V69%iFN^|JB}m$HUpR?ZPP%ks=X7M7wpP1|hlx!{{-hMN2Sx z2{QVSghVGg(V};vM-R~n(aY$aF+>|>4Eq{M?&p4<_x--#``df}x%1z|HM7>W%6XjY zJkR4;$OYio&fPGvVwS570gcC3Q$ztcq)@HASdBWN8PRi0?BwyZ=eMnc^8)HEwi>L* ztNc2d{vU2LmD*Z{>jQAsD1Mm%Cfo74qIPDP=Wf@f%vH~5qwKNa)MeL8L|6pg!OpEs{$t*Caxpjeh?(EL63gC!!T zC@+z2p_DGcm>0zo7ZxJu$69lfkqAAU5kHLh$>vdha%=(fRTB?SrM<3K3Z z&q>8}=WX^7x3(k+&61ZqFSWm`pHc%%Y==u-YY#$iNm~#zh79!+cLo#I>JBd z{;479#HLw=4b1!K0Y}}NHunx-Gn(FYbxX$=d@^ z6H$39R=aZRq_P6iVlp~ZI*87S)!cOsbM5`92|AM4a5}ZlzC^I{1=?H00)>4JwzDg< zzf<(u?M-i6rD}zZF!n(-*X&P8ilURfHir6TR$A4=dxU38qsV^7>MxXeQPxS4>qc5% zD{e0+@t57~3*@Qushl?8&mc<_bA>9GzLP*?k|+fbob8>>1i6r}qq6QMK^7nZu1wh{ z`EBE5are}SG&7mgZC23KU2g6I{l-xi17`J}u(x|=>pO!su~k8;etMm?x4WPV2|Zz5 zo@4e&pY~vf+M;7eDo4+{)rgo$myGrKg3v;hv$r^1p_OaM3L868sCfBJfU0y_k$|YS zW_(_U8QnSjz91ktkD1U*3UtRl!Suo6dGF$(f(xV|FaP-C!Ql_-(3wU;I%vaNo(ywj z1&wyqHe3MC!Xwnx)6=^4aO%2hz|%O#qi!DgU+<$QXYh61i6;0qHR`RY%^LfR(b{vE zsnpo}O)1L}n6&4h!`Gr+s&Hepw>=^PBvc_XUUH9XyOOtJ*7{pFYymq{A)#%g6|m6p z!#+H6>z>X_PF+^_WqHF;)pJr3{#1;Ad4`pYuP}0k2)-ENzpVq>cJrKVXie(nTEU1N zU8_>Ieh($Ph&N$J2gZ9GzQd5d2vzPqeZw-hz4(CyiJfqUH@E3g>3Ktr2I>nApUm#E zl;Oj{Bz~_+Xq6SDgkOkyrCB-$Xil$m-C`LVcCT)$G?27MVY7yHk@o9y*?ZkKhq5Ww z;WL_AhEBSh9_!o3rhqAxc!(<(KvkFDgwb6bZw7Fa_fc88c^}t9j#Z+?kExYY2Hq7F zPG=Zqu17se`JRZUm>uj8$@G&TM|G1xwDDdT;vnm7zR0+rpFyU|S`S~EDm)CR0$}A@ z@;5A~?1*uctX+%4E+eD0vo@uFER!GTT^tcz`y$qmV>tg6QO4sg`Op|^8V6}-KyA?; zAb<7Y0Vb`un536e_q|p`gttOpYcN&u@z9;odSl;e3Z?Y3bN200Mlgw=*c&V}OG0xS zQJlZ$S?@8@^<&PHI7P9aGu|#gwQkO5f*;P0C+mlz3LDNpoOYw4bw8GeXB|gJlV2yk z5m#{LGXRMo)1?A{?&SMACuJ4LL->&6R}%-8eEJKETqkq;YI9W_#qcKD(Xr!bHn;8 zeaR6bXUSF$_C6@od;6^$+0w(mc#3GvdIssSPr3L?EI%aa1d~!yygW9!6Ot$tT2`iz zpiuM@&GBsZd;;Kgd1e8=6&uKHo!45w{3;lWNnw?wn}@*)my2Q*Qx(;`bnXzFRHo$H zo`a(qvn^=AaMTs<2RG+yl?Lv!Q~5rjhmHyPm8Cj~)qyIe^k~0QvXsm^2fSzkoEjKT zk97jKBCPeUfEC#{D*QIQqEtxJ@tiM@$6J69cJReo6sU7*sR-v9eT6|J*tSdQAS7w^ zwC2XO%9<_;eUlM9R^e4TNFl4rPeygi1PCUZOCzvX*lQH*t3fw(g4YiRlLC{+jan9&-H>y(xC8BfrZyrGR9J8Sz?srwRnSwx zZMaBova&vdfsRA(C_vNYoEhNoPJ`YP0Zvm4kima~F;30DWxT9%$fp(^j7in2bxxO; zS9>N;A<=E#(z7mTEpOMdK|LQ$jaH0FwQjs{=TQ3v3akGplCsL_Bg&NazYInsZl%MO zaBtQ56Ftfx4ErMzezQ|mBDmZ)cL+S(7Et{`cgQz)>9I%nNXXP<<8*TcklVif%;&>{ zXY^vBbi|0np|T6S)$l7oiQ7*#Cn{)f*$XqLYsBO}BGB~3 zty3B{(Uih#{Jiexv?|G8PTlR}npNmy=xfl@$@Xe_m|%k<${;4S!|$2j(*u-o=oA@! zPCFMNopIM@Imh!5%0ASqP+|6IKi6+*N!*a zzUe;nZeBaD7c$&lqoRA32a;0?+S?A9JdfBkc+WG*W5UU#Lu7_@lgjxf!+5QEe+UTY z_}6Vl9w!j>Rc^#?)C{<#&z%Sp+oU^Z&N`ddzk@E6cosSyj=UeN2g;(frkZ|N@cwM- zx|{#(&HGftd9=PO8%MWHW1|2YvVnR?PHS#Zjwg-LG0N=?P{Q?1${`_@bgi0_MKA+0 z<=7$iqiQk;s;o{)xX+=PF?utgvD2fNIDLDW&w8hzvLcVpJdsepuHcl(nEJR5nL{e` zNPk&GO$+HodD_vnq8cy}V~y?ZmOV=V0cnsUGogO_)oI7{5gq4DtRS*CI1cqBKySFs zNTgru`6oao=SI#-r~B2!v70Fyqd3UqiD0g(jdWL^*^|>PbT)Q{IO=1fb2bGV6DT}Q za-^PW3g{`zudsCv%=Ol8*IV3C@^hHmF(DMhA+hz-MoaRC$j$26=yy;#Zc$8|rdEK# zXElBnXmL6lHEm{ zQFQ;FZ1K~eQXiN%@Yzd#s#n~k~*14XG zA=E%NBH5iWB(gv}S(nfd?K5j_muChD2`1vB7QCm4B^_@(tZ`*)RNF7;W{cOXk{{DU zWog>nko!+gYrBNjE_0&#y56EATvUcJ=@Vl~$-|X4Uczhl^6<7TwB;4GZb)P1K$HI( z?^OJ|t)#TO=+WT%qsl(@#Gs8_iaw3Q>mprr(rs1leVQKeZf|6H^uB)WOJm zy$eo8Z*ge!+Hd95)oEwZwcB9ZFCOZ#=+TGAO+z0<#kRLCEo+5lO^8O@AU{k~m2#%r zh(=4>ertx}jlVH(qHnbEf5*dYc|KGJR6pOj2pdUjSiqBRDC#c7;y{d$b+lcLIZ3+%q;X^ff!V=?#dByTmdU@r=U?6tR2Pg;&_Z?bO5N*(6gj>c ztX~(;U8`4BX|(55EpCvdVu_Xl3hLxwafkAiiI{)LuAB_* zBdmW-Uy>8+cGr|Et7n;$DB{MITx zg8~)}L7wHKyIuX^;5Lv{;V&s3zOn%{sF;Dm1N3|;mk`HZwveb zU=;uLqLLVGR zb6)KbW{-9R=gQ#gvDnkwL6VWFGynFa`hDJU-{MA7!)=FUPnqKOAM%v|8FAA->nCAB?*=Aig>2q__*iyMK4EtMAH#wy(*7x1~6>V}r(aQ?nyJ`dq5ML$sRuD`R97`%@ZV zTJ;3nSjoW>q~Z4NqVpcxVmv*z>`pG9?yco*Ei-sAZ%cwt)=`FSh5^UkF2^)soBn#B z?UB@tlYK3t?8REnvW%1^lU0&&B1^O2kn{W#V_L z+NN6#B~Qf)NqC?e%>=^%Vyq>AZrTKgL9@OQ;-fbAtsHWlqkcu9sJIaMt>-`ZL1E2^Lc={RCApve0n}B|HiVVw$ilT8*=O8T>Z+qS=r}*I{OG)FN z!A%&|FitcG0 z@bJ{W-c(YVQ|prpXh$$eP8MGbtg!j!r_GZ!Kpg0|L+!b@-POKdo7v7ojalXQ4N7N; z3#T6u1gzd!mdG-6k6f z?3qsp(5|n(1FFJT>=fFb?3{nFAQsq7<2)~U1$0D3FFfM6EzYn#oQE&wEs_dwXg(@6 z(n;EfkrPHSoMs0qGf*?~v3D}Pk`Gfd1H^JLc2wW(bt!(da*GHy5>?aPi}r|`1y0XNk&F6li^EzX9>1_}PKR`Q zQ^P-dRuA#sUSC+#Skj}LB`u50*RWu7EOhP+;;|WBj%c`rX~{eFV~rePo*3b7DssKf zj7e|zbathLv%13x470MF`ex&XDcyMD#t_ZH_nNI< z11Tg0OWBiv%uoMG)4e>tt(n%qb<>!U>n@$e*U=xBf^KLAL;~86*NrbS$0n>uTTNX} zuSE!4y}F%64Jw&s`0KKIhur7IuV|eC>-VE~M@KAd7?{B&$?rNKcaUK=z)*B)q zpt+fTBjQ^@>|G0u3lI7$ZhX7JasO$}bwEj+}1ty z>aCwQyJHQv8T;Xtigix@54RU4Lc9`7>sX!$%3rO}XB{qAl&wOZKSm2!4Xr$pT>>$S zRJp!et7cFPC@FCS%3b9G7`VZLB>i$~CPAdx@Ead^X<(97)iJBQs!f{Ktj0JKw4NYR zXU|)u<`Dd~LXwu%^VBca5oO|jJzJ5nRYirUkJ=ENwM^BBO(_6qsTqo}a! zUc_6;J~0f;j36PKX!-NcY8CQ#{Gm4th;H4fT*0C}h}9)o6>BeCvOYpw>QzGWyRm`Q zfBfbK+`@AMsEjAu?hnTW^i^=(wRO{N|FI+Wp0@S*BF$a4n)5hj$;XC$Hvth#8ZJsz zqeXqMV1En6ut)N3oXW{LR-XCw%EbV?aJ{!eHO2PfE!+_ZFbbZJNhBa+`6U)Rn((?0 z2@>%f_|)E*3B&URi0I*Ej-3L>e68D`0)tC~>#6g*8FRgM(&}4*00A2cl4BWjY?G3|Om9V1uRddx>AZS8|_MJE#v6%G>{u!DNN&`bfju#x>@< zVqQD|D_z!WwS0x6{x;&>w3OY+$o>B4x6!1kGl_)tYxH25Ij3_msS%DWbh;7~g@vGS z2c`^&O3wFsvW|!hICRK3X?>AMFdWJhjlR8PoyG%rsv)XtVi{n%8`la{#ntyE6PMFN zCwxkHp3xvNd&?#==qwi=9O4@NZFsE=uK{U1bU`wVFrho@Lwnoyf;Das`3x@ObDxsc zBTW@G4v0hSFc&19Rx_yel7-tyoqJkk-QDH{F? zy~pt6g^bh-9XZu{zCrNf(Y0IO_f7Tnp#9_~NSDavpZ)KWU-?tlFdHBUT{YRovF+Vh*~$dJOk4ngvEd-3;o9S;rB`g=D@M*?0=b zt@;_NGs)evHe#F?Q_~GCAT98GyU#u_FE)ni3R$HV*iPBs98ytK`B>4j(i_eV{~nA! zq+Imm{kftZYIcgTIV81iSTrsOR<@QJrKk%znac6Jfb86F9*u5DJauDtpL9{SRtQ7N zOaAmhQ?T$KvH9KsJrG>|V85z3Ta9s}8e*HjxeX3NI ztIMzskWDm|@9P}VOtHX zy=d&~*X|NLB@|=RJH~s10;t{We_t#GICC3Ki7TM-H%Z~&SA_O%i;;cGAdVFvG(X^vd2!kE-nahWigUF&!bbF2lRW6fso+>anqLED{+IiD*_kRF*-T zduKZ$SFyDg(^}DEZrt!*ws2tog~)&zZ5MRI+JV`4J+yD146%xi8q|h0X7>I*JZYx@ zCx-u-f`Bh0`0>y1R8xj@&5x+;i!)QoKrGmI!?G8SNQjPW9)}>V^=EC) znT@Qno|fcOu+d{c58MrVVCpLi_^^E!Iu^!2FCUFUEn9!Xclq5BRw|;^JhXk`K)YRy zPlo<#0P=k8Piy}pD)7;+j=_n}9!1__ftW(CMmEZKK?Ftqms`R=x+4zc+Yf;HGnqS; z7r&5ExhmdtrRi&{!Q>{x_mt0X&)+fgvZeF*T~rSQ-NLf}Ekyc%^&?;rH&^3h)z11y zX4fI47W8Vh-WUnV{Vs#%i=Vw4%JVXQI4>v`t}BME=Ide zdd9BB-thjb%*to0>pH)oG%zs0>4ZV81$~Zt)0pKQWU5W+B(f6sxw(ylIb%`Cxeyxs zmoWaY(pCQs!QY)f3=9OYO30*h6LC1vpD%HPRqGp>tS?x(&n!1^lS}Py>0{d z-bMa8^!H`_Y4`X^0OYB?=<6=`j~%-1?Nkk&)`u@eB_^sb>Ql#bi$));NzB-wVtu;GHk$iZ`3@b!_Z}_+0 zubEV&bVH$?hbDiH)RwB?%4l_6oza&KhYP$6J^Y;K23L6gg^dX@2B{^Ec-Ti9gnwPPIX=F`oZh#Dtn=a`!6yqd0f z=a0#CKSQ?t$rMQZ*=-453foiwD9`zm3^ObcQO8)cO!WvBEX8*|z)U~0QxX@c1e87Vo2voInReDk zCr2e7TN_`U0Ee(Crcj~LBG2~4)xs~9l;nnsWaL@RKoXk`{HNRv%ROLPXrKbZS5)VjX*I?@_musGwuhz)mu;db|g z!8NO9ndmr|tG_pMB%NYF`cKGVn7T<0miVA#Gs4^Q6-41|bGJbQ_)2>Sc9$`YH^~8n z3#URcS(CC*qD^S>yDNK&4T3)rgxy9ueC7)sl{N1kP4A??)wqV8E7uQ2+{jVo2&w;$ zcwn^I%dII}GqOkWnLxQ)p)VJP38TGZ%>Ml@ICW8l>p}s=;<;MC*b6d%yRunQ5Y?RGHZ(tB}Kk z8#PW@xu}4JY0hv4rU)kn>0p+Qz>-BbkfNy5x|FP`$|Jl*?S-g)_74t{U^Y(h;0pV! z8g_|eAT9CWVl>Twq=Me=N!|Er-$+-RZ5pdX9T_nytKKuWkG#&SXs@R&C1l1Ny}K_( z1f3DYNv+zk7D->V(cR$(f_75WgOI?$gad*$W!r9&yGXVm{q+3BAF{FCkm9X9tLRHg zi#kO8$r1`CV6<!m{kv*7-3y=gieK$G6%>~0KbI^-MLmhdOX!F7#3K4d@a;=J- z3vPcq7u#w6+M`f11u7OGvIN*-U`byWX3S{6{g>H-T^@R(ii&kW^%8y^o5db}jE!a} zwvNBdpEgWXT&()g)$bh{hn&0BS{J%<_(Jp^M!_<_EVb0h!@wdoK8 z#aBOns4+`BSSoRM?H$qCzA_!zK(q%8(%G5B3MuC+h6Pps(4nA;PtdlfxP~m+wd%|I z;Xwhd;ilS7i@xZYG5vvBgXMO(Ne*Jaa&}k8<2PoBEqCist9F$sC)0>WbJLn-x3|fK zBsI@6vOXoIQBe=G>DhY!S?Mh+Cf)DF94o#v07lX1f6LCX(HJ4=FszJ`jQ{BVqkni) z(b3BA(ZW$rVc408`6oQRBYX)Ez77(9qt*Y#j{xxfcL71bRrp7U4d^t!&TQ;&wAl*( zpT)c&Q@qny+YQayjk}WII8@?PaCXfwwle{H%i3L2Du>jPVJwEf#18EmkS)5OC%UGK zOqjw9xi75@j1(#2zIHF|B%YO{|AA1gkwN46%$vndlkRK82jG@vkJR1SVH!)^`+evW zO;6f#c(CQ<5*N~A@65f(3KHWj8V`xoP*42JJ5ffwb5Pz8sBEg`X>+l@5~d%%?m*rt zFB;B@8H@B8^Wb)zY+ zZo5NGrd+3LjqCW=HZb*~oybink8uN|=T8UtLyu6j40r9t6A$8FJNRJx4#(Coa&zvL z3T*{Iu(zu{thgs0p1Rvpg9<~eUB{zsE^F4nu-(C5P0)0Czk9q6SAxPRte&xa_q+`- zq5{RXs+CqDI@_^@5ousqoQ0oENIlPq%~)ovH8P?XX&9X#?yeejxmh1vp(>?}0`>1$GY`Ad*LtT>225I77QqWk2isI{m z7c1t6v*hULVE|ZZ|LHx_YB;wqC8%3n;g!SA)gd@InhXY8K1hdzkb(8^HOZmw|Dakra6$YPM*r8PAoj zfwB6ndju4um2C`C43jX$R<#kEx;&p}i#^L!lKvW?XC4i=7?Ihb?He#D7oYY6rf*=O z^mPwc%8DGA9eo(1N{JULtD$?$^sHn|iSy6qZuoQs?*ojLx#m?HKva^yN8u?p)+I)G z9A%=y<0uL9^`(x7?U26RjCKRLkjMj-pAMoDBz>4k_(IDHYVX_&2U}^wBqMA+!SFS` zB5^@O4*k?xIeTh^yM)>P@|^>Zc~+oCrHHIEIeMafaORFcZkpoz_?PW&F)y}kiyZPo zstkhfX3Ug5;{nQJ#;>0rhzTZP`%~a&iu9d?4hQWC5~<%dtFUiRAd@~7VIlwH(}G!- zl_V-WzUb?#Gc7_C!V;kI2v9gH7@dv^1YK`zX4Q_f(|_*KzX)PEGS^^3h@i~VfTd-Q zbF)Rdh%fBvI-R!%dQV(}Fm$Ih6R+GGH||Be3e*zCl|M)|@YH){$bEJ3TuUNcY7_Gq>O_&0)-fdCUkTnvH*)%H{&r z$cXE=RdiTW>g-LbOP*=(d$2y9xF!rG+p-E+@O!IlZL%9tZQoy}^0SplL-LJ#ltqX9 zz%V63H|_S?I0X_D>~Ko-OmqypCS;s6S`bF8e|2{ms3by+?SGq!QIp$$wbi!G=IdQAMb2-?*!Unb(Bu+h2Dkaq#x!nTWmqktI0 ziZI1Uq>pN+!lH6W=ql9`|H7@N$!HFGe%X9T>i*l0nJ#vuaPlmp0^oIibSq-C&Mf&^ zbb}j@*TwJQsZm7*notuQCax9(>5M3qth`yTRjcAt-$z}QuiQeQrvx};6!QzVLyjR3 zZAOZd6@<`u0KrC)y^-qJ`eqv|PqazL-gz&FuTzzV7Yam7brtWIEmyxPUFp)|8O7`Y zmj)C#52`_pZPjf?tRA$%8ej;VFbcaFAsFLyKAalqZviSHkW&#K%lwk>K;ini!#`fH z9ZpOVwq-r9U9ne0^c#LP&z8*@{FWlx+AP_XRniJP=ebNw6EQet#wwMNyB!Yrv*PiW z09Aqq!hWkR9mM<;{x+c7ymGK(Mp3t__?PDDwzdxR*d3ziJH~4qFPS=FI(9|rt?btX z5lchK9@GHL|7k~J%_ZB~VNW0Z2k;R4=1B{+`~aukd5==Q00YAxZVVvq-|P6fzq) zu%Q;XHX=YW(79$L&@=(-(!##)HY? zCIP$8ilUKgd|Hn+^I$|}X_6i>n7aw>rO9yAFvniZMY3Emku^`^-EkB+d+#&y z0I&$!VXlItYZVvP{M)qr;OZT^qqtrvl_EabFZs!CagJF2$>cFtiJsn}v#|sYOfMMX zV`CVEp|7HZ>~$B7>LPCn*f4uOyPZ1jSFIVB;1I#;KC;SHceAu`$hW-b;Toy6dq~7F zhr?jYx}vh8@R0**x*$nI%S+yPa{3zsDDHrPl#;6YgX7`JVOISe764-sfO*ZFF~ainVGh3;A}lU3uTy<+5wV zN%*eFEYy!PHR&zqaTsqj0PL*b+&CCt58AhEdI>Wj3m3!kBfp@HzY1QQJJ$PtwrZPe z0gr$)h=gIYk5z%>BB%DP6C#Gp42W1oV(oX*l`@s;9vE=ZUU=%03bQ0GAhH<$_3z2xJ zqL`DrXUclzx^eYpC`v+P?MvHX)kom|DaccQWgwI8O;6&cb05^d=B`{LpR*+&t=sLH zFH+%2jii9RzfT!X%JxDN(Y6w~K7gr>)ed9^&-$r}LBj6gUMBb;(x_3 zP~6g0mv_@|!+4;m5$zD+Cot#(nS2McQ-eK|(=`TTM=FWgZA(UQOMYJH!54JH1mQy> z;K-AJrCjsuS}9V7gfEta+S**0|9et0$>Qcn?&r>bI13tZrG$`G?HS``;}=`;*D|71 zUhrYVXLe{;4E@ zxNKtML`PYoNJq*+q~&MA239(5{d-suAU?UL)*}bn>RL#era5MQZr-fV9*B3Zcq}g-UpBP65;HY?i?y5vCrpsoX`CAM@*RZbXd1NaAM!#f7`bd=i?^0QwiYE9=(|K=eEy2|}A%1Wn{=7OS?cYujMGfMy`mw(?LOWkaS zx7cR|GJpwoq^s;bT({?8ur2ZQ=IOA4ng#5Lz?jB6Qfos(jSoGSNoHVHOE#|vEv z568M+VL-KC5X5r;3+netuQ`|OP^DAKubwHoQh2RfGn0cLW$>mZ>jK(eL`#o(El>z0 zijCcHd}*4A&I&;;|0VG^nkaWGhIMUQn^q&QW_W%UNwMABWjAK(Jj#g?{{Q&~HKJBm0J3+t7};lFx9{3R;Ff z>1uAIzps^_YhTcNzZh@XP%>yS|2)zPW|Hk@o21VgREp%yZpaQjZYx$R_=Sg;zKV3{ zBF2xLxvh|nT>IKQbXbgX{?2&Pa)k+%@^1s%_{+cwC5LLXVV(uRaOI9jxvHrGp8dZ^ z*GM_wSQtcnHL$s{sUYZu(L=!XUu))IxYYD;`#Cq(M?QiD*&#AoQmRTK9BxF#Ts$5J zpR6!z(KNj3qoJz1IYBXkzBCq%x1z5tu(RvaRqNG;|87RTTmQT&9xA`$ji*1$neilg z;|fh0_MfO)?{9kwU8oS}Fu<*)3HKnvU~H<8)j=gk_)hc^!McRyeNZVuuNfP);kyV?xYB@Ap5y0(iQ zk2qtW6U38m55=VM4o60eWz~!G6c1D`S98@FaupiEqQ|1W*R?*c@)w=x?f0pft^GYz zKw;c2Du6l-QWFygGeCWZy(i*5lMeyNSeDi8_D1``u`Wu}1p33+mxDX{k=DwptMI)& z(_PODlWY;FwoLwBNl5jTvGs4PCLtJGWH@%7h5m@_qhoSnmjRZUvC__lhpEG1Os-O} zaULV=*L!Az=|KG^a&$LJ98Tpo_s%2um42Qne*}O*MoaD!5`U@_42=t)oIX)Ez>N<6=KyK%TLb(=ek`297f9w3vT4jX6i54`|M>czR@!c=w{w-h7?nXX;n~}rBfI3dUh%bOTf3pp;&ktG83-n50f$)d2 z2?J#%DdD&7kuNlZHPzKmG&SD?8b8m!e-wbIkQ1t?`T9q3#(w)^libJHz_?w*(7^v& z`{#^j$kub_H_o$w5eP(fUY@k6X{N}i9kN0>ivK$4-9wtN={tvsly~m}*GyG;?p~oQr;W2kJh?+T-EKK;%j|K+^=Ul82``XohO^ys3N_8g5a^Bk0T<*cN z`eGV%qW8)MrK62i3%a@SEailkTv1CPRSIxBYVGSE&NEq&f{U}WbAL)N(_7TjVPOt*o=nbA(R2ctv5qIJRaYamo`N2CcJ@4e z{GymW#%OS`f8MPi4C()kUxA#HgfDu3vAxZa5qu(JX%1;iczQE0r% z1KIKyK!O|%D$174xkV!^!t`zhI4@g9LNfD@#kil2~D2VDleP0pM2*g5F zsHeiVX9V?O3g8fGd55@uD5KKJY9X0GwTY6iPUyvEnTjzzqiErI=t#!t3`~=*)v=^A zCf8;S7av-eL`ACgoAo?L%>!cR&bb9rZ*trnC88Gom?`@%{YnQ(T0rNucHxN7Z*n0V z7U3iJ785`+R@g^C44*Y*#^PJOhV5JWVVyL{S~Vlijj^CGoJ?KFL|oCMd45EzZk05o z97T}j4v^r`URj2B!s*phrjCN zYa^p&86<=)Q6$QDX}JFEb?IBE-|+e{`aR8d8;3&}>H%mo?UWrhRWl4}&4z&t8k>#p z;oPGx`GwZkJ%{y>aI+-+qphb;^$OEhUa2JA1Btk-WZzEP4YIy-=yLkz8@mGms05hQ z_P&I#t1Of@!r8If+l*^i6Ug9@Nr$DesvoX50FZluMW!D778@X2c9B-Hk zJKKVLhp?v~l7L!dFd# z$vajb2*X(}$zKig;Oycr?U|8+mmXak?WxBv2ws9lPqmWrDMfafNQgFi*G)67v@lKO zQ6{oMH?Hoc@Y@A16}Px8A=pm5DIGO)0Fa426Y#MQi6MjU1*L3zh^r(`#&aV7*i3xrMCt78e08J9c(Y?> zE07eQLnr65(s`B4z2j`qQw=eNcEfV-ZPip$*txp1DMI z5kv&+pLaps)MQCq-wtzSnNnC6}4o&${YCE(6_ydY4j5Nj-%PVqu^Mgg5NR~@^9 z38eD_Qb48$-!**p{7l{_z)>nJHLmsFEfYSu{fA{A0O$*y1@M0j0uYh?n-u@sZUg?? zuZj4LHNP8w3R(+WjhCsUJk)M2DCGa6y>}*qHJ6Z+o?T;iflGfr+m=R7ZIaC)&o@k6 zfXS0MCI7|U8#ZYxYd5$d|M8vY(7`d$Hy|2he z4W9OTrLiLGGj4Yn56?Hj5fPDk4`*Q+Qlf6$!_Eem(5-Rf{nPuG}# z17jIodb#zBc*Uvd|MR1PQGFT=&jiXOA{U}khoGyR&YC7SvMP&xoK2YRdpgo~>^GE#<6yBg8tvT^7zeL5uC6!n;oop`UMwO7p6!&mB z{iX-uJ<-dbB`a0(CEpFXD>lzxwpPx2tUT}pZyAXf3_Znx4#4m;5=nqS)RPZr>wG6`4%pPy)i$kt?BzeVqvdF6Zqrq}GN~DeZa7MAYxGFT zTGW*`&-qSTZ6VJmNjxBH&wV2D5hyT8HgF&%wb7}lly&q#CEDzo!?;KIT!#}Kgnc~g zO#J<8{@8$!5lCOit_ZL`n#aAPzMLjna~dEnIM>==;PH$NjFH*!z_EF5ZJ(bw?)0qv z%uI+J=$#*_3AC?Ik^I$SY#Ix4VNYgwH(bxzD?qTEg;Yi+(nFh3AjQC!Oq+$7^ZCcF za@)^~tGivz?tj0jM!*rvOU?#LtmD1UqBX>-84Gh2BO&uKcImS(4QNynY`#j{WzA+% z?9XDfZqDGmGY)=gziW3LtRyDfU3QwgLc3EPf3D*LYIn8xF9rrVgZx!CX2{grxls2w zY)kLs3YyQ8++ScrrLv8Kh^^hgO)HHW7{JXY2RbvE&SXmFg8>j$fK4(bP|KTyj z$2lw;>p}6HP)W-IZKH0tQDU^oI$H1Z10YRL{k>Hu(jl1zR@Z%d9vBU^fk!`T`WLXW zZ*>k+i&dR4CAmDj(VJoN;u|Y?)T8XW;s|Wk`{H7@#nsyOXa8Y90mn%kL7Htt+ceev zthOTU9M#Pjx)^59q;FLpY2jSqx`zM7UWJEwH5aGA z?^1%f4t|<58;peqvW}R|ZIx5HYl&Ifn)IKUV+3e1 z067T%kyN2iEWNUl0tSPPmYUtfYkn@6Y_SA-i+#JF0`|09*q8GRUD&rUGHN<@Iuqth zb#0%S`3tz@>_c2!4)Hp`hF3pbg8@;X_RZkqllINNuV2prZXm%M=!2V-G&CC0C)=;N zPc!hof983>`RCep2gD_|-I*FxBO)S}Dg%K_&H)USUTCmfe4}@AO^X=Ubx{;(T@Gm7 zFYlPPW0Mr`R=x&)FO2XdrR)8eFbaP_QN=BPGh|Pn!H4NG@FI9&ao`ea`~?RxkIpXJ vxk3W$NZ@~^2>#!n|96IE3c9O)e0n}i!g0Y{`Frj;;GaBLMW#^d`P=^me7P0i literal 0 HcmV?d00001 diff --git a/Documentation/DataLoadEngine/Images/Remote_Attacher_External_Database.PNG b/Documentation/DataLoadEngine/Images/Remote_Attacher_External_Database.PNG new file mode 100644 index 0000000000000000000000000000000000000000..c2dac2e08eb45b34a2f80480ed47af5c48ee5557 GIT binary patch literal 38839 zcmd41cT`hfv@U8#x)f;!kX{5rml_0wpdcLtDG@@I-dlj5A{`QuE=@o{q>1z{HT0r% zq=p_^Kp?d60{qUo=iE2O8~5J#{&-`AL3Y;KYp*rex8^tJH+Q(EI+&D*f#}+`YoyOq zo@!sab_03s+I88xcksWg&<^zC|6O;}1}j`E?gy;lFK$`OzmUIntrS6gYIYlcP3Wv* z=yvTIS?k5$b)?f*i)+^s)t)_-*Y!5tOe25+meQImZ`SlobAUhga7ca(yvt6j!WGPV zcTpvb>ITd88zGu%_L1fUze(s&j`q&+em8r5@EcVwIrdbt7kG88`;Yh;8G4NDpEYgQ zN!N*#i4^FSd5tw~g9pIUBHJhZ(q#n`){urebbR{Do9`apz5Kj+;tGr29SUja6cM#? zWV`rLBP9JMA^5Ij`oSne`}2L_d7r`HxA^YW`k5=$B-YSO+pGttIik~{l(o`AswymmmlL^kF2Sx6Eg5D0vvv(>=3 zw6e_NbLdtbmr!IFTJ3AG7?&w1aIM13P0N2{n_F&IRQOk!Xb&Y7@*`ptm3BxQIR(MC z&~8>jR99sQK$t35HUCQbfm4&NvH^Pjohd6EuKuHSx`zhy!#H-!IG!c%FKivp);Pjm z^n>T-q2otuOt9rD@{=26EW!d*gozQN=%~lxw(VKuC***RE!h(#xf^6m^186mu;cHC zYBZi{k2*p`EDii3T(WN2!Q)@=3*7_}^0sh+i7=QB`PD^Pdo4;Qk}; z^LFVNFDn*dvc8U-VckCQutyrNTyGG;2ju-br@qSWQ$~2<06Z&|b_pDuAwQuk-W~_i z5w!6l5P!u&@lQr<^ys-_5`t4BYHcs(|^~p|G3~M1s@2!Tq zp4rl~j&NQW><1+pVbyY|!aK_J42e`rkdxAfBj5>y35>ig$m_OY&w0k|f5+2fI$a#; zU0Fd-nsMkDDiHaaWX|N$U3pyvvIGH0_~*|LhsVaUi;Kzb1YK8BYVYg^D!t9TaesuA zyU6~jUJ=ET-iuCPK3Ey_><3#lT~oa;8(I7q%G;KyKfz+b+)yB^d+)pJiZX07un}Ip zsqt9~o6)I41ojn|RX2H8ZfrrouaH0Mq*c1YWKmK_9&XmWQLCL}l1}Hv&Vn041cu0+ z@AF)T;Eo?Vw)B+EwA4g)VKY)1g9FdT1|lh}`t$c(J~{Z@z}sD}sf?znzD{wcpiUZ7 z;Kx3`E4x|!yRH>pw4bHA63EC^WA%Z<0*S${-ks`t&`t%;+91;pr~m#N*697 z{5!_$FBkWp$Am}nax^`!mg06$4a|?H9Z|UZJPb?4_spi8&+ki}6Od&zXDLO#zvm`d z{Oy~9sVS4Tw)Qw<7aPZT96-uzs?ELHk4h6@L?jVKMIZ$w=#0SJ zOHHxuU`?V9WwL0IlliTbN{nB&liRlWSaGd7=?hcZFTZ;={XuqLTvP8?kp`bQJ!zil zTJX}}(C>47ZoH2O7b*4kLvPlosHstF{}~>B|Ko?Sg{376FE5dci_6#IVr4fsk;%zP z=c8RqU0vPJ<>k~OR=wYyG-SfRE`}8u&1oj2%7R`tBI0(q?6-NyQJPA!WJ{bZ)eo;@ za@IQYd%@2$Ra`_nLi(|igZdJ3HX(! zmDp?e*Q-whq0P-l&TGhUxBBmBCYhS~JZ!TXP|eV@%wQR|O%MEo%>hPIKyiorm`_FH z63T6L24jx0MmTW!_O*ptt49*Drb580>!O)~;ME&naF$;&33m`vw zLitA&`)hpfosvWLQcQzb9us?XsK<>|Ky^NZcDp#<{v{&vA>NnFknrR~;D$B#__>o> zR0+8GZ6Fvol)>_722wANK(JabmhO6PRp2p zA|fK~Dy2g<)J~%XMMX;L>SPH(2%M@}>sw_no4$H91F!P|*7yjd-I7s%NS{QPIEHdo zrxc)OXKVft1>@`-S)%{StE%Lw1;DMOKtkt#LSpXY(ST|ZonkoiS&ABybQ^~IPT3qt zemp>`&$K6Vhx@gp3aR&fKT);58eaPX!n*(8%G^o1Dly;n*6)p zC^z^KG5vn$w{ms6l#c~2a?(NHAg7Z;8{F06C**1E=O+hVJ7tiwlTI+ahHmeu;g>Ki zYX3py&%jyvHV(^qZS~+MlT4H`x8%;RwdJkL+9;DbAqE~LBmgZ_4!iQS0aC7j#s^ib z!ZQcztHkr+<8^X_^AG-l{y?}F$A?G%$%;&gS(l@7se>q)?%p2Ojc0qj#s1FU-`W01 zN8)sdB#wsWb5`cf2V^D+23fx9Wivr46*I!DEN2Dnp4Thm_Np9KrrI947X|ioPPw?c zKih8kz<5TuOCIGEJXY}2-`{3_+67{(wCAmd37t0aV zqVr03E&0)pql3tjAA6krk>%0n`r8@tV={>4fQjvw8E0>~K4fG71rlJxNj<;}<>oxy zwvLVn3Uk~YS&(^ep02ac@>C+9VM}@{W!z>sNU~95b0FscGm(heUs{RG^h=MQ+Jcn6 z(AO9V$reXCn;uc^ACxNqzB#te?@FZt`=7YW^Zg=5X}t~u2WavA+GWE;r!>MpxjB{6 zdJ3r|nnY4dxR~w_1TSGN0yAm@5e)lnsG%Qy5&JmJgo<><_S>^VN27IpgAM0#Q}%~3 zHI19wNNs%r#hvra27x>NY4=y{fXWas;ivUoijLy!Bk66Wp}fO|jRkP2-K1#h`67Ae zl<#BLs@YJtI#YRtdf%OQ+eRmlFVI@gPDJa_k!GAr1+uI2+_M_sMd~wekV_j-Gf3xG4Egq@u?Kuh&974<5#+sVkUkn)dmtBobO$Fx0T<|U_4!OSbs_@9$ zW!ae2iwx^&_+Z_m{zZ|Nr5)r;6UANt^=1!l2Ea5$q~HJJ#Zgt!Z?6XAd#fgkDP>H{ zv9>1%jetq?z*$ez!$;`jm;uS<43*|tY=;w)bH!|L)Wl$=&ahr+dpul3B4D@hpFs25 zz}7smjpJTOltj-|3|oUje72st`4oJh?ZGzI0c5h!mhU!two73tfCS*e@7MRXuj;Z4 zyYTpu|6zcAe@rgcH!WB(%Cdbb)wbe+qn!~FIz)LnR zbC%tkPUe)SlAH}=+0G@mIXm?E&_o9&$Rxh*m8X1~6_$Za8|SYr&sLC@XHenunB*RE zuVM>iiGldo@5Y}J-;f{rv%hKd@67mZhf|RomWe&lJ(ZOz&O7tgg-(S2 zt;+sz+e7x~NA{HZOyDnni+m`$1WLli{61fj@hp*TNXP5ebVc6x_1{qJ^aFxYmmX!n zc-757KRbS=sU2>ny%Q=B#^havk^Wq`1~b_C+qw(zyR6x(fxH`rO?n8fr3%IK@$!LC zl9#a5ukaE2YQPwoFUvn&m^0vX2<>=ete;#Xlb-$b&<$Yk%po09k}pBy=}qA6-&Pu6 zwU5U>Iupz;TZJns1>11A4qCj`S1y_P<^e+fWS32r3m-4TCZ9fgmV6J@+iOop!eP$H zEVOJ6Tl>Yq(hR%yJ~l3nJy+iuJ1mJuvF_Q3@qkkg6Oc{zWo!CPQ;h&dGj66?VXex9 zv7-lCn>`1IoZx9|PkU`f)7eSdm2IdSJ;B+JQ60uSDu5(C9D>^=z-OBEg4>%9z#!LN zIo=nPzvn)1vt-(hIZ@u|1;=y1Ma-5gSy8`&HKjj7;brE%6Qw=mv;v=2p0TLKpT`e=m^A3OUetz<;5TT$IP_8Zhp55f*+Z^T0x;Q+#x(IO)XI=Zkwz3%oW z-a^7u*-69hpMNwmqHlgo2GABv>P+XqLbu<9_zni|z-%mNUs{3?FY1L9EjF_nBAuy2=C29>K^d~ELFdY#OoZO-78 zguo)l>**PO?15KfG#IxH#(KBv^Gz~^u+a*~#N}Wb-_Q3KWV;D%VBpK8-;^s}lt1E^ zegj?+FDdal53dy4Vyioo$Xy+G8q#A9-TI`s%55B7`3JExSh$G|9te=B+8~+vD;p|p zUDTQ;9S{ToU^&u3`=aU6I6y$<9$BiIxLlr%`C7&iKp3CSePB#$;| z-kzOI)6Aa3`a$p4WAvh!-{MO5kmQ^L_W%d2s*exLrv;}B{=EZ$>0VCdxPy;m9{fdR zp%=5XlzM1j-J32iI5{n_BN7-uuOWAKJ5~B@>B6nv;C*4M(#0olFwDV{bsEcKPWdY) zPM=++OE6N~KsSO;&(rEvu!wo@eD=@0f#g}GluvqZrmbPFWbfc+6(kJ+PDc*3i=JIl?bXU%^s+CEq*_3;YBa88g%+!7Is5Zr+)3n|4X!OHqsUu2xO^ zERBEnQ+Be?{pd<=zc(}SB?N^JAX|dKm_DOMwqlm;;_)b3=CtO*eY2!04w7cASLZ?u zT|Q!{VW?yIzQs~SRo4mnD`VOed`oWnM5TclNe2i0(?4nBu9s$JUVl4LWOUx#;;sOv zB`dEOql;~RbrK*t@`%7?=sP(rF?q}O-&xEUqZfCY8x=)tz zn#1_eMfhVAqd-%azv+0=RxnrPpOD$2GhuNnUzRT;#5#vbOIYK#w66#nocYFxw{Q8( zPiQo}e>zo_=T$6PDe0FA-{LCT$4AD0({I+?Q~k{y5%J}y-AZVA%pIh_7cP`&j$@+4 zK@`Yla~%hwW=hwMBa5b#os0QaGj7g=)H6Z!ySTu-t%s;NCkt%TV=!{L4I}Ytd&eme zXgBP3EkoUXYV$mD%BhJ=?x?G2p&v3WxElk-Yczpyq4R2STgG$Ii0G6mHVJlptpqm# zK2x%UWiV6s2mWRhuoz#HIROb0@OC3w*JbGmp=J#om6_4BZT` zVkVM8rg<{Vk1*{jz0dy1$YA!gpMAS2J&_jUOQ{;P()!q(|6N%>YhC$`f-@>*F62Vx z#`Qw)hc?p_zCFvHh2-j7^!jjm`kMGo!m~W>h$R5_#uUgX?)NvRStsJt_?*9UG)VqO zY)shGw6a}-sw29Jf%x^V$f_cV$MF6X-xSo|*Xp|x|7fJmojamv`#&*&z(v>c^Yg>P z!oI6l%(>bNC%1WmT6U$tchILpYBfU zJi*{}YIO5V58djwmM7M?@6_5&Y|8uo^gbU3QnTF7s@tM&QB0SA{RQ~3 zOwVcj!1^%AqiGp+|LnIvz$n3^m~j%B>jl!S-a)lZTtG3i!Bpb+TJTR!kriKjTv>*X zD?@1=1KNl~lL3h&-t$BJrr4nZQ*Ym6U3M$mv(42*aNTx4Ag?C%th95>uif_yB;j`f z$RlbZ+w!EQW_3y^8Lhd%&(lePXqvsZyzPI~Mp%LmOL4x)Gi~NX(6hB1Z?~-*rvr~ch%r+fEo}MF zgPjxRxQNf8?g|F(F)~vuGsoxZg|s%w^K(bWY42&@A>(jhjNq5x5#t)A5T-Zzt0J$ahC0BuOEQ1 zCNLzwo0;|yM{2FU^~*-X<8*XSq{A+>o*%rT82w0AT^WNXU5T)+osXixQ{R!Mg=R4b z^{EBihZprYa3xq{uQ$7=>D3|!sO3cgqX?XdmSHMYR=wfmSzK9B;uo93i%dok2!xQ> zIAcX6K!eSBsyk>2JQgx;AOrW-r~$a4)L#w)-FUw3M2ui63~~}z^vdjXxt~($bLgDk( z>eV)T%*ZGVCz5d^OMq=9x{pu=8pxe*DS5XqqCI)s`4%LKa_`$w{qk@7KrVszbGq)_ z{p`VPtZWo6g?$LQNO_yo8S4kN0_xEK>+i&^r5RbT<<1u)x;741w47Jb3sp%@y`y`; zJ6)Y$O4j1t){UuekTW(Xi&(d{#|OTDkfGfy*Ab0SaAsL%LmGd~(Xzx~OcZPS%%IU!2b_F8ohnjU*q7=Je|LIQdF$pfwFqW;@ba0>Ks~L= zQk|ynmj4%Iz8r59ol`Ynd+VUZFR|ARp0h89TN@~H-AF+PyERxz;&2Xf^*3hW$l%p( zh$8!iLlHA%;7i z=l7UlSSN322xRwAwXC*QX^8b;WXNdqSSW5)7d7K z4xa~q^0qz2qfK4gw{-R9xqE~Nt$7=`!$oWPY>ut_w9k8?plHI$`gmt(%iCTyAv%rA zZG2nJUdElUk>F#gXW)%dEYfU0oes7@8?9HDU23HP7~P6SzG*Tjw5-b3>DU%YEurr3 zbg}xyLLG+iKr#RJBz9FaRn-3@fT^47EHam!O>ghE9cfA+l%ja&fqonziDqG8k?=bx ztR8fwSZ?VB7M_;DkAf34sthX4i>#!JBj_q>T4)UeSo~a?&1A&n?rQjZ$GP0l0KVoE zMeSkm9sny5yYleQZrQBi?V|+d>Cjk~12*tO0K(SuyEBlNyF;OJ-rA z0LHZ7--F2~?2~1=pJmTLcVbaL(zwY3n3FMNjbDT5XsZDypXvp<^mvdZ{Q*c`WRGk8 zy+LRKv=JW2mQx??5{lR{QPTSDN47X-Ya1ZMER+x_MEg3RFTo-SrSICS8Aakpx8P%Y z3)vwTZh72$@*xI9ubQCN(S$>@W5yA8&og2W0M(tCiLh~ZfE+#Ft@uEOw!FH-h{rvc zqXxY0?Tw%-nBNQcVaC$JoGF%1R?g;r^{+_ofk+;k^HC&dM$L_zr$(FoVs!Yq_La3c zzf&PKg8N=7$pCF?b?4olRvyokG1b1IjLG?L5^Tr`tw);ouwskp{*)7H|A@HuW;>le zhAg9x4mLUw8|$WM=NRH}wdUs5yd4a(pp7ei$M*ipKnB>x=IcFkTr%<$`j2{sY^cu4 z<_fxw=0{+nOYyI;fs_LExg4d?oj+OGNwss_>1HxLj&fxmx`bmF@>ZBTelrWd4UoGg zw-eCI+j&~xQZdQafoMq!R8qy+mfzn3+l{^|&jo_*CSk_v{tx)O$_775-0XP2Vipw` zfe{t3E(CWy6F|O#XsJbU8N(CZea);QwBfU#>h2B5JT~dX_)VCOx7|7(dn5-0&3eyf zmPLqnkgh?NA_5B_K7g)L(O12eFFzUaMN{uRo_i+T>+)EV)4qOxOQ9aSK)TU*M*ALo z`pERtL5cTb%m5T86A3_A^p`T=N?zl5?wvw^Nx;-yUlsRTN2))BHSL0;RvuiFjWJu2 zsJbyRgpRpiMMa-HnK9itwzpZIidB}479Nv2MfbHrMVxo-=}ZHTEM@kGb-WkCxpo#~ zHHdHrRZa6Fh9!2R92=7K>Ct2Q0Hj@kI*Nbxb$vy;TmS9}ABtqgVrRCbh@^$Q;hc35 zi>jCsox$dgR+hB2GE}pEvnq;!zR67Hp`d+T2F8xAawaq@4e{`Bkf2j7 zQ|l}|qz9u|SHF|e_gqrFo0}@z+G)d=scCv4SPr(%vCO*Xkm&l&`5Cl3EAR6mfSZ|q zIjeV8ux~brt50!xFFI!mfWuL)iI)^OBAiVe>oKa;N6XK==iZ_*Xb1$6akk-mx(Xgc z9Zc~@lE-Aw_J)82Hw7BQ-POgt+qVD_b@IP zbbB~|_an-Vx>oSwjxpZ902PUrg6evD{MF@0w6y?VH2aUM@{a(jtMBh`bqW33#o(bN z=T#a1E-^du->zoO=mS689eg4G@`F>pniIa2@_+PsyO{KEJKu-@9QgJ}=)ZS;f6awN zdj?Zn3@HTvnqf!y&e=qCiEcCF>L)V%P4E5WuSTZ2i=R+6?bYByZ~xbtjH!ClaCXw= z-O-QnH_n?%a3Gt3KFvKnOxuW;5+76Pk&NTYy~{g0`&i8UK6Cb7|4ShpuB z;%C0qp0sEnG`*1XIL`82Zp)P8aqx5aZwJfz`YrHx?y?(=UeD3>b5s^Qsuz!kzAkj_ zjkz^pD71XJKp}NDSgh)Aa-gohSyQIcY$AP$e$2p)`u?uimQNG7aLL9y66g(TZow=ec6J?d!~NBJ_H z^rOxzbC|-9zLstc6HauND7;%*DCEt%PHVv+e*u?{*qH9p1Ugj-wtxGYy$arlcJrDc zXCCP+9Z+44fh?_-)Pq`ugB5tWa`m`#->f3|F3{IzLzfRg+{eSj8K}&C3Ap*03;f`r zd4JM$>FQG5=zX`8TuJ!IYz{@j%x^}tYwXSNJn%vYx zmi8G^eOl;E<&>(Y2$2+imhFBU$Z9ii%FC zhtckGJ@hGnGU9^WS)Xp=;sr?dC<}=yy})RK11z?A3||q+TgU=di2R`f1}o5)L>a&* zdMqFlfw3FsI3LD+sRs0WYFoMPN+qmKzPF2G;0sKfT^Q;_(Idcut&6OrbwMN=mZ^`El#ClqN|nH$z#E0}~%Z7h#yamNvM87z&{?9Voh zioFBdrP!ybVZE#WCe!-RPlzdXruO6hU>O#$xVg|x2U+o{niz0^z+cBbiE-tz*LET2 zC!Zzp)+mk&0_-WRJyV4?&^gSl8rw7SKT_u78;4+csJm}jxV!Qhtqjk~MOsPn?Czdc zZUmQkJWupb(U<)D*{;*sl@G3$8XGf1zP`t0lxz3hG4<$V?h?UJ*rEZxniv~%oW&~} z(*+zQ6KG@gbp+J6L{R^%cjyJ`5?ww*XlXz@ z^yALZ39(tc13lg+Kfx?R-_Y-Ez?4!W;*a}xTzkoJ$=W+t zm`=pRLTUDb4SkyLpNHtkB9U6FepU0%eJoA}p|6ffKVi)B8`hw=+iKW*%BG zO^)ze!_O*|hhG{K^jgAT_D<`1_6HyA!bDsPsb4d3m9#2^e8t4sgYzJJM7NWBjzIaG|?OH^r7H(l<8OmCtg8=z~$T`#|hl|jnjH7=~S^$5V$C?aFRUtq1zM z-IxGNe@VjuLo772wdifr^a34!3JM0$(l9u$ z58J|aySOC0wo>WD8oe>B-wFz1Qc@z>!0?CM+$x1ASJ=El7##3m?ol*dB!vZ}QaM8D z7tFsr@9xg-kF}eXhV;Js*v7>rgLj7Xr5dVfuZ>dh&+)W;ko8fGsn6lL<(JC)I)Jw4 zreM3YVX~eu1Q^)uxp~4m`l+WA{_+HCKz+7rX4L+?4~3^1Y7OEGGG&H-L=iAuqv6NL zKm2}XZ`71Bgl?JS6ttHU?R8=#?Z>}gzkmO}jg8H{rh;$ZH1XwR_T-YD{{9dIg1kRt zECw2Uuji@ZdsT*5U{`&0=c}dO`aWJug}E=aWb2qI;aJpyjcNIC%rWbF=ldbY*#!Um z$udM?;et5AcZwSh7Fj=2DV@#O_uf3n^GE?6Ck=FcN4~(Hj&zF4jqhMb8V7_%3I&o& zp%(^CnBN_oRw%$pLnr}q0XLXoJJ-$4&D(5Tl$Ar0w43L9*4EbUl96>l&W~TFN>UZ_ zAlLJ$&A*KDn!M?XQSgGMXR24k_M3$N7$Q8Q~uNE;In(s|hO9XjhJwvtH=|kt=k!=$O2}773i^BnxiBOR6 zHx4~>QEh{)S2y%}6jr|sSrII4B=nu=_a`$J!(oP=MnabIAO$Mb zyR!`1FPOUi)1MdL8`dn_8&%ZMR!$l507jRMpG{mlmqR4(%Nk1HEw{ql5&8zwfYtozsI71JR|PT%bTIau9YDPD0$`NX8BF&9PX5|@ zVzj=gmUT9G?}){-|A>Rhe^CRw=@f`&aP0*vZ05RM1hc{B7X*cOb*6BgvmrhDAl#_8 z`^#-*uO*tK?w8HFwRcTIYF7DRq-j1tA<6WJgF@tBy3lQhHF9A|&_BmJM_)+|-{Trh z{~jJ1y{dl|lS?P@>En|No{9L^CshNgtQf&Y{X*qSw_B3xi8|UW1v&ZN3k5Jmxom%y z^d$R~ZJjN-eqU5c-C9jir)|%sZ%1i!?eQSpQe>w5SdzN@#c16sS4{N2r!=3Jbf2Y1 zYyMo6r13D*W_nt%FtJ)yKBlc!+Hk^8lR_{$e4k!Wy0TpPavCR>2?K&K5lTW@njtTA zEG;@H$NHW+mhO__UR;r3TTiLFudlDBp<({#($bjlzaOwgGaWq_U4j?XFSiu+t;@yO z7{IBK&UaF81cdDuK<;O#V4|I1H)A{Dy68p@`*1^-~?`yb5-Wj>3;6t==c*Bu@iE#!2=0S z_otrWV-oyK;FmLLRa)tV*uD!RD zn#|P68%@$iyLG;*e+v2(=SAnn-!$9YrIB&p)fh4u(R9A41Ur(GL)4BZ+W?b7`dhd# z73asaG6!uUo!r6;#}6Ez%yn?6!S$HYoa_U{L)1a*l)AN4ZcjM)&Yw;(< z2bgzNJ(FA)q*~*8^?Wr~*h!ERPeR3Fb^9Em^4s1jx)A^3)a~QRo{4SOry)$ui(Rqz zw0^yllRdl9xSJEGXt#xT3ukN*WYdU}(-l%|pN@{p_j-ykAxSb>FJ&=Td!{t^uI6WU z_=jV^mM*Itjl@EON~$8^bgGxR)61&>2cJuv9G?_9OC|vAx?N)A)H>8iJXW$xB@*b<0am)-qX;_&>P!eA`IDlaKpDQ)4gnHJA(^y!zqj)h}u~Zt!$=42+UdA>H--65X0AJyrT+ zl_}^LghF3rwWn7H?CT;{<2U&vOP5e!Y@sggDWb8377(*q zx+tOBCx)HYl?*B!KHTB?nY4iaG1s2}^jO<2|A0(%6FpimxvLcarn!I&*$S=OvyX&` zd+g65dZ!B6Qq5KoSI0VRY23!XvIPuZ>UqV6>^PNOEsRw3^1nN|H>nvltq8yf4V2=Gqktlpg1t6jl9v zf9A6zB*e~6^z3(dj#K|gBEF$xO1J)+R${!lA#?A^=gDHP{-=z|?^RD?Hbusfzj6(h zfj2D9To3+8O5IVNwP(GEJ5UcUy|6MLQcq7$zZ%e!h)3{$r+X_WCDjg^BnDvSfNmsi zhT*SA;*Y8JRW#qv*Dpc7Q)PmJgYe_O9tHu6lEz4j3EXd9QJffXK;_o+(v8?zy4Z9s3;pvx8czCPyYX7D8K>q zry6^a{^d-|QK3y&yVaueAe{7v15u_n@kf|XwZkj6Xe+HMh#bs_Z99XndBdOxz35L@ z+<4Je3F6IV=gzWMj4LC`uOU5gYh`U434Qkc#|xI8Z)?;b&@XBle7^HknrvWt2l?GQDW|sV!D09c$4{;v0uz~8NC!s z7Co$4^@Xzg!ie81aHVZJ3-tbqJDza5SO3NCq5-3IBTESfl)Uy0rI1ax5@he$U-3JF zJTq_8*Tn?mf$#DcU%21zdt!~#bzcO%gBQ+zmPHBt6~FzN#p?PxTjGuNfycSVX?Ix| zA>rjOLOL;i=@OCz&j}io+0gNzZt0a}}82bFeaaHvbCUln)$y z{#y`>7xq28TECkv2;5epL~mJGZp$nE6y+SXwz*8JxE-(Xj?V*yMB4)Hw6AUtJ~$mw$@7m%uLcDo@f4_#X`Fo(Cq(p=9-eSvPCOEKPWDM4T(6e`s+K9Y(gMkD21#Xj zT!je4_{p=m|HjGUDoyTUU?7FPz{+YS8Ih)oz`V zm1Mx1E)rOt4R1X|AXy17^Pi-&h?2ax1*7mjtce0C4VPmxd^H&BK5Jz^#oV%p&3XQzASOQV30 zK+!XXKqqg2#%M#>3W?Ke=pc?dT{kTlg0zQZI zJudOys%VM_fF_7iS1l9D0U z8)r;E+(88STl^L%Wb0lQ{^h&(M%AdQ|KP%9# z70>XV#;^Qy@Yei~m{X_=bCW#j_5=kOF9hZb-S_Xf=N@0&p{NE2#?xu(PyMs->CW)* zSXOc6HPVXUuf+w*E)KUNzCF@3o&IONg-A3`?;)swN#`XPZUA?Pi6=c6O7d;461l#A zV9CFuq~9TRqiB-zss3b%PfCSkySUGyzu1^ocU2K-5HQgS{NJWR$Sf(oye)5S0;okR z4yHHJG+Da>w1*>J0xQgY-#m()zfo>A6EeEt_W|MtKtF#uAN$eL<}0rvy3Q%nV@Reg z)N*c3tiQX|KJA0C?c&If8R6#zu)XwJ{-Ymf0obtb&iX?Mt_-0bDL%~s=j#XF4?cQF zc2?RQ#9y5FEy^qR5%5Z>ky*+vduaN@Kb0-AHeHiM&<=ffvE1yYyz4;?Vsskd?wY3sL3_fQqa8zw9oZ){;t7<>+qI#>X7(CRY&@9ur$k__X|=OwhsacbPMsB>SSmJdPHu#$hwL5C0Aq zDtaxsCUq}o7GOQ_-uvnK(mhdjetG{>N)oca1#L-SRU5tR%1=5h_Vd4Nnc_#@sJ;@f zvSVz}Qz?0qvFlKKGB%l5M2c5R4DJ7$xx+K`W0nYXOo*sZZ%E3yGMmIG&-j~H?WEqP z*QTjOycFgMV9s4IT;+vbPxI0E%(i0ommZPD62P(!O%yI=Dlg|m81ioB@v8_Sb|M^R zKszzYj`wvK7e5xz=-W=frcUojxyiQQExrfIm-Trdby77&8(F9k@p#epf3oaoib8}t7wmbws|BK)qrEIx164RqJ*_}nDMG(A!`WKwfGcUMm^UO*wyOhAel zqG&y%ss_)?l0^%ZTdT<%9S5nw$v=Z&pmoJWd;#WSbNT<%o!E`KlrI4kjZ7a#brrl; zGrj|DQJ_PuG}9fDeS)rhmh=_>lx-@nv*_L-Dyk%d7$t!Wrw81O)@tmHCg3a5!Vea_ z0J=(7T9(Fj<3o-=EPdSxWVkbXqFKJ>NH?nbC+D+@`E*+0hV0k7NgZv+B+>uV9E3tb z#s@yroiQ@3GE*o-Jd3(a4s)*tU0Ax11i9nzg`|s8%GLaXh2<|>@l~tKMna-X>Uyb* zPg`n-{^-JMClz${tTM~l>6LB^(J^QTiizSW4S$ZV^nv0ez~RBCdoZi}GcX@v{RTP0 zM>Ap~sH10L;k#_|m$pV;xpRsx7Zn>iar^Pl#IwcyuOS4-FZS<};%=1;B)wN@>Y`Ii zf8Vh)kbQa@Xg+|9#r&G(n(yt3U(n%C)ewZ$*Uxd_z1r&uaTme^vy)xIO1>IxlAZm8 zG;$|Mr4`@&%Gd*x=W|lLFRpBGnc=!%-KDGWgBAZ4vy7nN#Z%SomUwiz`kL*4FItY$ z9{;3%OsL3Xx~hUDio!h3US0@9?HoT#I%?TBsN10_G(0`u>3Ua!%JzY@E_j_2?^&I5 z%_JXXc<;rRm}z1xAHIqC(YgD5LF&$R)Vr0X!|R7ta@YiOLshuZmB#znzh2V=cW1+RIe76J(A`xMUPsT5nU$Zxpq2V$HjZyDR zHcXGcq&P<)eeU?P)n`fU$_XbQ$G5#jHicq$W;ULAyB)k}jRNd(D`}~{{g6u^7GDqd zP1Bbw+NtC}pqB#7j`0?Z6?3#UJ`~L0VrF)l3J^W)V>Q~| zL>I_3ee&Qtd|QIBY!IzbLd!hb25!0u07ZIT@MkLG!Ies))rHB(WiD6z2*q*AO8!e* zCH1SpJz9@H@BU#gSN6?(G_PC$$jiX7<;p!KEE(8onQ(|5cK~7qb`&;do*A5m(1Mcu zq@^ZP_!~ZC#qqoKDp&%wcYnOv{~|Hou{6lr29w)$T|S#caH5M|ZgPzyjYG~VW8I28 zcDavp?l%H`nB5N?c(^tr5OBY>G!Z>UyK`^I$O7L;sZjze1Xw5g=fBPvUmP16w+}{K zq-o0V=p(;DL)uW=0bjGB6w}qHWWHrU3Z;{UIgD5s5yg^@Yc?`C+Tmw>2-2?+9C zRa3MjaR7X2&#^t?s46R$7mWJNS8k!oQKT^XWe7}qJvPfwoqk`Jhy@`YJIvJJQEfVz zaeF4gvH>Rd3B5Qp6)N@(=ctLsZ_`MMt)CtB3+{_Ns`Ulr9^Vh=y`n~Bs zscnH)VmX~+#2U5s@NPRRthT0(5-SWI~4{UZM^;a~d+NnnR~wz*g6ULl0~ zR7v8(s<-7lnz6f7KXBymEo`D$T*;4Y>jBN#rS?CDFMwQX|4^|{uBmkdxgGzz>Ekh$ zqp)}sTyNWzsFsE8{KD?1(BXbdttypoS9r8DXBajW5U1nRP*Va|RvKX&k$HuF%9s6} z=&c3P99K@Xx%Xv^E1$G1u6rQal+j~qS4wIswZVyRNY?QpkHP$hBVp=W0X!%>ef>?< z%-lHo?E(AWD&?#u4m$rAq4<{*W1nH3bGC1{>dFwC1Oq>dY!kXK%0z)sTB2>hr~Vt- z1g?C{?Ru*5hHCBNBghiNSrSe)&8AdpQ&ZLc_fID6jfa?1zOEP2YTFnACred|>3cfA zjY<`Lohs|D4{?68nF6i^BAq&8*S}8b3p^SqgQd(!CdRuQRO6j|d+`59x#9m0nRb8q zlI4OICjWyRtHsa~ldkYge&D^1MLj`Ifgm9>mMF~L>nJ~hzEZbAfb4@lt&-0?u2N8j zPps)+yAa?Tji$F9+>qstR~h=3!f}M%m2{5Re|7X2))f=7Th49iVB3mAwRL6&K$paK ztl=JBKYkp3W_`^-g%%P0SCYL_4e!EyWO=xB9g<0%A^F@7aKOJDWGOTnb1q$(O3$hF^jF&{oWY*hR5_3K?T zGqdqBYx1n@?4gknOP&iUh&tk5#kznKxw*O7+1uMzZ8-WDd52oY!(6BAQ;@uD4fc2XK}=4(G) zdq2bfAlX3U%0ymQQUYFB`7kIb^r?X0Kk1%`;SG`V!npk6(h{pI8)-pBw5zBv2hCF>oCx1SHRg-j#uHSNPFwJsMdd9d~Zcbk&=*-?nY!tX%P?s=@JG6grPw|Lg}2LL!`TV z=#HU7nxSC;3F!`h3*GyB?)TjLJNMju{^j*zX02z7$7UbNlBkmhQx;@&i%Mhp)4tD5Sw zGxTbH#ECdCRRVDy1oXrQrTuU}dWxP=%|lcr(~1EqCuvnj*n!)<7DHry`fEnLTT*q* zj}LKQ?!crFb&C!zcrn~#VX1W%CgvjVnj6Ww= zQ1DgG36GUW)2pqs+wDp8_xALE0g-zNVwauqyL$qa-S{bT*kgUeBXz zi=&V=iZhg-+^GSF!TY}Q0oJ;F_<6{kdi|g&v~jq8mEpJ@ZZm9T{Pvgh;dggI7YoiP zo5rHPh3vy9$PZ6bD>!lIj%mz?3`5}CG2D7zeO7L$(`)l0Mne5B?!SrFTvVN5kd~FD z6%nCOQBj#HcfAY!@#Sum{FcJ=H{}d%S+7{$YNV-SWaKBm1G$#v!H5)p>H(C`jIn(^#xN^;x-9hjYz$!q}$n%coa%o7?Tv7he#{w^jFV;0eHk zudqS%|1}QIHiX!x?;l z#Il%(b}cj!Tad= z*Nb5wkFP?Pq!J!fWrkb(x24-(eHBkn*<&Tp;9R@3U>K(Le{jMHM9+B1IHhgW@Z8={ zq}AJl6ip9@)vtu=g5Rb8w8wZH8JjBR85CvTA7{$jDd*#q!{2>dS;$!$9-LIl3aM{B zvNxZ9w^INWoLNYtfpH`E%WHf16!urr>9IuK zCo@kcd74N18|GO{MS_LO0R@CeC4=U)7jYw;|6+h{lTC;Yw%O3z2NHaR5@TO~)A-PA z(~?U3mX|a8o0ltC>3_CbWwI@ysYH8UsdmFsENH7`X6VYSBl*#|P*F%3b*>0}%I zDZ$WLx#5Uo@=CK=zxnJx_&eC2dt@e4)Vo!7jy+0GZo56{qJ#c|IR7WmF4g4T2y$sY zQ112v?f&(Fc{dyRp%Pw}B6S5HGV1NI6;TCmS@nH6+w~lYoBRhG;^HZkWj;=T&4=Q> z;Ya8R6x7c>&fLcWHU~h?$2DIl6h`Ir9QYG~2PDGtw~*%_v%-C|E-l|4|87qIrPEPT zx5#~{4aNGIW3<=!P-g3gzfog{K2|XO#9UAByif2a3&p~rn(y7eL`lWRzbm~Z`_MF+ z6Fz2HuYmg(a=zil1izV6i(gO3mKkBNyDq%e#JF=zx9UK>y7VEjV*g@^w!x{8sDzs3 zUG)Rv;JqJtFGaqSf0o}}x0qnMRYz<3{4O|v6aVyVZPBWqAv!iTy{rshzUO;jh_ZyD z6>qvR0sf*fe@U8OmG;IH3bdOS#eR-D`cz zENk-Sj)3*7@A1|ocxR?ctLpuu^76w@2=I+hFxvG0(h*AAQxH7%2QF=+t-4^#-ROqF z-O`Stsd20ek;7W{ES>6-_I>F^MTMaAHtFp_cy)D04+kct<-#u6Zfr}#kG0MvP0Pkc z&8AWKy2tLp*up0Yirm@Lai7y7^roiA$U8`mMwCSW%&i`Kk#hd->R2iAO0esj9Qt}I zO}Sgs0C#Bq1JN-UxhpbbyTma?K%iaM^0FpF?0yTLLB0y~Nnioy@6LlR63p3h_wL4;BeH?>%7n#CQn9~KbbD~tr47}1zs1w7#p zC3jdKbdrQBEFqaMNh;K3UA46{@dapzRO!pOtI5BPg0;=2m1`hEzE_4*VxZUq>n6Xa1JNwj|a%d zedRagZHZ2s4@S}F1Xcw%1MCy8J^HHqB1en0;3qP6hTTIIwI1~djnpC$E;iW;juH35 z+zX2?_nRya3vo_N@0!TJ?w%FUBp&%VYXR|k^2%LoHV`Q}57vuNnhjR{u)$>~$>2%q zKEzju{GK+o&$@V1AeklY{x^h?%VH1AcKghgu`5l`l@$%4Y|xTbX$5Gz&Ei1_`=F=} zDIKPw$tc?Ugh~~G@yIIqAlGSDQe)4&27OG|kDDGnW%8LiU;KBl7pfHj)>YDvM3gwwmQ+C?kPm0n0Qc-I`!@AzF0W#mFLXdrf(bE~rmA}+% z=6wc1D0E()1+GVjA94cmSEk9q+zt>y&!00i?xhFJgexI}t(j+}>t~=$gWfjFl16{| zT~zOk^IoP+1Wp!nZf}9?skEtlKH$`MT0J%4cY_7zN+if8Iek=clXE7iuo)yc=De>2l6AWwApchT+8YwOW7k4=xosRLnAGTpNjZSSY z2#kQ;4C>)LsLbGjaFh=C6wi@SEq3PzEUxg~g$=ooN6mkb zg`T@{-v4HE?pEc3^X$>vcvH@$;1S)tuXRwrmUq)ahs~`6mmlCgl;kvMd4Y5RAq6TL`l>DFCT6lH{VANt0l=flrYhDH zk%y;lwkf1Dp*s|(zV1BLv^7Tss>e;f4eJcVl-kKB90*)>;ZJc8vD|s@IGzohCtc4! z5?tugjVus^Hf2j~{(FlP&vY^rREK z$y9odqE-+2`Xyh9I6+xOC;NNc6R!FG>_*7cvsIRp?~3I1V~`EVXuZuij4tei=+w_0 z$$d@jtleZl>n-XKTsg3}3Lsnve`b07-ti5S@o;oV(^}L9 z1m~@4!(GvF%eSnCXlgx)fM)UY;8os6w+9-fcC*CCF8vJ%)^n-;JM8+$#Knm@#9-sL zMdO_K%J!?CraruFAM_eY9ZA;YHeA%ks`!0Ho3 zt>l>xx#8@PsSU^8Jc(MloBJ>YmPOqtZ?7o$*^2cuM%ZIRmS0DIU{D(!>g+oMGMs=c zFu3v$etJ*uY+qCJV8g)d8dvjR5A-k|hhfsE^Uho^V#W)zl95(dB@dx`xNjP*Lg9dth+ z$t*dLJ;|>RZ`4_Sbx)h1z_ZhHATd`?ZK-a$Lw({G!26&%waOQI;2>m~BE#&R&q-hEr`%5HlffGw!7(+3r|d}*GAC0bYS#r=_K9!vgbP8@Qk7pq08 zS_$!mJucDpyw$q(e9)-*0SvZA+jw$eE{-4DW?kf|9 zShsOm?!kVD-u>0x$(F5H92bw0`*-h3#a00w@!WExJ$uKoXM>wgt#z2YYe#(6n~^nj z365N1ZBJnh4cUzf7az%FP`@V|Hxaqm&7?R~S=lGe zW5QVef?!98cPar7V>8XZ5Dy%r%gggUprs`>3lI2!#G0=Z!gRsup(w)}oWvKpzBUz* zm!DjF?-<&!+2}GwNoglOgy7mfi8wGl+|gV}woY_=vYWVOmy2ATTPOePg$N84=w_O~@*LQWKqr zrfB#W0?D=#P)Jr~KHi$9S5=SQSuo%p-?3Y@FP!w(5XC$fZ~`?#pff8#`_Gt@F9c|t z2k(mS@-D3rh+ma`pJc*l^RROT;bJ}jl#%3)OUQAu?XBm{c?R6y1u_~*7j)>?D3!lB zih?6kf4<{d6n<>pV^{jHp`oGBTsZ9&K_i98OB8u8eUFPE(t z%ZG@1nOB_Y5RZ{gD1}?`RZ_UHwsND=(SX1*%P>VuOf_PO_Purv(qC4TGeVWDQSAGP zW&!nMtqZ!rl2MP9zH$-@HhM4nx zokN2^nuL2mAYua6IGMZdjVNysT*>@|rA%D{V`BI=8!=-(7g6tcSYJ<>Tkz+{G_Q); z5x5-&sn#%>65w(&JbG4VzTdn*l0xT+9Auc7ZsKvFr{~sZw+NC+Uf@oYFGp z5uBWZ<0Vf>DQsMlL#6w>dN`PKd%mgg!zRF2itrUPA#nNvb0vf0wGza6K8HkGB}&$e zMKTytStk}Lp(?Rr3J;q=Rt`1z$~!`E@VKRvI_y-6hgSI8P5vd#3cv8=#PQlm|FvDY z4aIy;GdrRHClLph+H}uP4=@%w*Z5JK9-rzi*|Qr?M6b|##fK)4ZVt*RgSi2v0`O`` z8L!rejgWLjGnn_eQf8+&r-@#|m_a=ErsWo7l&&P+BqauH=04ikd243&;@Hk+6JN)& zhF^i;J^r`11{$I}k_-juqQ`GAs49FFM(N$XT}6@QaY;#u)qrqwPjC|(xtTGZT?@U% z(}phTcPmiv5(rj+aZ35h{>gF7c*g6lXPhhu3L5uvujW2EXB3i1H7k}K2wH>SOcUKh zksm}Ot=$qaQ>+tloSL8JsCUB)wjB78uXp>{i;U~Tha*lWFZ`R9E|?d7yjc*8y~#s5 zXE!zc{6aSXJUwh{I)v93&tY77=y~GND#dg;Np{zpBQmLB^}5GNom<0Snu(*B3}Ksx zLusO3JZ`K8e>OcMt8t3NfxGw7^BY{myOseNHFoEt6lpOq=0ursp-47g6^1zH>hpA% z4FC8je`ankvXgO4;Dg4oj3VYpJg-i*2uiaf#>1ENFA%qLx zzh*TM%wrC`{_YuIq%L&MQzKKx=zTdhgiK1X6dSzCW`)M{$>(2j6 z37INppbXXC&Std5e4{KC&ao9-QI||qp$IeAktr>)YHVn#gIXm}BTCi8T~_g6@_-2`C9MvLm03sPo@VI#(NsxWo`D_w+dltsh1bee#dLmkXnih00NKM(Lj6b4o(Ho%CZ;G8Ua3f{yV{I@JG!0DDZ00vISg zT1^dTRq`Rz}RqHd~Ucsd7 zx%7InIb|GEy3_XUe=CACOuv82(yF$mbocP+L^Kx}Hd6sfC5ov5> zdPKZ5*?dzK4!)@`O~n(x{;piMHL5S@epZV2J<+-@2vF(RwF1E+?j1EvX)*G4_j-GK z0m^|TQ;<`PY=kE$Fcc~$QlFHZ>?aj~iyZ&G!pGW&BwVwO_4k~GWit9>mdBA=)bkSt z&5V7X*IIa&kZj*v11F|kZPH)O^rz0@RM<$Ha1F9JX%+^wB`iHc z`>ReaPyCWWw+FXwH;tkT=@+}z&DW4wub7N8 zo$3OcU(d_3g?@vEkpYXvWNRWciXJLk>5>!>pIRW}M4Ji8gc!)%jUnUQL=7k0c+$G2 zn+(gqCYOsa(E_{*)V|xSlE_OI8Vt?FFX1a*m(^bFbTBdR_2i$z?}n27sBiWH&U%zR zy~+6A0Y1w(Lr<~~vv%lB=L0s&`v_`8B}q=1eMJKM!V7@fta> zEiIi|T8bxg-zR)!xRm+p!=UGTwILXoVUkvQ1Y(!leDwlWBrl7Y96z1~?n^^sfy(yF=YkGJ8HyiT_8UCnLMX%F|Rp9(x~zkVg3`uYILRJ=>n zS1N$h^zoqFcs8zkKD#c)WO6onx&V`-_IRV1V*cK$&4&7_D0~I|=dJ|50RQs3_|IDA zb&rx~p5my}vs161A7UsDOT1#eM*OxFL_0`4LQx80F38vmuyu&!{`N+X9eD=4mU`b6 zB1o(fsAeW*dvkTq*WaI3RP^o$59S{5RY}6)ohLwg-i`-GJ!udGS&ghR{q<}LY$fw+ zm>VzP+lKS8LYrPb==;3inh_is@eRdpNd_LE$gleXD7E`U-?B$=xwuF?>JL<@O_+PN z7P&|ZFJmxgUqA!mVp5)!QBP#QG4#~aS%39jN!FjHUIy%{3KhK&53Wq<})ToaZk>eIcB7U2hpj>&^ z03MPkCgrv`>nj55o0EaF=^;;lt}BGs;$`)$Z-AEv+v8>Ty{X;iu-%0V8O1OBm_J+L z`_J~~?E8u&^LJ2Fj79eickQl%O-zwQSTf>n_OnypVBKE@D4>u}Ugj+Dd$Nly5u<{3 z2mHpyD|iG1pViDbOHmFOLaqc7X0H~FO^E8DR+Tn=ZfRp~lA5jeBqK2?#|~&a-^ru0WQcNY~uZbGi$I=qx8Ql9fw@I%w-S{1sWsvp?o+WJlNGtl?V z2PAT@#;FN^y$CHk|37p!P2*Q`>lz8gvNIU)lj(Om(KP4K`ULf68c9BsYdDzv8PF5e zPuCXJpf$Z^gI7)2vww!g0cZu5t&Ub}%F9S3_QR&ARj_}SzC5EE=;|G6Tcmcxznyq8 zQG~QR>&EvHm~OWnHQ^h~ z|D8r(u|K!aE5>O@1;5~DERs<_av#fS@_)tH6_(d5H@~z%gKA044HW&<7PZ@AYs91T z`2Fz0q?v_EFdb;3hO$3LF5VBRYu#u(S7-em_2Emb zPH;LC0yFF0S$mrJ;^}9#z2Vjc@OT+N`&3fssleARL{ojodK{Nd15p4psU=auft$|u zKCXeFcMO|2+_KO4_6JohXVak~Nkd5dy)?>V+^sejq;BY^^EOjVXHwW)Zl;?4r8_7v zuA04|Z>u*_=1x)sxX3dXIj*l!mG<$6$y8BX^^(=c!Pp9`D1`zIN?hus>SR83Z8N;% z+72OL#S)LG|LpX-FZWHI1zX{u>+TnmbF0=K%{f zUC!x7d|b7uS-R+{u|A+jgVco0CHsE=>4N#;YTGQZ$%QDTe#wpJ0r+6GXIV+)fD23I zWr3THpN@G#{7WQ}LtqT#$rOUNe)A$9PTy{ds`C57r8Bc^+8xM3MZp2G-Xd+AZa|wQ z_~A$G-Gp?l3+9w>!qm`@IrW^Wtw>%;oMEQPq~ zs^2AmWh%pGMXF5Y2e13k4B?m6{5C#8k?v!!@>kw?JaWDRwontT%NvJ=(@&6CkQ=T! zC>{ge9j<9NPR&~o)(k3F@OA9~V=zfyf4cnm#sS`F)T<5aDOdV>7u|_|H8S5^4tpZG zK`d@(X_FQDx&5+rbDuU%8!CPY(V3z8&Kg{u1>`_qbE7pZ4^J0_;{l; zYaepnCz9;8$wCw`gy@RlTKtai@{64MU^nL!H8MFFaJt|{jmg8b-^t}p6##DtEPc&5 zp|2lHoTVi_9!#Gn;%0C8suX65rpfcoQo?GupacM|3nu}cps{a) z-jU2Z_AS0kg2VV=!b?KoPRDRGzN>a$b3yIY3#Z!L7fyvXi{;Uy1zd6pZ)+TTP+HZL zeQr)bST=#X;e-IGUhZln;|ZHE^&l1Of|P<|D@nNM@JUhi59N$dV873}4t`doeL~~4 zl&Kw{P?5#H3sYB}7BKr^Ij|$-R5cj15mmQ?KSoXYdn@But;x^t#fk%;m&a06uU@8A z2t1y9fN9P)dl$S$3^?IsG}{?)PR5Ki?!0O=5Xlf7Kk!=ES>G5bF><-Q1)ea z=Rzo1*ZEJckEf(-#Tae-YD zTT`)d5RvS^mmnL?l1wZLx>?ZUnl*sLE+c#U`xcPrPp_`$qFhiu?Prd=WO^839e&}q zXesL{4*k}KHG!w|ABT!Xk$$RMm`P6pN7AD4ylrLl4E_gViLxU2e;0FvL1CM4GZ3~r?0km+rrSg{dcM%;`ZlHTnkSq|9 zsCldd)AzRJ0Iq)AZjJmvhxZ@3?NbHZc`h#9)Qv?VbNyl^(?ZQER|T8X@7nA>=%~Ka zu(v~Ka5f>vMKt*=Jui#8Yr^xSo_zc%2Qx>BoV+$=u!5x#a~}SlsnbUt62puT*E)pMX#>e9s5wyRu z(VgIZ(*JQJ)){?9)y9lqC&0h^p^BFs3pc;H1%&_BXUxc}V=pofunb9Z3Rp+O`Ma3r zlxz~_q8b=&ReOq{SI2UirZ^Y6DSGsCWtMpN>e(Ng?dut0j)26&Z@JCR1~kE{JT)w2=DVKiltSohxfEeP`^AX@0zwSO zl3;)jSV;-vTH{Ocy3Sm=yBdAgra3(vvPTn;OL zDp&&O?fCMAy`mz!&NrDFbe&Tsi5lYnZqwN%c&jmFu`QrrkFI#+tZP{YYy4niUl&G3lGdUrPe#UriAL5F;$ zjpxnMT)>vxlAMp|=3a;Fm__oKZD}{uY?9vG?5ngoq)>Do9W#|}Kc}bV?H!&N2bLnO z#{6pxSd7++$na*wH&T@oaMFi9@NT%G-P)EV=!Pu1)P#hk4KT;fL<1aHI(-rmK%l~9Um_`}q+ zGD=g_Q_0W}g2&HCCST9)%jeUBpSwLSi&*|koshN-qSbE5aL^#rRl zpH6AGmBH5nU)f?x$k}2&dZviJnP?Vsnnm(T_WkE^a&b#lN@lh+ z?5Yfit3J_&MN^p@8S0h2PHF(IAB_um&D`Dv0{sX`tGRPuiY#gciv}?bI^VwMz`9Hr ziBFP3Z|xwz*0*^9tW-d>EI3MMW2(+U$NQPFEM4WAP-1)`8-qjos@8nAjHBs+B{4Xw zpYeWgpWq0W9n=&Oiack)@GiZ5fplE4#^oJeCmekS(CwP4fgP>Vb=u@hL5!Yz>dXbr zp$wOFwxbJ^Gd(=(2riq}9tWex_Eu4vJ|)i@8a#{)GHp6DUKhUsqBf82*LSVG0-y$7 z`wquoTB$rzmRCdFU0Icie=cS#)u9}4i-8Yz1v}) zUuh7Ue^9sSba6a>xmw^dRs+OoW8T{TfN77~OaC+V znN=ItI~#;%$_3ibgKpK?t2Vc*h1ofmjqAo5$c3!?&4}sQ%-A?M94v>@>NvtY90O#@ zh6u>ng;tK83-a-&l4^)sfO4tnQ$PDKX=Uhnjw)#|8z5$mGZxeLR!({$uS8<}H8z>S z+96Wy8=4-JF2i_d<{NO+ZJoVKf!R8nZ)m&e2Mo?uP2ZppZx$r#RfX*nP+J0gWtEjf zIdX#YalT5>&S)%35WEV33}1c2nfu=Au+Hh;yqj1*MCX!CoYQ|t=2O%$z`HWq3WGcs z|Ezc-yxbkz2!O`v1vJt$2tIe^w~?p^E>UP9wp$@h%cJa)Y9aVh`cNmimAz3-_;R*c z05xbLMI$ZJs2(Da_6TA=!54QJiFMyW{=~?Vqq--MWzrUC^YUu3r$K&_p|yWFfS|!p zLhrBP@wBpm=ZU*D^oI-XLc{_@;Nu1@BxdkLX46ADhUuNQ(F z_Y(TLC$_5SqOLohYfK0|l@fa(34L#PM_W54x>p5oh#-ZuE;v^;PTg~nt9|7vMu28S z?)Kh@_qzXu{R34rg3m))2Nmzoo)Wtlaqkj8Mjcm8nfl@az0-`cxpb4+I%%zd4gts4 zEhr4m?bMLF&8Sq*bv*(ELidr)5c&j3Kp(+`h*4Lnv0b7aG{k7HJxnv6L5o`nSyGYw zbdCXFFEyk*(7;d$b1mQSjna?^XNZM9_i=<0&G2k-Bs&fGF>jF;7l{hf7TfEPq%pZ^ zw>jrbkBV{)mUk!@*8zO{<0jGrc);Al=Aw}qQ`UpIS6bT?9f7N0vR(53|_VDh9vFC@Jt&3B4(%#fit_9*#lBaZcs&pv2ux zh8hmc2y6p<583A_s5+E`2PQOHiPc+0A`0TJephQE>JEPf4M?jsw^iq}b`2uq57nGri%zS7owq8s;^5C_t2v;I^ErX875EDoEhG zJGVi9$Jtke*DeOWkNWhy=sP*Z&P~sdU%WyQo@GTLQa1YOMFC?>lJb_bgNK9H5fs=} zJaOYU&~2wRqaCuF+_iTFB=CMf&3bR$ukje*f6s;ZzqnsNFZi!7R@?{-2I6sv80fW#SWz z4+uKcRQx|O+g`b+Ain$RNy{LBoWb+-oiIlOF|j)2Z9IgNVY#U*GB}TtsJ9AV6tt#V z=-9MwU7|W)@TBPWdp#pls+pbOYT|+YTIZr=j5=xok|dy1w!)OF_|}@OCcL z)RdsFukTX-^jiv$J)sLo3z7_z>$UrBq{crCR^vO|UpjfAs7nT&6rg`Q(ItK5;J~|PrPQe_%(wTDv7f7C z9y%^zV>FOMrYrZ^rbLq?qy#SKWx%y0}8s?C? zmMO926BoU3UqMSx1_7>TGlyBT=%^VekfARZx<#&&Ky-LY5uK5quKI%K=jMj_P0Yy1 zexOp^nvZpa+UP=`6!FqAu>J8cY{gZ6T4ZiQ4%!Y*G4I;4`@Sglc>RzmKJe1DvPRV~ z5EIz+|4J1L3=5Na=|)Ao9oTiGt|A$1xDXlq3~stSKnaS!)TVh?7~H-P%idM+(JQR_ zp*T)T3xa$7Ox%u_om3%?(}^9S`DRMCf=ljt+em>4uko?`Dcii zwtzGjFpSUM%4+8kUHg@q#ryZ~ffUToz8!DSsD$qPY`k2mHsb>5M0~Z}lO3durXc{2 z;L)J7$^+aPbEaKosI2*b6P2+le_;*uQPr8wV5d|fTj5p>dvDy1UAbX8B;ji)5=15L zOIm;y-2ADA)mx+()`#h`y|MK2w1Vx}uZ9*tE`Ftn0t6UZq^d)ytr@@jMA!255N~OV z_+5unw512l11{j9cl}ch1YBo>Ee)y7Mk76D+{dGa%cyJRrH%89v%U6v?SM~O?uDP; zOEv5_bd{}eh`a5C2uEu}VYspk`8R%yd4EWflcT1C*MZ<>Nvzf5#Vw8nriDSTKIEEd z=CP`WldcZrup3z(y^qS6UIUjVp*x6Y9*Y4V0MQn@c}soi1Vy4$syT`#gyMuuSu{o4 zd-hQ$G;b!I7A0TA*Wm8Ktl4=I_Cu1a%i9Ycul z=0p$gn;h9Utnq&|)(jb=0TBrc$;E-=tENPpBWWPX&RvZV21X%6| zk=k@!+PU6CO~w2$)AZ`cUx6b3Q~CRTA+U7IUnHy=DM}BhzRRPN z&x@aVLMTwiP9j@XDi6t@nVd@ziSRa-CWu2~_%FD#%&bS%UyY2y;hX1&yHB%^gW8DS!e0 zyg_=?g}k1sJyf$=Xo{Mf4*)9TDY!?yzAn%g>pS|rXk&j=(!@coLgcjlQ;AKMG|OuN zt9I3CeCued;Iv1^zuJPlN*xm_E8J^A@EElt`y92*G`Dt~>Dgn$$A4)({V>k8PhcnF zoxv-!L?7pYbptT{9IIIDwQlDSdu%#~f7PJvy1r;IXS=g_?rD||)u>(fM36K81#|n& zJ|kpQ$WuZ5=`P(lkCwH~JCE`)%~`|pm5+gqfvObskLJ6iXNLyIG*kFAH?i#){-dQ> z+MceciB9$@`pgqw%|m+lEUEpH{MAy5a8CPan-v;kN&Anftf@y8q`I_gMjlhbT9gNj z2I5C%SFYE}dy^l2KD_I5H&(;dgn*$#J(82SvpeG+jmG;thtfjps7}W0e7CyGD1-`f z({s<(HF_d%Q^!QJUi=E)z|{*b^^1#w=be_!P+Q8elx-fcOS)%y^jW4LbPf{50 ze62_GU(t>oW?$qM#}cru&MCZK0kfJD2x~~m`~cg{Om&uB>zJQ-D~Li{RlN2~%2lCc zOEUXTiVNyLdd;gIT>^$CXixE2)BF)e;_C_lj&<5V(b^C=YUlfoTA`VDYWRI^&^-aW z3i*H)=OPb{Bbjj6BM) zpy>G$2zGT~N4OENc1n1S@9Z^3ad&r6HJ9rg~4GBetHSQmJWjtkzJ*b#JJMH|$|Kn#{9 zX;C%cAlj6T2fkGO>I|LmY1hqt<&No@sW~FULwqX-bAc)fyCoqEiWrg%|srV z9-axLH)19(=o7sUR@%f)eSR~J(oz)ILLu6&PQQtr@M(ca{!mEQbzc2LZL;b80${Eg z!kvN_=&hg=D@xVLKAv0i8XSl4M;_{N<1O~Dx=^0ZbxpX%VynC8=r$Ol_!l?s#X zLIlHL9v^~+I1MXZSa2zYKR@$Xdn>z$L(yQ1#-{0MH5of`PmTI8H?pZaFL}EoD&60m zd7@8dtmgu2Ou1B*+H}ha8F4bu8_Z*phM7AW`%8h$6WNG`yBGm@|DuU(b*gW zFO=yvJ{?St`Kz_2+zbkJH1lmP0B4RD_lL6C$j}eF%BV*;*Vf#xb%oJ5 zX+yepJgFo+MD_BxIywM^*{c7*Cy&!F$nNR=nJA#G2>LymOj3F76=Nq`0#WlrwS9KS zM@^pw8<>V84!JKvDfj6IP2^{~;tV}>FRDw*3&|=1S$@h`TN{FGRBcl7kKLAS zza*(yUaon3*%ExOxgx?n?G=OF3|OD*zd7fHPPj3+cOc!wvDnvTL$PLJi(rpDn;(8m z5fs85cLPDG;_6_)#7M4%H~=|-PYCFG`I@u#>ckIUPWN08Xf|>km^LCiT$cZNYwO`g zyqWmk?)<?vi`Mssp?|5HD9(gZ1-~&pZ4)1%|J2RtoZ*fELa2O@NHNPnG&fKTZ=49*Wp;bvQ zR(12W)&3199ClbR(Yfse9&g&8?DXJ6*fNK|**GompIZDVaUqN}Z9EBN;>szuoL)=S zH*Mxv*B;?=0--z$Jw}8LSvzXOdu_)&N?#PzK5#bds}}a45L(v^s}tSw$o%OtD&|?_ zK+mx?H=z|vd1(tIg!HmN?G#V>z-a4}{o_AEZXp<+KtGh{u2`-2P~GzAMW?XV>1aY8#7AWgGv%rAEKwn_h%`|>7z8i^KG(T^j{ zg&fCUmst@}SDea22}xlnp;6w#RijKt8rjWWUTj%O~$ z1l`P3f3BYA!`_F8-=(@BY!Ppo`oOoyhpkoL+R%U4EB-m5Wcsx`ItI{h2mJ*kbUp># z9T!x4*C}@lo5^bWAI`#xE;)y-(3-BBTTb*_qK5ALyad1fDX~bt@F%C~Fb1FBa`0dJ z;9$%5t1ibgp1NR55w={0aBggT;DrGy$tWJzcq8tH|88;>Lf04$EI6 z!D3F5Uo^_>3bdh@=vH^!&zNlwAuW-xPtk7(03J*IZ;3F#gZw4C!?(YB_3WZ<(+;VO zzK;w*S26fg(Ur($6U2v5z7zMYUyc3j2&<_$TSF?`$72zNGHpd~jU8V}d3)vpOi6#M zO`(1?v$&N7=3e2mLkH;OkFr41P3cpY+xPx|!HcR6i1MS>QW;T{U3zbr9Ofk;2guye zkB_>%@0t(Hht%yo8^-&|J9Chema&L=<@pl7>Cy*x+o0~w&%-1MX-YA~s@6LMz-9IM z#dBN+`|aatJ2C1&O}G5^lruNfm{$@bdhHY}x!!MvGIQoh8>x^TV80BHGYp*-@k#A| z4V;hvVln?k02!J+y%1s@o-}~M2b6MatudTFOr))?pXYwu#hAe+;bHM40A_JYkEC7h zAFRmj?_IN;joKXIl``C$+lk}g&}n1%h9o&@+FY|2p}Ed^&B!^}%R`MX5X|+{TvDEe zWQBmS>%N46XT-6yj!vw;{_Wd|hqiS7ca!@)~ah1vF0?vf)#s?*(!FjRYw=mXIn>-lbR1Sh1=K+LT&hD>v+Vv0( z&}_HYNB^AMEG0Q_c=*}%@mBScUY`4$RzjeF6LVNs_Wr!)rV7#ne#T5t)vLz>R2pb@ zVav|;S1otnNjP`>+XRde?FQT~6iGbA!?d;ftlUexr<`24(sOf|xd|eHU#m${*XsVP z3{X#Hv7~#PiEPkWHlhoRvG-I-dY7_oC^<`@c?IS&8 zihhg{DROWESTwRcB4} zHFIVeEio68c$3_K4rZ|9-y~0fi$i+U8!QP;Y@cXm^kB9-EGgwZ3_&L$JPI+!i^to2p!1U`n#-! z_5;EN5x|IzN6_yVubi@&(V^!HHYUWQB;=BXoeOyC&fVSw$#Gx`Yh{6T1w-@ z;J3oB0VDm!nkt&p& zt>0`|+)xk|3}L_Y;ADzJqOFtSoQsB)F z(0lkQVSI64fI}MeIb8-JYOztL>MWjakleC?N7X%9?&3IGKst`4DEfgzXK7=pZFv(V z(2{xA2mnF_fp`CC9LdF0mJl7{U7niaz0!^ge^A-0lP+GsD+3IFWG9BGF-61 zN#p|_y}o)gIitr;zh4`FGbn}GKaaRkE5x@~xT8=jG!nJao6o3MV~fAe<-0U{{+b8k z-M8Pl$4t}W?kh;ouNO)xC*AdgpI!2W<5cI1dUz#g`~}Xlbu9)ONezAzoiYQ?^j8y_ zsl?QIu=3-&ofntyO#R;On`%JgG%<2~-3*EOXFgbf-UZ^n`+m=Rt+vc2U~(y=0W-YuaSeg=3}>be-sbWFh%d=E08bvHun6`VwvgL`SUW{6`@1G4%^(bwCX4`ZPs?>Ctjv=l@tPgmG9ZK=GneAuT!79C}y6@bz%wqd+GkN(&+5janeRX zH)~U3)9V-$?c9oMe?Ja=KlxyP_udMQ2l@&kPv@-BFTDC&DgN4SU*HS?*|Te37oOXz z`2JQ`P8Pdw^@5nEz@?}gA1^QXb0l*P?>>c=A75q`C(S(EI%Bm~i6bn8( zoo30Gx?Nw%&hDmFluqpt>3B8EeZM*;79N#Je7?={bj23nvZ<|&?x!~X%%0zVX7T=> zg^hK~1;NMG?}n<@#!APD-ly!V>|lf0~Tcj@+*z-0AIab62cEG8Pkwxp747TYER*Z05eKmTm^OKa6n-dM{31fH&bF6*2UngAEP B(2W29 literal 0 HcmV?d00001 diff --git a/Documentation/DataLoadEngine/RemoteAttachers.md b/Documentation/DataLoadEngine/RemoteAttachers.md new file mode 100644 index 0000000000..a3ba7a784f --- /dev/null +++ b/Documentation/DataLoadEngine/RemoteAttachers.md @@ -0,0 +1,71 @@ +# Remote Attachers +The Data Load Engine within RDMP supports 2 methods for retrieving data from remote databases. + +The Remote Table Attacher facilitates pulling data from a specific table on a remote database. +The Remote Database Attacher also facilitates pulling data from a remote database, but allows for more freedom when specifying what data is returned. + +## How to setup a Remote Attacher +RDMP's remote attachers require some external database to pull from. +You can set this up via the "Create New External Database" function in the External Servers section of the Tables tab. +![External Database Setup Location](./Images/Remote_Attacher_External_Database.PNG) + +Once you have your external database configured, you can add a Remote Attacher (Table or Database) to your Data Load. + +Attachers can only be added to the Mounting stage of a data load. +![Attachers Location](./Images/Remote_Attacher_Attacher_Location.PNG) + +Each Attacher's configuration options are detailed below. + +## Configuring the Remote Table Attacher +The Remote Table Attacher has a number of configuration options, the required fields are: +* Remote Server Reference - Alternatively, you can manually add the remote server details +* Remote Table Name - The Table you wish to load from +* RAW Table Name - The RAW table you wish to load the data into + +The full configuration options are + +| Option Name | Description | +|----------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Remote Server Reference* | A Dropdown of defined external databases to select from | +| Remote Server | Alternative to the Remote Server Reference. A DataSource connection string for a remote server | +| Remote Database Name | The Database name on the remote server. Not required if specified by your external database | +| Remote Table Name* | The table name on the remote server | +| Remote Select SQL | Optionally provide custom SQL to query the remote table, otherwise a simple select will be used | +| RAW Table Name* | The Table in RAW that you wish to populate with the fetched data | +| RAW Table To Load | Overrides RAW Table Name with a specific named table | +| Progress | Used for Load Scheduling | +| Progress Update Strategy | Select the Progress update strategy | +| Timeout | The timeout to use for connecting to the remote database | +| Load Not Required If No Rows Read | If no data is read, then the load will stop if this option is checked | +| Remote Table Access Credentials | Optional Remote Table Access Credentials - create these in the Tables tab of RDMP | +| Database Type | Which type of database you will be connecting to | +| Historical Fetch Duration | A Filter to select how far back in history to pull from. 'Since Last Use' will pull data since the last successful run of this data load 'Delta Reading' allows you to move forward in time a specified number of days from a specific date. This specified date will move forwards each time the load is run by the number of days specified to look forward. | +| Remote Table Date Column | The Date column of the remote table to base the Historical Fetch Duration on | +| Custom Fetch Duration Start Date | If using the custom fetch duration, this is the start date | +| Custom Fetch Duration End Date | If using the custom fetch duration, this is the end date | +| Delta Reading Date In Time | If using the Delta Reading fetch, this is the earliest date you will pull data from | +| Delta Reading Look Back Days | If using the Delta Reading fetch duration, this is how many days backwards in time you wish to look each time. You may wish to use this option if data on the remote server is populated in batches, rather than real time. | +| Delta Reading Look Forward Days | If using the Delta Reading fetch duration, this is how many days forward in time you wish to look each time | +| Set Delta Reading To Last Seen Date Post Load | Optional overwrite to the Delta Reading fetch option. Will use the most recently seen date in the fetched data rather than the adding the forward look days amount onto the stored minimum date | +| Culture | Optionally specify a custom date format | +| Explicit Date Time Format | Optionally specify a specific datetime format + +## Configuring the Remote Database Attacher +The Remote Database Attacher has a number of configuration options the required fields are: +* Remote Source - A predefined external data base + +| Option Name | Description | +|----------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Remote Source* | A Dropdown of defined external databases to select from | +| Timeout | The timeout to use for connecting to the remote database | +| Ignore Missing Tables | Skip Tables that do not appear in the default RAW columns | +| Historical Fetch Duration | A Filter to select how far back in history to pull from. Since Last Use will pull data since the last successful run of this data load Delta Reading allows you to move forward in time a specified number of days from a specific date. This specified date will move forwards each time the load is run by the number of days specified to look forward. | +| Remote Table Date Column | The Date column of the remote table to base the Historical Fetch Duration on | +| Custom Fetch Duration Start Date | If using the custom fetch duration, this is the start date | +| Custom Fetch Duration End Date | If using the custom fetch duration, this is the end date | +| Delta Reading Date In Time | If using the Delta Reading fetch, this is the earliest date you will pull data from | +| Delta Reading Look Back Days | If using the Delta Reading fetch duration, this is how many days backwards in time you wish to look each time. You may wish to use this option if data on the remote server is populated in batches, rather than real time. | +| Delta Reading Look Forward Days | If using the Delta Reading fetch duration, this is how many days forward in time you wish to look each time | +| Set Delta Reading To Last Seen Date Post Load | Optional overwrite to the Delta Reading fetch option. Will use the most recently seen date in the fetched data rather than the adding the forward look days amount onto the stored minimum date | +| Culture | Optionally specify a custom date format | +| Explicit Date Time Format | Optionally specify a specific datetime format diff --git a/Rdmp.Core.Tests/DataLoad/Engine/Integration/RemoteAttacherTests.cs b/Rdmp.Core.Tests/DataLoad/Engine/Integration/RemoteAttacherTests.cs new file mode 100644 index 0000000000..98ca7a5147 --- /dev/null +++ b/Rdmp.Core.Tests/DataLoad/Engine/Integration/RemoteAttacherTests.cs @@ -0,0 +1,141 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +using FAnsi.Implementations.MicrosoftSQL; +using NUnit.Framework; +using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.DataLoad.Modules.Attachers; +using System; +using FAnsi; + +namespace Rdmp.Core.Tests.DataLoad.Engine.Integration; + +public class RemoteAttacherTests +{ + + [Test] + [TestCase(AttacherHistoricalDurations.Past24Hours, "DAY")] + [TestCase(AttacherHistoricalDurations.Past7Days, "WEEK")] + [TestCase(AttacherHistoricalDurations.PastMonth, "MONTH")] + [TestCase(AttacherHistoricalDurations.PastYear, "YEAR")] + public void TestRemoteAttacherParameter(AttacherHistoricalDurations duration, string convertTime) + { + var attacher = new RemoteAttacher(); + attacher.HistoricalFetchDuration = duration; + attacher.RemoteTableDateColumn = "date"; + var lmd = new LoadMetadata(); + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= DATEADD({convertTime}, -1, GETDATE())")); + } + [Test] + public void TestRemoteAttacherParameterSinceLastUse() + { + var attacher = new RemoteAttacher(); + attacher.HistoricalFetchDuration = AttacherHistoricalDurations.SinceLastUse; + attacher.RemoteTableDateColumn = "date"; + var lmd = new LoadMetadata(); + lmd.LastLoadTime = DateTime.Now; + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{lmd.LastLoadTime.GetValueOrDefault().ToString("yyyy-MM-dd HH:mm:ss.fff")}')")); + } + [Test] + public void TestRemoteAttacherParameterSinceLastUse_NULL() + { + var attacher = new RemoteAttacher(); + attacher.HistoricalFetchDuration = AttacherHistoricalDurations.SinceLastUse; + attacher.RemoteTableDateColumn = "date"; + var lmd = new LoadMetadata(); + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo("")); + } + [Test] + public void TestRemoteAttacherParameterCustomRange() + { + var attacher = new RemoteAttacher(); + attacher.HistoricalFetchDuration = AttacherHistoricalDurations.Custom; + attacher.RemoteTableDateColumn = "date"; + attacher.CustomFetchDurationStartDate = DateTime.Now; + attacher.CustomFetchDurationEndDate = DateTime.Now; + var lmd = new LoadMetadata(); + lmd.LastLoadTime = DateTime.Now; + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.CustomFetchDurationStartDate.ToString("yyyy-MM-dd HH:mm:ss.fff")}') AND CAST(date as Date) <= convert(Date,'{attacher.CustomFetchDurationEndDate.ToString("yyyy-MM-dd HH:mm:ss.fff")}')")); + } + [Test] + public void TestRemoteAttacherParameterCustomRangeNoStart() + { + var attacher = new RemoteAttacher(); + attacher.HistoricalFetchDuration = AttacherHistoricalDurations.Custom; + attacher.RemoteTableDateColumn = "date"; + attacher.CustomFetchDurationEndDate = DateTime.Now; + var lmd = new LoadMetadata(); + lmd.LastLoadTime = DateTime.Now; + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) <= convert(Date,'{attacher.CustomFetchDurationEndDate.ToString("yyyy-MM-dd HH:mm:ss.fff")}')")); + } + [Test] + public void TestRemoteAttacherParameterCustomRangeNoEnd() + { + var attacher = new RemoteAttacher(); + attacher.HistoricalFetchDuration = AttacherHistoricalDurations.Custom; + attacher.RemoteTableDateColumn = "date"; + attacher.CustomFetchDurationStartDate = DateTime.Now; + var lmd = new LoadMetadata(); + lmd.LastLoadTime = DateTime.Now; + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.CustomFetchDurationStartDate.ToString("yyyy-MM-dd HH:mm:ss.fff")}')")); + } + [Test] + public void TestRemoteAttacherParameterCustomRangeNoDates() + { + var attacher = new RemoteAttacher(); + attacher.HistoricalFetchDuration = AttacherHistoricalDurations.Custom; + attacher.RemoteTableDateColumn = "date"; + var lmd = new LoadMetadata(); + lmd.LastLoadTime = DateTime.Now; + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo("")); + } + [Test] + public void TestRemoteAttacherParameterDeltaReading() + { + var attacher = new RemoteAttacher(); + attacher.HistoricalFetchDuration = AttacherHistoricalDurations.DeltaReading; + attacher.DeltaReadingLookBackDays = 1; + attacher.DeltaReadingLookForwardDays = 1; + attacher.DeltaReadingStartDate = DateTime.Now; + attacher.RemoteTableDateColumn = "date"; + var lmd = new LoadMetadata(); + lmd.LastLoadTime = DateTime.Now; + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate.AddDays(-attacher.DeltaReadingLookBackDays).ToString("yyyy-MM-dd HH:mm:ss.fff")}') AND CAST(date as Date) <= convert(Date,'{attacher.DeltaReadingStartDate.AddDays(attacher.DeltaReadingLookForwardDays).ToString("yyyy-MM-dd HH:mm:ss.fff")}')")); + } + [Test] + public void TestRemoteAttacherParameterDeltaReading_NoLookBack() + { + var attacher = new RemoteAttacher(); + attacher.HistoricalFetchDuration = AttacherHistoricalDurations.DeltaReading; + attacher.DeltaReadingLookForwardDays = 1; + attacher.DeltaReadingStartDate = DateTime.Now; + attacher.RemoteTableDateColumn = "date"; + var lmd = new LoadMetadata(); + lmd.LastLoadTime = DateTime.Now; + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate.ToString("yyyy-MM-dd HH:mm:ss.fff")}') AND CAST(date as Date) <= convert(Date,'{attacher.DeltaReadingStartDate.AddDays(attacher.DeltaReadingLookForwardDays).ToString("yyyy-MM-dd HH:mm:ss.fff")}')")); + } + [Test] + public void TestRemoteAttacherParameterDeltaReading_NoLookForward() + { + var attacher = new RemoteAttacher(); + attacher.HistoricalFetchDuration = AttacherHistoricalDurations.DeltaReading; + attacher.DeltaReadingLookBackDays = 1; + attacher.DeltaReadingStartDate = DateTime.Now; + attacher.RemoteTableDateColumn = "date"; + var lmd = new LoadMetadata(); + lmd.LastLoadTime = DateTime.Now; + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo($" WHERE CAST(date as Date) >= convert(Date,'{attacher.DeltaReadingStartDate.AddDays(-attacher.DeltaReadingLookBackDays).ToString("yyyy-MM-dd HH:mm:ss.fff")}') AND CAST(date as Date) <= convert(Date,'{attacher.DeltaReadingStartDate.ToString("yyyy-MM-dd HH:mm:ss.fff")}')")); + } + [Test] + public void TestRemoteAttacherParameterDeltaReadingNoDates() + { + var attacher = new RemoteAttacher(); + attacher.HistoricalFetchDuration = AttacherHistoricalDurations.DeltaReading; + attacher.RemoteTableDateColumn = "date"; + var lmd = new LoadMetadata(); + Assert.That(attacher.SqlHistoricalDataFilter(lmd, DatabaseType.MicrosoftSQLServer), Is.EqualTo("")); + } +} diff --git a/Rdmp.Core.Tests/DataLoad/Engine/Integration/RemoteDatabaseAttacherTests.cs b/Rdmp.Core.Tests/DataLoad/Engine/Integration/RemoteDatabaseAttacherTests.cs index 917f29e1a3..3b48e2085f 100644 --- a/Rdmp.Core.Tests/DataLoad/Engine/Integration/RemoteDatabaseAttacherTests.cs +++ b/Rdmp.Core.Tests/DataLoad/Engine/Integration/RemoteDatabaseAttacherTests.cs @@ -8,6 +8,8 @@ using System.Collections.Generic; using System.Data; using FAnsi; +using MongoDB.Driver.Linq; +using NPOI.SS.Formula.Functions; using NSubstitute; using NUnit.Framework; using Rdmp.Core.Curation.Data; @@ -63,7 +65,7 @@ public void TestRemoteDatabaseAttach(DatabaseType dbType, Scenario scenario) lm.CreateNewLoggingTaskIfNotExists("amagad"); var dli = lm.CreateDataLoadInfo("amagad", "p", "a", "", true); - var job = Substitute.For(); + var job = NSubstitute.Substitute.For(); job.RegularTablesToLoad.Returns(new List { ti }); job.LookupTablesToLoad.Returns(new List()); job.DataLoadInfo.Returns(dli); @@ -102,6 +104,162 @@ public void TestRemoteDatabaseAttach(DatabaseType dbType, Scenario scenario) externalServer.DeleteInDatabase(); } + + + private static string Within(AttacherHistoricalDurations duration) + { + switch (duration) + { + case AttacherHistoricalDurations.Past24Hours: + return DateTime.Now.AddHours(-1).ToString(); + case AttacherHistoricalDurations.Past7Days: + return DateTime.Now.AddHours(-1).ToString(); + case AttacherHistoricalDurations.PastMonth: + return DateTime.Now.AddHours(-1).ToString(); + case AttacherHistoricalDurations.PastYear: + return DateTime.Now.AddHours(-1).ToString(); + case AttacherHistoricalDurations.SinceLastUse: + return DateTime.Now.AddHours(-1).ToString(); + case AttacherHistoricalDurations.Custom: + return DateTime.Now.AddDays(-1).ToString(); + case AttacherHistoricalDurations.DeltaReading: + return DateTime.Now.AddDays(-4).ToString(); + default: + return "fail"; + } + } + + private static string Outwith(AttacherHistoricalDurations duration) + { + switch (duration) + { + case AttacherHistoricalDurations.Past24Hours: + return DateTime.Now.AddDays(-2).ToString(); + case AttacherHistoricalDurations.Past7Days: + return DateTime.Now.AddDays(-8).ToString(); + case AttacherHistoricalDurations.PastMonth: + return DateTime.Now.AddMonths(-2).ToString(); + case AttacherHistoricalDurations.PastYear: + return DateTime.Now.AddYears(-2).ToString(); + case AttacherHistoricalDurations.SinceLastUse: + return DateTime.Now.AddDays(-2).ToString(); + case AttacherHistoricalDurations.Custom: + return DateTime.Now.AddDays(-14).ToString(); + case AttacherHistoricalDurations.DeltaReading: + return DateTime.Now.AddDays(-10).ToString(); + default: + return "fail"; + } + } + + private static Scenario[] Scenarios = { + Scenario.AllRawColumns, + Scenario.AllColumns, + Scenario.MissingPreLoadDiscardedColumn, + Scenario.MissingPreLoadDiscardedColumnButSelectStar + }; + private static AttacherHistoricalDurations[] Durations = { + AttacherHistoricalDurations.Past24Hours, + AttacherHistoricalDurations.Past7Days, + AttacherHistoricalDurations.PastMonth, + AttacherHistoricalDurations.PastYear, + AttacherHistoricalDurations.Custom, + AttacherHistoricalDurations.DeltaReading + }; + + public static IEnumerable GetAttacherCombinations() + { + foreach (var db in All.DatabaseTypes) + { + foreach (var sc in Scenarios) + { + foreach (var dur in Durations) + { + yield return new object[] { db, sc, dur }; + } + } + } + } + + [TestCaseSource(nameof(GetAttacherCombinations))] + public void TestRemoteDatabaseAttacherWithDateFilter(DatabaseType dbType, Scenario scenario, AttacherHistoricalDurations duration) + { + var db = GetCleanedServer(dbType); + + using var dt = new DataTable(); + dt.Columns.Add("animal"); + dt.Columns.Add("date_seen"); + var withinDate = Within(duration); + dt.Rows.Add("Cow", withinDate); + dt.Rows.Add("Crow", Outwith(duration)); + + var tbl = db.CreateTable("MyTable", dt); + + Assert.That(tbl.GetRowCount(), Is.EqualTo(2)); + Import(tbl, out var ti, out _); + //Create a virtual RAW column + if (scenario is Scenario.MissingPreLoadDiscardedColumn or Scenario.MissingPreLoadDiscardedColumnButSelectStar) + new PreLoadDiscardedColumn(CatalogueRepository, ti, "MyMissingCol"); + + var externalServer = new ExternalDatabaseServer(CatalogueRepository, "MyFictionalRemote", null); + externalServer.SetProperties(db); + + var attacher = new RemoteDatabaseAttacher(); + attacher.Initialize(null, db); + attacher.HistoricalFetchDuration = duration; + attacher.RemoteTableDateColumn = "date_seen"; + attacher.LoadRawColumnsOnly = scenario is Scenario.AllRawColumns or Scenario.MissingPreLoadDiscardedColumn; + attacher.RemoteSource = externalServer; + + if (duration == AttacherHistoricalDurations.Custom) + { + attacher.CustomFetchDurationStartDate = DateTime.Now.AddDays(-7); + attacher.CustomFetchDurationEndDate = DateTime.Now; + } + + if (duration == AttacherHistoricalDurations.DeltaReading) + { + attacher.DeltaReadingStartDate = DateTime.Now.AddDays(-7); + attacher.DeltaReadingLookBackDays = 0; + attacher.DeltaReadingLookForwardDays = 5; + } + + + var lm = new LogManager(CatalogueRepository.GetDefaultFor(PermissableDefaults.LiveLoggingServer_ID)); + lm.CreateNewLoggingTaskIfNotExists("amagad"); + var dli = lm.CreateDataLoadInfo("amagad", "p", "a", "", true); + + var job = NSubstitute.Substitute.For(); + job.RegularTablesToLoad.Returns(new List { ti }); + job.LookupTablesToLoad.Returns(new List()); + job.DataLoadInfo.Returns(dli); + + if (duration == AttacherHistoricalDurations.SinceLastUse) + { + job.LoadMetadata.LastLoadTime = DateTime.Now.AddDays(-1);// last used yesterday + job.LoadMetadata.SaveToDatabase(); + } + if (scenario == Scenario.MissingPreLoadDiscardedColumn) + { + var ex = Assert.Throws(() => + attacher.Attach(job, new GracefulCancellationToken())); + + Assert.That(ex.InnerException.InnerException.InnerException.Message.Contains("MyMissingCol"), Is.EqualTo(true)); + return; + } + + attacher.Attach(job, new GracefulCancellationToken()); + + Assert.That(tbl.GetRowCount(), Is.EqualTo(3)); + + using var dt2 = tbl.GetDataTable(); + VerifyRowExist(dt2, "Cow", withinDate); + + attacher.LoadCompletedSoDispose(ExitCodeType.Success, ThrowImmediatelyDataLoadEventListener.Quiet); + externalServer.DeleteInDatabase(); + } + + public enum Scenario { /// diff --git a/Rdmp.Core.Tests/DataLoad/Modules/Attachers/RemoteTableAttacherTests.cs b/Rdmp.Core.Tests/DataLoad/Modules/Attachers/RemoteTableAttacherTests.cs index 9eff584697..464982691f 100644 --- a/Rdmp.Core.Tests/DataLoad/Modules/Attachers/RemoteTableAttacherTests.cs +++ b/Rdmp.Core.Tests/DataLoad/Modules/Attachers/RemoteTableAttacherTests.cs @@ -7,6 +7,7 @@ using System; using System.Collections.Generic; using System.Data; +using System.Globalization; using FAnsi; using FAnsi.Discovery; using NSubstitute; @@ -24,6 +25,7 @@ using Rdmp.Core.ReusableLibraryCode.Progress; using Tests.Common; using TypeGuesser; +using static Rdmp.Core.Tests.DataLoad.Engine.Integration.RemoteDatabaseAttacherTests; namespace Rdmp.Core.Tests.DataLoad.Modules.Attachers; @@ -206,4 +208,169 @@ private void RunAttachStageWithLoadProgressJob(RemoteTableAttacher attacher, Dis Assert.That(tbl2.GetRowCount(), Is.EqualTo(mismatchProgress ? 0 : 1)); }); } + + [TestCase(DatabaseType.MicrosoftSQLServer, AttacherHistoricalDurations.Past24Hours)] + [TestCase(DatabaseType.MySql, AttacherHistoricalDurations.Past24Hours)] + [TestCase(DatabaseType.Oracle, AttacherHistoricalDurations.Past24Hours)] + [TestCase(DatabaseType.PostgreSql, AttacherHistoricalDurations.Past24Hours)] + [TestCase(DatabaseType.MicrosoftSQLServer, AttacherHistoricalDurations.Past7Days)] + [TestCase(DatabaseType.MySql, AttacherHistoricalDurations.Past7Days)] + [TestCase(DatabaseType.Oracle, AttacherHistoricalDurations.Past7Days)] + [TestCase(DatabaseType.PostgreSql, AttacherHistoricalDurations.Past7Days)] + [TestCase(DatabaseType.MicrosoftSQLServer, AttacherHistoricalDurations.PastMonth)] + [TestCase(DatabaseType.MySql, AttacherHistoricalDurations.PastMonth)] + [TestCase(DatabaseType.Oracle, AttacherHistoricalDurations.PastMonth)] + [TestCase(DatabaseType.PostgreSql, AttacherHistoricalDurations.PastMonth)] + [TestCase(DatabaseType.MySql, AttacherHistoricalDurations.PastYear)] + [TestCase(DatabaseType.Oracle, AttacherHistoricalDurations.PastYear)] + [TestCase(DatabaseType.PostgreSql, AttacherHistoricalDurations.PastYear)] + [TestCase(DatabaseType.MicrosoftSQLServer, AttacherHistoricalDurations.SinceLastUse)] + [TestCase(DatabaseType.MySql, AttacherHistoricalDurations.SinceLastUse)] + [TestCase(DatabaseType.Oracle, AttacherHistoricalDurations.SinceLastUse)] + [TestCase(DatabaseType.PostgreSql, AttacherHistoricalDurations.SinceLastUse)] + [TestCase(DatabaseType.MicrosoftSQLServer, AttacherHistoricalDurations.Custom)] + [TestCase(DatabaseType.MySql, AttacherHistoricalDurations.Custom)] + [TestCase(DatabaseType.Oracle, AttacherHistoricalDurations.Custom)] + [TestCase(DatabaseType.PostgreSql, AttacherHistoricalDurations.Custom)] + [TestCase(DatabaseType.MicrosoftSQLServer, AttacherHistoricalDurations.DeltaReading)] + [TestCase(DatabaseType.MySql, AttacherHistoricalDurations.DeltaReading)] + [TestCase(DatabaseType.Oracle, AttacherHistoricalDurations.DeltaReading)] + [TestCase(DatabaseType.PostgreSql, AttacherHistoricalDurations.DeltaReading)] + public void TestRemoteTableAttacher_DateFilters(DatabaseType dbType, AttacherHistoricalDurations duration) + { + var db = GetCleanedServer(dbType); + + var attacher = new RemoteTableAttacher + { + //where to go for data + RemoteServer = db.Server.Name, + RemoteDatabaseName = db.GetRuntimeName(), + DatabaseType = db.Server.DatabaseType + }; + + if (db.Server.ExplicitUsernameIfAny != null) + { + var creds = new DataAccessCredentials(CatalogueRepository) + { + Username = db.Server.ExplicitUsernameIfAny, + Password = db.Server.ExplicitPasswordIfAny + }; + creds.SaveToDatabase(); + attacher.RemoteTableAccessCredentials = creds; + } + attacher.HistoricalFetchDuration = duration; + + RunAttachStageWithFilterJob(attacher, db, duration); + } + + + private static string Within(AttacherHistoricalDurations duration) + { + switch (duration) + { + case AttacherHistoricalDurations.Past24Hours: + return DateTime.Now.AddHours(-1).ToString(); + case AttacherHistoricalDurations.Past7Days: + return DateTime.Now.AddHours(-1).ToString(); + case AttacherHistoricalDurations.PastMonth: + return DateTime.Now.AddHours(-1).ToString(); + case AttacherHistoricalDurations.PastYear: + return DateTime.Now.AddHours(-1).ToString(); + case AttacherHistoricalDurations.SinceLastUse: + return DateTime.Now.AddHours(-1).ToString(); + case AttacherHistoricalDurations.Custom: + return DateTime.Now.AddDays(-1).ToString(); + case AttacherHistoricalDurations.DeltaReading: + return DateTime.Now.AddDays(-4).ToString(); + default: + return "fail"; + } + } + + private static string Outwith(AttacherHistoricalDurations duration) + { + switch (duration) + { + case AttacherHistoricalDurations.Past24Hours: + return DateTime.Now.AddDays(-2).ToString(); + case AttacherHistoricalDurations.Past7Days: + return DateTime.Now.AddDays(-8).ToString(); + case AttacherHistoricalDurations.PastMonth: + return DateTime.Now.AddMonths(-2).ToString(); + case AttacherHistoricalDurations.PastYear: + return DateTime.Now.AddYears(-2).ToString(); + case AttacherHistoricalDurations.SinceLastUse: + return DateTime.Now.AddDays(-2).ToString(); + case AttacherHistoricalDurations.Custom: + return DateTime.Now.AddDays(-14).ToString(); + case AttacherHistoricalDurations.DeltaReading: + return DateTime.Now.AddDays(-10).ToString(); + default: + return "fail"; + } + } + private void RunAttachStageWithFilterJob(RemoteTableAttacher attacher, DiscoveredDatabase db, AttacherHistoricalDurations duration) + { + //the table to get data from + attacher.RemoteTableName = "table1"; + attacher.RAWTableName = "table2"; + attacher.RemoteTableDateColumn = "dateColumn"; + attacher.Check(ThrowImmediatelyCheckNotifier.Quiet); + + attacher.Initialize(null, db); + + using var dt = new DataTable(); + var within = Within(duration); + var outwith = Outwith(duration); + dt.Columns.Add("Col1"); + dt.Columns.Add("dateColumn"); + dt.Rows.Add("fff", within); + dt.Rows.Add("rrr", outwith); + + var tbl1 = db.CreateTable("table1", dt); + var tbl2 = db.CreateTable("table2", + new[] { new DatabaseColumnRequest("Col1", new DatabaseTypeRequest(typeof(string), 5)), new DatabaseColumnRequest("dateColumn", new DatabaseTypeRequest(typeof(string), 100)) }); + + Assert.Multiple(() => + { + Assert.That(tbl1.GetRowCount(), Is.EqualTo(2)); + Assert.That(tbl2.GetRowCount(), Is.EqualTo(0)); + }); + + var logManager = new LogManager(new DiscoveredServer(UnitTestLoggingConnectionString)); + + var lmd = RdmpMockFactory.Mock_LoadMetadataLoadingTable(tbl2); + lmd.CatalogueRepository.Returns(CatalogueRepository); + logManager.CreateNewLoggingTaskIfNotExists(lmd.GetDistinctLoggingTask()); + + var dbConfiguration = new HICDatabaseConfiguration(lmd, + RdmpMockFactory.Mock_INameDatabasesAndTablesDuringLoads(db, "table2")); + + var job = new DataLoadJob(RepositoryLocator, "test job", logManager, lmd, new TestLoadDirectory(), + ThrowImmediatelyDataLoadEventListener.Quiet, dbConfiguration); + job.StartLogging(); + if (duration == AttacherHistoricalDurations.SinceLastUse) + { + job.LoadMetadata.LastLoadTime = DateTime.Now.AddDays(-1);// last used yesterday + job.LoadMetadata.SaveToDatabase(); + } + if (duration == AttacherHistoricalDurations.Custom) + { + attacher.CustomFetchDurationStartDate = DateTime.Now.AddDays(-7); + attacher.CustomFetchDurationEndDate = DateTime.Now; + } + if (duration == AttacherHistoricalDurations.DeltaReading) + { + attacher.DeltaReadingStartDate = DateTime.Now.AddDays(-7); + attacher.DeltaReadingLookBackDays = 0; + attacher.DeltaReadingLookForwardDays = 5; + } + attacher.Attach(job, new GracefulCancellationToken()); + + Assert.Multiple(() => + { + Assert.That(tbl1.GetRowCount(), Is.EqualTo(2)); + Assert.That(tbl2.GetRowCount(), Is.EqualTo(1)); + }); + } } diff --git a/Rdmp.Core/CommandLine/Runners/DleRunner.cs b/Rdmp.Core/CommandLine/Runners/DleRunner.cs index a69afdae0f..54346a2e8e 100644 --- a/Rdmp.Core/CommandLine/Runners/DleRunner.cs +++ b/Rdmp.Core/CommandLine/Runners/DleRunner.cs @@ -5,6 +5,7 @@ // You should have received a copy of the GNU General Public License along with RDMP. If not, see . using System; +using System.Collections.Generic; using System.Linq; using Rdmp.Core.CommandLine.Options; using Rdmp.Core.Curation.Data; @@ -18,6 +19,7 @@ using Rdmp.Core.DataLoad.Engine.LoadProcess; using Rdmp.Core.DataLoad.Engine.LoadProcess.Scheduling; using Rdmp.Core.DataLoad.Engine.LoadProcess.Scheduling.Strategy; +using Rdmp.Core.DataLoad.Modules.Attachers; using Rdmp.Core.Logging; using Rdmp.Core.Repositories; using Rdmp.Core.ReusableLibraryCode.Checks; @@ -31,12 +33,10 @@ namespace Rdmp.Core.CommandLine.Runners; public class DleRunner : Runner { private readonly DleOptions _options; - public DleRunner(DleOptions options) { _options = options; } - public override int Run(IRDMPPlatformRepositoryServiceLocator locator, IDataLoadEventListener listener, ICheckNotifier checkNotifier, GracefulCancellationToken token) { @@ -101,8 +101,38 @@ public override int Run(IRDMPPlatformRepositoryServiceLocator locator, IDataLoad execution, databaseConfiguration); } + var exitCode = dataLoadProcess.Run(token); + if (exitCode is ExitCodeType.Success) + { + //Store the date of the last successful load + loadMetadata.LastLoadTime = DateTime.Now; + loadMetadata.SaveToDatabase(); + List processTasks = loadMetadata.ProcessTasks.Where(ipt => ipt.Path == typeof(RemoteDatabaseAttacher).FullName || ipt.Path == typeof(RemoteTableAttacher).FullName).ToList(); + if (processTasks.Count() > 0) //if using a remote attacher, there may be some additional work to do + { + foreach (IEnumerable arguments in processTasks.Select(task => task.GetAllArguments())) + { + foreach (var argument in arguments.Where(arg => arg.Name == RemoteAttacherPropertiesValidator("DeltaReadingStartDate") && arg.Value is not null)) + { + var scanForwardDate = arguments.Where(a => a.Name == RemoteAttacherPropertiesValidator("DeltaReadingLookForwardDays")).First(); + var arg = (ProcessTaskArgument)argument; + arg.Value = DateTime.Parse(argument.Value.ToString()).AddDays(Int32.Parse(scanForwardDate.Value)).ToString(); + if (arguments.Where(a => a.Name == RemoteAttacherPropertiesValidator("SetDeltaReadingToLatestSeenDatePostLoad")).First().Value == "True") + { + var mostRecentValue = arguments.Single(a => a.Name == RemoteAttacherPropertiesValidator("MostRecentlySeenDate")).Value; + if (mostRecentValue is not null) + { + arg.Value = DateTime.Parse(mostRecentValue).ToString(); + } + } + arg.SaveToDatabase(); + } + } + } + } + //return 0 for success or load not required otherwise return the exit code (which will be non zero so error) return exitCode is ExitCodeType.Success or ExitCodeType.OperationNotRequired ? 0 : (int)exitCode; case CommandLineActivity.check: @@ -114,4 +144,16 @@ public override int Run(IRDMPPlatformRepositoryServiceLocator locator, IDataLoad throw new ArgumentOutOfRangeException(); } } + + private string RemoteAttacherPropertiesValidator(string propertyName) + { + var properties = typeof(RemoteAttacher).GetProperties(); + var foundProperties = properties.Where(p => p.Name == propertyName); + if (foundProperties.Any()) + { + return foundProperties.First().Name; + } + throw new Exception($"Attempting to access the property {propertyName} of the RemoteAttacher class. This property does not exist."); + + } } \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/DataLoad/ILoadMetadata.cs b/Rdmp.Core/Curation/Data/DataLoad/ILoadMetadata.cs index f35c3eb34f..fdb6a00283 100644 --- a/Rdmp.Core/Curation/Data/DataLoad/ILoadMetadata.cs +++ b/Rdmp.Core/Curation/Data/DataLoad/ILoadMetadata.cs @@ -1,9 +1,10 @@ -// Copyright (c) The University of Dundee 2018-2019 +// Copyright (c) The University of Dundee 2018-2024 // This file is part of the Research Data Management Platform (RDMP). // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . +using System; using System.Collections.Generic; using System.Linq; using FAnsi.Discovery; @@ -64,4 +65,9 @@ public interface ILoadMetadata : INamed, ILoggedActivityRootObject /// /// DiscoveredServer GetDistinctLiveDatabaseServer(); + + /// + /// Stores the most recent time the load was successfully ran + /// + DateTime? LastLoadTime { get; set; } } \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs b/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs index dbd1514639..6d9cc25627 100644 --- a/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs +++ b/Rdmp.Core/Curation/Data/DataLoad/LoadMetadata.cs @@ -1,4 +1,4 @@ -// Copyright (c) The University of Dundee 2018-2019 +// Copyright (c) The University of Dundee 2018-2024 // This file is part of the Research Data Management Platform (RDMP). // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. @@ -55,6 +55,7 @@ public class LoadMetadata : DatabaseEntity, ILoadMetadata, IHasDependencies, IHa private int? _overrideRawServerID; private bool _ignoreTrigger; private string _folder; + private DateTime? _lastLoadTime; /// [AdjustableLocation] @@ -127,6 +128,16 @@ public string Folder set => SetField(ref _folder, FolderHelper.Adjust(value)); } + + /// + /// Stores the last time the load was ran. + /// + public DateTime? LastLoadTime + { + get => _lastLoadTime; + set => SetField(ref _lastLoadTime, value); + } + #endregion @@ -169,11 +180,13 @@ public LoadMetadata() public LoadMetadata(ICatalogueRepository repository, string name = null) { name ??= $"NewLoadMetadata{Guid.NewGuid()}"; + repository.InsertAndHydrate(this, new Dictionary { { "Name", name }, { "IgnoreTrigger", false /*todo could be system global default here*/ }, - { "Folder", FolderHelper.Root } + { "Folder", FolderHelper.Root }, + {"LastLoadTime", null } }); } @@ -189,6 +202,7 @@ internal LoadMetadata(ICatalogueRepository repository, DbDataReader r) OverrideRAWServer_ID = ObjectToNullableInt(r["OverrideRAWServer_ID"]); IgnoreTrigger = ObjectToNullableBool(r["IgnoreTrigger"]) ?? false; Folder = r["Folder"] as string ?? FolderHelper.Root; + LastLoadTime = string.IsNullOrWhiteSpace(r["LastLoadTime"].ToString()) ?null: DateTime.Parse(r["LastLoadTime"].ToString()); } internal LoadMetadata(ShareManager shareManager, ShareDefinition shareDefinition) : base() diff --git a/Rdmp.Core/DataLoad/Modules/Attachers/AttacherHistoricalDurations.cs b/Rdmp.Core/DataLoad/Modules/Attachers/AttacherHistoricalDurations.cs new file mode 100644 index 0000000000..5958d27ce6 --- /dev/null +++ b/Rdmp.Core/DataLoad/Modules/Attachers/AttacherHistoricalDurations.cs @@ -0,0 +1,19 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . + +namespace Rdmp.Core.DataLoad.Modules.Attachers; + +public enum AttacherHistoricalDurations +{ + AllTime, + Past24Hours, + Past7Days, + PastMonth, + PastYear, + SinceLastUse, + Custom, + DeltaReading +} \ No newline at end of file diff --git a/Rdmp.Core/DataLoad/Modules/Attachers/RemoteAttacher.cs b/Rdmp.Core/DataLoad/Modules/Attachers/RemoteAttacher.cs new file mode 100644 index 0000000000..ae5f56b35d --- /dev/null +++ b/Rdmp.Core/DataLoad/Modules/Attachers/RemoteAttacher.cs @@ -0,0 +1,208 @@ +// Copyright (c) The University of Dundee 2024-2024 +// This file is part of the Research Data Management Platform (RDMP). +// RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. +// RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +// You should have received a copy of the GNU General Public License along with RDMP. If not, see . +using FAnsi; +using FAnsi.Discovery.QuerySyntax; +using Rdmp.Core.Curation.Data; +using Rdmp.Core.Curation.Data.DataLoad; +using Rdmp.Core.DataFlowPipeline; +using Rdmp.Core.DataLoad.Engine.Attachers; +using Rdmp.Core.DataLoad.Engine.Job; +using Rdmp.Core.DataLoad.Engine.Pipeline.Sources; +using Rdmp.Core.ReusableLibraryCode.Checks; +using Rdmp.Core.ReusableLibraryCode.Progress; +using System; +using System.Data; +using System.Globalization; +using System.Linq; + +namespace Rdmp.Core.DataLoad.Modules.Attachers; + +/// +/// Root class to facilitate the Table and Database remote attachers. +/// +public class RemoteAttacher : Attacher, IPluginAttacher +{ + + public RemoteAttacher() : base(true) { } + [DemandsInitialization("How far back to pull data from")] + public AttacherHistoricalDurations HistoricalFetchDuration { get; set; } + + [DemandsInitialization("Which column in the remote table can be used to perform time-based data selection")] + public string RemoteTableDateColumn { get; set; } + + private readonly string RemoteTableDateFormat = "yyyy-MM-dd HH:mm:ss.fff"; + + [DemandsInitialization("Earliest date when using a custom fetch duration")] + public DateTime CustomFetchDurationStartDate { get; set; } + + [DemandsInitialization("Latest date when using a custom fetch duration")] + public DateTime CustomFetchDurationEndDate { get; set; } + + [DemandsInitialization("The Current Start Date for procedural fetching of historical data")] + public DateTime DeltaReadingStartDate { get; set; } + [DemandsInitialization("How many days the procedural fetching should look back ")] + public int DeltaReadingLookBackDays { get; set; } = 0; + [DemandsInitialization("How many days the procedural fetching should look forward ")] + public int DeltaReadingLookForwardDays { get; set; } = 0; + + [DemandsInitialization("If you only want to progress the procedural load to the most recent date seen in the procedural load, not the date + X days, then tick this box")] + public bool SetDeltaReadingToLatestSeenDatePostLoad { get; set; } = false; + + [DemandsInitialization("Internal Value")] + public DateTime? MostRecentlySeenDate { get; set; } + + + + private static string GetCorrectDateAddForDatabaseType(DatabaseType dbType, string addType, string amount) + { + switch (dbType) + { + case DatabaseType.PostgreSql: + return $"cast((NOW() + interval '{amount} {addType}S') as Date)"; + case DatabaseType.Oracle: + if (addType == "DAY") return $"DateAdd(DATE(),,{amount})"; + if (addType == "WEEK") return $"DateAdd(DATE(),,{amount} *7)"; + if (addType == "MONTH") return $"DateAdd(DATE(),,,{amount})"; + if (addType == "YEAR") return $"DateAdd(DATE(),,,,{amount})"; + return $"DateAdd(DATE(),,{amount})"; + case DatabaseType.MicrosoftSQLServer: + return $"DATEADD({addType}, {amount}, GETDATE())"; + case DatabaseType.MySql: + return $"DATE_ADD(CURDATE(), INTERVAL {amount} {addType})"; + default: + throw new InvalidOperationException("Unknown Database Type"); + } + } + + private string ConvertDateString(DatabaseType dbType, string dateString) + { + switch (dbType) + { + case DatabaseType.PostgreSql: + return $"'{dateString}'"; + case DatabaseType.Oracle: + return $"TO_DATE('{dateString}')"; + case DatabaseType.MicrosoftSQLServer: + return $"convert(Date,'{dateString}')"; + case DatabaseType.MySql: + return $"convert('{dateString}',Date)"; + default: + return $"convert(Date,'{dateString}')"; + } + + } + + public string SqlHistoricalDataFilter(ILoadMetadata loadMetadata, DatabaseType dbType) + { + string _dateConvert = dbType == DatabaseType.PostgreSql ? "Date" : "Date"; + + switch (HistoricalFetchDuration) + { + case AttacherHistoricalDurations.Past24Hours: + return $" WHERE CAST({RemoteTableDateColumn} as {_dateConvert}) >= {GetCorrectDateAddForDatabaseType(dbType, "DAY", "-1")}"; + case AttacherHistoricalDurations.Past7Days: + return $" WHERE CAST({RemoteTableDateColumn} as {_dateConvert}) >= {GetCorrectDateAddForDatabaseType(dbType, "WEEK", "-1")}"; + case AttacherHistoricalDurations.PastMonth: + return $" WHERE CAST({RemoteTableDateColumn} as {_dateConvert}) >= {GetCorrectDateAddForDatabaseType(dbType, "MONTH", "-1")}"; + case AttacherHistoricalDurations.PastYear: + return $" WHERE CAST({RemoteTableDateColumn} as {_dateConvert}) >= {GetCorrectDateAddForDatabaseType(dbType, "YEAR", "-1")}"; + case AttacherHistoricalDurations.SinceLastUse: + if (loadMetadata.LastLoadTime is not null) return $" WHERE CAST({RemoteTableDateColumn} as {_dateConvert}) >= {ConvertDateString(dbType, loadMetadata.LastLoadTime.GetValueOrDefault().ToString(RemoteTableDateFormat))}"; + return ""; + case AttacherHistoricalDurations.Custom: + if (CustomFetchDurationStartDate == DateTime.MinValue && CustomFetchDurationEndDate != DateTime.MinValue) + { + //end only + return $" WHERE CAST({RemoteTableDateColumn} as {_dateConvert}) <= {ConvertDateString(dbType, CustomFetchDurationEndDate.ToString(RemoteTableDateFormat))}"; + + } + if (CustomFetchDurationStartDate != DateTime.MinValue && CustomFetchDurationEndDate == DateTime.MinValue) + { + //start only + return $" WHERE CAST({RemoteTableDateColumn} as {_dateConvert}) >= {ConvertDateString(dbType, CustomFetchDurationStartDate.ToString(RemoteTableDateFormat))}"; + + } + if (CustomFetchDurationStartDate == DateTime.MinValue && CustomFetchDurationEndDate == DateTime.MinValue) + { + //No Dates + return ""; + } + return $" WHERE CAST({RemoteTableDateColumn} as {_dateConvert}) >= {ConvertDateString(dbType, CustomFetchDurationStartDate.ToString(RemoteTableDateFormat))} AND CAST({RemoteTableDateColumn} as {_dateConvert}) <= {ConvertDateString(dbType, CustomFetchDurationEndDate.ToString(RemoteTableDateFormat))}"; + case AttacherHistoricalDurations.DeltaReading: + if (DeltaReadingStartDate == DateTime.MinValue) return ""; + var startDate = DeltaReadingStartDate.AddDays(-DeltaReadingLookBackDays); + var endDate = DeltaReadingStartDate.AddDays(DeltaReadingLookForwardDays); + return $" WHERE CAST({RemoteTableDateColumn} as {_dateConvert}) >= {ConvertDateString(dbType, startDate.ToString(RemoteTableDateFormat))} AND CAST({RemoteTableDateColumn} as {_dateConvert}) <= {ConvertDateString(dbType, endDate.ToString(RemoteTableDateFormat))}"; + default: + return ""; + } + } + + + + private bool IsThisRemoteAttacher(IProcessTask task) + { + if (task.ProcessTaskType != ProcessTaskType.Attacher) return false; + try + { + if (HistoricalFetchDuration.ToString() != task.ProcessTaskArguments.Where(arg => arg.Name == "HistoricalFetchDuration").First().Value) return false; + if (RemoteTableDateColumn.ToString() != task.ProcessTaskArguments.Where(arg => arg.Name == "RemoteTableDateColumn").First().Value) return false; + + if (CustomFetchDurationStartDate == DateTime.MinValue && task.ProcessTaskArguments.Where(arg => arg.Name == "CustomFetchDurationStartDate").First().Value != null) return false; + if (CustomFetchDurationStartDate != DateTime.MinValue && task.ProcessTaskArguments.Where(arg => arg.Name == "CustomFetchDurationStartDate").First().Value == null) return false; + if (CustomFetchDurationStartDate != DateTime.MinValue && DateTime.Parse(task.ProcessTaskArguments.Where(arg => arg.Name == "CustomFetchDurationStartDate").First().Value) != CustomFetchDurationStartDate) return false; + + if (CustomFetchDurationEndDate == DateTime.MinValue && task.ProcessTaskArguments.Where(arg => arg.Name == "CustomFetchDurationStartDate").First().Value != null) return false; + if (CustomFetchDurationEndDate != DateTime.MinValue && task.ProcessTaskArguments.Where(arg => arg.Name == "CustomFetchDurationEndDate").First().Value == null) return false; + if (CustomFetchDurationEndDate != DateTime.MinValue && DateTime.Parse(task.ProcessTaskArguments.Where(arg => arg.Name == "CustomFetchDurationEndDate").First().Value) != CustomFetchDurationEndDate) return false; + + if (DeltaReadingStartDate == DateTime.MinValue && task.ProcessTaskArguments.Where(arg => arg.Name == "DeltaReadingStartDate").First().Value != null) return false; + if (DeltaReadingStartDate != DateTime.MinValue && task.ProcessTaskArguments.Where(arg => arg.Name == "DeltaReadingStartDate").First().Value == null) return false; + if (DeltaReadingStartDate != DateTime.MinValue && DateTime.Parse(task.ProcessTaskArguments.Where(arg => arg.Name == "DeltaReadingStartDate").First().Value) != DeltaReadingStartDate) return false; + + if (DeltaReadingLookBackDays.ToString() != task.ProcessTaskArguments.Where(arg => arg.Name == "DeltaReadingLookBackDays").First().Value) return false; + if (DeltaReadingLookForwardDays.ToString() != task.ProcessTaskArguments.Where(arg => arg.Name == "DeltaReadingLookForwardDays").First().Value) return false; + if (SetDeltaReadingToLatestSeenDatePostLoad.ToString() != task.ProcessTaskArguments.Where(arg => arg.Name == "SetDeltaReadingToLatestSeenDatePostLoad").First().Value) return false; + } + catch (Exception) + { + return false; + } + return true; + } + public void FindMostRecentDateInLoadedData(IQuerySyntaxHelper syntaxFrom, DatabaseType dbType, string table, IDataLoadJob job) + { + string maxDateSql = $"SELECT MAX({RemoteTableDateColumn}) FROM {syntaxFrom.EnsureWrapped(table)} {SqlHistoricalDataFilter(job.LoadMetadata, dbType)}"; + + + using var con = _dbInfo.Server.GetConnection(); + using var dt = new DataTable(); + using var cmd = _dbInfo.Server.GetCommand(maxDateSql, con); + cmd.CommandTimeout = 30000; + using var da = _dbInfo.Server.GetDataAdapter(cmd); + da.Fill(dt); + MostRecentlySeenDate = dt.Rows.Count > 0 && dt.Rows[0].ItemArray[0].ToString() != "" ? DateTime.Parse(dt.Rows[0].ItemArray[0].ToString()) : null; + foreach (ProcessTask task in job.LoadMetadata.ProcessTasks.Where(pt => IsThisRemoteAttacher(pt))) + { + task.SetArgumentValue("MostRecentlySeenDate", MostRecentlySeenDate); + task.SaveToDatabase(); + } + } + + public override ExitCodeType Attach(IDataLoadJob job, GracefulCancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + public override void Check(ICheckNotifier notifier) + { + throw new NotImplementedException(); + } + + public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener) + { + } +} diff --git a/Rdmp.Core/DataLoad/Modules/Attachers/RemoteDatabaseAttacher.cs b/Rdmp.Core/DataLoad/Modules/Attachers/RemoteDatabaseAttacher.cs index 4a9f36e719..0bc8380fed 100644 --- a/Rdmp.Core/DataLoad/Modules/Attachers/RemoteDatabaseAttacher.cs +++ b/Rdmp.Core/DataLoad/Modules/Attachers/RemoteDatabaseAttacher.cs @@ -1,4 +1,4 @@ -// Copyright (c) The University of Dundee 2018-2019 +// Copyright (c) The University of Dundee 2018-2023 // This file is part of the Research Data Management Platform (RDMP). // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. @@ -12,7 +12,6 @@ using Rdmp.Core.Curation.Data.DataLoad; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataFlowPipeline.Requirements; -using Rdmp.Core.DataLoad.Engine.Attachers; using Rdmp.Core.DataLoad.Engine.Job; using Rdmp.Core.DataLoad.Engine.Pipeline.Destinations; using Rdmp.Core.DataLoad.Engine.Pipeline.Sources; @@ -27,9 +26,9 @@ namespace Rdmp.Core.DataLoad.Modules.Attachers; /// Data load component for loading RAW tables with records read from a remote database server. /// Fetches all table from the specified database to load all catalogues specified. /// -public class RemoteDatabaseAttacher : Attacher, IPluginAttacher +public class RemoteDatabaseAttacher : RemoteAttacher { - public RemoteDatabaseAttacher() : base(true) + public RemoteDatabaseAttacher() : base() { } @@ -54,14 +53,16 @@ public RemoteDatabaseAttacher() : base(true) ")] public bool IgnoreMissingTables { get; set; } + + + public override void Check(ICheckNotifier notifier) { if (!RemoteSource.Discover(DataAccessContext.DataLoad).Exists()) throw new Exception($"Database {RemoteSource.Database} did not exist on the remote server"); - } - public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener) - { + if (HistoricalFetchDuration is not AttacherHistoricalDurations.AllTime && RemoteTableDateColumn is null) + throw new Exception("No Remote Table Date Column is set, but a historical duration is. Add a date column to continue."); } public override ExitCodeType Attach(IDataLoadJob job, GracefulCancellationToken cancellationToken) @@ -96,7 +97,6 @@ public override ExitCodeType Attach(IDataLoadJob job, GracefulCancellationToken continue; } - string sql; if (LoadRawColumnsOnly) { @@ -104,13 +104,12 @@ public override ExitCodeType Attach(IDataLoadJob job, GracefulCancellationToken ? tableInfo.GetColumnsAtStage(LoadStage.AdjustRaw) : tableInfo.ColumnInfos; sql = - $"SELECT {string.Join(",", rawColumns.Select(c => syntaxFrom.EnsureWrapped(c.GetRuntimeName(LoadStage.AdjustRaw))))} FROM {syntaxFrom.EnsureWrapped(table)}"; + $"SELECT {string.Join(",", rawColumns.Select(c => syntaxFrom.EnsureWrapped(c.GetRuntimeName(LoadStage.AdjustRaw))))} FROM {syntaxFrom.EnsureWrapped(table)} {SqlHistoricalDataFilter(job.LoadMetadata, dbFrom.Server.DatabaseType)}"; } else { - sql = $"SELECT * FROM {syntaxFrom.EnsureWrapped(table)}"; + sql = $"SELECT * FROM {syntaxFrom.EnsureWrapped(table)} {SqlHistoricalDataFilter(job.LoadMetadata,RemoteSource.DatabaseType)}"; } - job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"About to execute SQL:{Environment.NewLine}{sql}")); @@ -143,8 +142,16 @@ public override ExitCodeType Attach(IDataLoadJob job, GracefulCancellationToken job.OnNotify(this, new NotifyEventArgs( source.TotalRowsRead > 0 ? ProgressEventType.Information : ProgressEventType.Warning, $"Finished after reading {source.TotalRowsRead} rows")); + + if (SetDeltaReadingToLatestSeenDatePostLoad) + { + FindMostRecentDateInLoadedData(syntaxFrom, dbFrom.Server.DatabaseType ,table, job); + } + } + + return ExitCodeType.Success; } } \ No newline at end of file diff --git a/Rdmp.Core/DataLoad/Modules/Attachers/RemoteTableAttacher.cs b/Rdmp.Core/DataLoad/Modules/Attachers/RemoteTableAttacher.cs index f78290bca3..953a7e9db9 100644 --- a/Rdmp.Core/DataLoad/Modules/Attachers/RemoteTableAttacher.cs +++ b/Rdmp.Core/DataLoad/Modules/Attachers/RemoteTableAttacher.cs @@ -34,11 +34,11 @@ namespace Rdmp.Core.DataLoad.Modules.Attachers; /// Data load component for loading RAW tables with records read from a remote database server. Runs the specified query (which can include a date parameter) /// and inserts the results of the query into RAW. /// -public class RemoteTableAttacher : Attacher, IPluginAttacher +public class RemoteTableAttacher : RemoteAttacher { private const string FutureLoadMessage = "Cannot load data from the future"; - public RemoteTableAttacher() : base(true) + public RemoteTableAttacher() : base() { } @@ -289,10 +289,9 @@ public override void Check(ICheckNotifier notifier) notifier.OnCheckPerformed(new CheckEventArgs( $"{nameof(RAWTableName)}('{RAWTableName}') will be ignored because {nameof(RAWTableToLoad)} has been set", CheckResult.Warning)); - } - public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener) - { + if (HistoricalFetchDuration is not AttacherHistoricalDurations.AllTime && RemoteTableDateColumn is null) + throw new Exception("No Remote Table Date Column is set, but a historical duration is. Add a date column to continue."); } protected void CheckTablesExist(ICheckNotifier notifier) @@ -362,10 +361,9 @@ public override ExitCodeType Attach(IDataLoadJob job, GracefulCancellationToken var sql = !string.IsNullOrWhiteSpace(RemoteSelectSQL) ? RemoteSelectSQL - : $"Select * from {syntax.EnsureWrapped(RemoteTableName)}"; + : $"Select * from {syntax.EnsureWrapped(RemoteTableName)} {SqlHistoricalDataFilter(job.LoadMetadata,DatabaseType)}"; var scheduleMismatch = false; - //if there is a load progress if (Progress != null) try @@ -457,6 +455,11 @@ public override ExitCodeType Attach(IDataLoadJob job, GracefulCancellationToken ProgressUpdateStrategy.AddAppropriateDisposeStep((ScheduledDataLoadJob)job, _dbInfo); } + if (SetDeltaReadingToLatestSeenDatePostLoad) + { + FindMostRecentDateInLoadedData(rawSyntax, _dbInfo.Server.DatabaseType ,rawTableName, job); + } + return ExitCodeType.Success; } diff --git a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql index ac7d12b279..e513488869 100644 --- a/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql +++ b/Rdmp.Core/Databases/CatalogueDatabase/runAfterCreateDatabase/CreateCatalogue.sql @@ -917,7 +917,8 @@ GO ALTER TABLE [dbo].[ColumnInfo] ADD Dataset_ID [int] NULL GO ALTER TABLE [dbo].[ColumnInfo] ADD CONSTRAINT [FK_Column_Info_Dataset] FOREIGN KEY([Dataset_ID]) REFERENCES [dbo].[Dataset] ([ID]) ON DELETE CASCADE ON UPDATE CASCADE - +GO +ALTER TABLE [dbo].[LoadMetadata] ADD LastLoadTime [datetime] NULL; GO SET ANSI_PADDING OFF GO diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/078_AddLastLoadTimeToLoadMetadata.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/078_AddLastLoadTimeToLoadMetadata.sql new file mode 100644 index 0000000000..15e92ed3ea --- /dev/null +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/078_AddLastLoadTimeToLoadMetadata.sql @@ -0,0 +1,7 @@ +--Version:8.1.4 +--Description: Adds LastLoadTime to LoadMetadata table + GO +if not exists (select 1 from INFORMATION_SCHEMA.COLUMNS where TABLE_NAME='LoadMetaData' and COLUMN_NAME='LastLoadtime') +BEGIN +ALTER TABLE LoadMetadata ADD LastLoadTime DATETIME NULL; +END diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index 22f1e5afdd..00edd93938 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -1,382 +1,384 @@ - - - HIC.RDMP.Plugin - $(version) - HIC.RDMP.Plugin - Health Informatics Centre, University of Dundee - Health Informatics Centre, University of Dundee - https://raw.githubusercontent.com/HicServices/RDMP/master/LICENSE - https://github.com/HicServices/RDMP - - https://raw.githubusercontent.com/HicServices/RDMP/master/Application/ResearchDataManagementPlatform/Icon/main.png - false - GPL-3.0-or-later - Core package for plugin development - Copyright 2018-2019 - - net7.0 - false - true - net7.0 - true - 1701;1702;CS1591;SCS0018 - embedded - true - $(NoWarn);NU5104 - - - Rdmp.Core.DataLoad.Modules.DataProvider.WebServiceConfiguration - - - true - latest - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Never - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - - - - - - - - - - - Never - - - - - - - - - True - True - GlobalStrings.resx - - - True - True - ChecksAndProgressIcons.resx - - - True - True - Images.resx - - - DatabaseProviderIcons.resx - True - True - - - - - PublicResXFileCodeGenerator - GlobalStrings.Designer.cs - - - PublicResXFileCodeGenerator - - - - - - PublicResXFileCodeGenerator - ChecksAndProgressIcons.Designer.cs - - - Designer - PublicResXFileCodeGenerator - Images.Designer.cs - - - Designer - DatabaseProviderIcons.Designer.cs - PublicResXFileCodeGenerator - - + + + HIC.RDMP.Plugin + $(version) + HIC.RDMP.Plugin + Health Informatics Centre, University of Dundee + Health Informatics Centre, University of Dundee + https://raw.githubusercontent.com/HicServices/RDMP/master/LICENSE + https://github.com/HicServices/RDMP + + https://raw.githubusercontent.com/HicServices/RDMP/master/Application/ResearchDataManagementPlatform/Icon/main.png + + false + GPL-3.0-or-later + Core package for plugin development + Copyright 2018-2019 + net7.0 + false + true + net7.0 + true + 1701;1702;CS1591;SCS0018 + embedded + true + $(NoWarn);NU5104 + + + Rdmp.Core.DataLoad.Modules.DataProvider.WebServiceConfiguration + + + true + latest + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Never + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + + + + + Never + + + + + + + + + True + True + GlobalStrings.resx + + + True + True + ChecksAndProgressIcons.resx + + + True + True + Images.resx + + + DatabaseProviderIcons.resx + True + True + + + + + PublicResXFileCodeGenerator + GlobalStrings.Designer.cs + + + PublicResXFileCodeGenerator + + + + + + PublicResXFileCodeGenerator + ChecksAndProgressIcons.Designer.cs + + + Designer + PublicResXFileCodeGenerator + Images.Designer.cs + + + Designer + DatabaseProviderIcons.Designer.cs + PublicResXFileCodeGenerator + + \ No newline at end of file diff --git a/Rdmp.UI/Menus/LoadStageNodeMenu.cs b/Rdmp.UI/Menus/LoadStageNodeMenu.cs index 44aa3a9b9a..665e7d5ace 100644 --- a/Rdmp.UI/Menus/LoadStageNodeMenu.cs +++ b/Rdmp.UI/Menus/LoadStageNodeMenu.cs @@ -13,6 +13,7 @@ using Rdmp.Core.DataLoad.Engine.DataProvider; using Rdmp.Core.DataLoad.Engine.DataProvider.FromCache; using Rdmp.Core.DataLoad.Engine.Mutilators; +using Rdmp.Core.DataLoad.Modules.Attachers; using Rdmp.Core.Providers.Nodes.LoadMetadataNodes; using Rdmp.Core.Repositories; @@ -31,7 +32,7 @@ public LoadStageNodeMenu(RDMPContextMenuStripArgs args, LoadStageNode loadStageN AddMenu("Add Cached Data Provider", t => typeof(ICachedDataProvider).IsAssignableFrom(t)); AddMenu("Add Data Provider", t => !typeof(ICachedDataProvider).IsAssignableFrom(t)); - AddMenu("Add Attacher"); + AddMenu("Add Attacher", t=> !typeof(RemoteAttacher).IsAssignableTo(t) ); AddMenu("Add Mutilator"); } diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index abcc7124ff..4345b6c353 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -6,7 +6,7 @@ [assembly: AssemblyCompany("Health Informatics Centre, University of Dundee")] [assembly: AssemblyProduct("Research Data Management Platform (RDMP)")] -[assembly: AssemblyCopyright("Copyright (c) 2018 - 2023")] +[assembly: AssemblyCopyright("Copyright (c) 2018 - 2024")] [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] diff --git a/Tests.Common/TestDatabases.txt b/Tests.Common/TestDatabases.txt index bc2a95b27d..495b65fb84 100644 --- a/Tests.Common/TestDatabases.txt +++ b/Tests.Common/TestDatabases.txt @@ -22,4 +22,4 @@ Oracle: #MySqlLowPrivilegeUsername: minion #MySqlLowPrivilegePassword: minionPass #OracleLowPrivilegeUsername: minion -#OracleLowPrivilegePassword: minionPass +#OracleLowPrivilegePassword: minionPass \ No newline at end of file diff --git a/Tools/rdmp/Databases.yaml b/Tools/rdmp/Databases.yaml index c688f62761..434fc12411 100644 --- a/Tools/rdmp/Databases.yaml +++ b/Tools/rdmp/Databases.yaml @@ -1,2 +1,2 @@ -CatalogueConnectionString: Server=(localdb)\MSSQLLocalDB;Database=RDMP_Catalogue;Trusted_Connection=True;TrustServerCertificate=true; -DataExportConnectionString: Server=(localdb)\MSSQLLocalDB;Database=RDMP_DataExport;Trusted_Connection=True;TrustServerCertificate=true; +CatalogueConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_Catalogue;Trusted_Connection=True;TrustServerCertificate=true; +DataExportConnectionString: Server=(localdb)\MSSQLLocalDB;Database=TEST_DataExport;Trusted_Connection=True;TrustServerCertificate=true; diff --git a/rdmp-client.xml b/rdmp-client.xml index 683046fb35..38e14e0382 100644 --- a/rdmp-client.xml +++ b/rdmp-client.xml @@ -4,4 +4,4 @@ https://github.com/HicServices/RDMP/releases/download/v8.1.4-rc1/rdmp-8.1.4-rc1-client.zip https://github.com/HicServices/RDMP/blob/main/CHANGELOG.md#7 true - + \ No newline at end of file From 1e5a795db5b26ecba9349c1ee6ded34ff732b349 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 5 Feb 2024 15:30:42 +0000 Subject: [PATCH 19/27] Task/rdmp 137 MDF attacher override (#1729) * working path only mdf attacher * reduce code reuse * fix null check of string * fix path * add some more info * add documentation * minor tidyups * improved directory assuming * fix bad copy * update docs * Fix botched git merge text --------- Co-authored-by: James A Sutherland <> Co-authored-by: James A Sutherland --- CHANGELOG.md | 1 + Documentation/DataLoadEngine/MDFAttacher.md | 21 ++++++++ .../DataLoad/Modules/Attachers/MDFAttacher.cs | 53 ++++++++++++++++--- .../Attachers/MdfFileAttachLocations.cs | 34 ++++++------ 4 files changed, 86 insertions(+), 23 deletions(-) create mode 100644 Documentation/DataLoadEngine/MDFAttacher.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 37a2e52292..33a8aaedec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Changed +- Allow arbitrary MDF files from foreign file systems to work with the MDF Attacher, see [MDFAttacher](Documentation\DataLoadEngine\MDFAttacher.md) - Update Excel Attacher to read data from arbitrary start points within sheets - Add Time based filtering of remote table and database attachers diff --git a/Documentation/DataLoadEngine/MDFAttacher.md b/Documentation/DataLoadEngine/MDFAttacher.md new file mode 100644 index 0000000000..baae08d6e7 --- /dev/null +++ b/Documentation/DataLoadEngine/MDFAttacher.md @@ -0,0 +1,21 @@ +# MDF Attacher + +The MDF attacher is used for loading a detached database file into RAW. +This attacher does not load RAW tables normally (like AnySeparatorFileAttacher etc) instead it specifies that it is itself going to act as RAW. +Using this component requires that the computer running the data load has file system access to the RAW SQL Server data directory (and that the MDF and LDF files exist in the same directory). + +## Attach Strategies +The MDF Attacher offers two attach strategies +### Attach with Connection String +AttachWithConnectionString attempts to do the attaching as part of connection by specifying the AttachDBFilename keyword in the connection string +### Execute Create Database For Attach SQL + ExecuteCreateDatabaseForAttachSql attempts to connect to 'master' and execute CREATE DATABASE SQL with the FILENAME property set to your MDF file in the DATA directory of your database server +## Attaching an MDF from another File system type +You might want to load an MDF file from a Linux system into a windows installation of RDMP. +You will run into issues with the MDF file looking for a location like '/var/opt/mssql/data/my_db.mdf' while the file is on your file system at 'C:\Users\me\my_db.mdf'. +In order to be able to use this MDF, you will need to +* Use the 'ExecuteCreateDatabaseForAttachSql' attach strategy +* Specify an 'Override Attach MDF Path' - this can be an absolute path to your MDF file on your file system, or to the directory, assuming the MDF filename has not changed +* Specify an 'Override Attach LDF Path' - this can be an absolute path to your LDF file on your file system, or to the directory, assuming the LDF filename has not changed + +If using the directory based overrides, it is assumed that both the MDF and LDF files are in the same directory. \ No newline at end of file diff --git a/Rdmp.Core/DataLoad/Modules/Attachers/MDFAttacher.cs b/Rdmp.Core/DataLoad/Modules/Attachers/MDFAttacher.cs index 5053b3f7eb..494ae9a607 100644 --- a/Rdmp.Core/DataLoad/Modules/Attachers/MDFAttacher.cs +++ b/Rdmp.Core/DataLoad/Modules/Attachers/MDFAttacher.cs @@ -5,13 +5,17 @@ // You should have received a copy of the GNU General Public License along with RDMP. If not, see . using System; +using System.Data; +using System.Diagnostics; using System.IO; using System.Linq; +using MathNet.Numerics.Distributions; using Microsoft.Data.SqlClient; using Rdmp.Core.Curation.Data; using Rdmp.Core.DataFlowPipeline; using Rdmp.Core.DataLoad.Engine.Attachers; using Rdmp.Core.DataLoad.Engine.Job; +using Rdmp.Core.ReusableLibraryCode; using Rdmp.Core.ReusableLibraryCode.Checks; using Rdmp.Core.ReusableLibraryCode.Progress; @@ -41,7 +45,8 @@ INNER JOIN sys.[databases] d [DemandsInitialization( @"There are multiple ways to attach a mdf files to an SQL server, the first stage is always to copy the mdf and ldf files to the DATA directory of your server but after that it gets flexible. 1. AttachWithConnectionString attempts to do the attaching as part of connection by specifying the AttachDBFilename keyword in the connection string -2. ExecuteCreateDatabaseForAttachSql attempts to connect to 'master' and execute CREATE DATABASE sql with the FILENAME property set to your mdf file in the DATA directory of your database server")] +2. ExecuteCreateDatabaseForAttachSql attempts to connect to 'master' and execute CREATE DATABASE sql with the FILENAME property set to your mdf file in the DATA directory of your database server +If you are attempting to attach an MDF file from a Linux machine to a Window machine, or vice-versa, you will have to use the ExecuteCreateDatabaseForAttachSql to be able to handle the mismatched directory structure")] public MdfAttachStrategy AttachStrategy { get; set; } [DemandsInitialization( @@ -58,6 +63,46 @@ public MDFAttacher() : base(false) private MdfFileAttachLocations _locations; + private void GetFileNames() + { + if ((string.IsNullOrWhiteSpace(OverrideAttachLdfPath) || OverrideAttachLdfPath.EndsWith(".ldf")) && (string.IsNullOrWhiteSpace(OverrideAttachMdfPath) || OverrideAttachMdfPath.EndsWith(".mdf"))) return;//don't need to fiddle with the paths + var builder = new SqlConnectionStringBuilder(_dbInfo.Server.Builder.ConnectionString) + { + InitialCatalog = "master", + ConnectTimeout = 600 + }; + + using var con = new SqlConnection(builder.ConnectionString); + con.Open(); + using var dt = new DataTable(); + using (var cmd = DatabaseCommandHelper.GetCommand($"DBCC CHECKPRIMARYFILE (N'{_locations.AttachMdfPath}' , 3)", con)) + using (var da = DatabaseCommandHelper.GetDataAdapter(cmd)) + { + dt.BeginLoadData(); + da.Fill(dt); + dt.EndLoadData(); + } + + if (!string.IsNullOrWhiteSpace(OverrideAttachLdfPath) && !OverrideAttachLdfPath.EndsWith(".ldf", StringComparison.OrdinalIgnoreCase)) + { + var _path = dt.Rows[1].ItemArray[3].ToString(); + _locations.AttachLdfPath = MdfFileAttachLocations.MergeDirectoryAndFileUsingAssumedDirectorySeparator(OverrideAttachLdfPath, _path); + } + else + { + _locations.AttachLdfPath = OverrideAttachLdfPath; + } + if (!string.IsNullOrWhiteSpace(OverrideAttachMdfPath) && !OverrideAttachMdfPath.EndsWith(".mdf", StringComparison.OrdinalIgnoreCase)) + { + var _path = dt.Rows[0].ItemArray[3].ToString(); + _locations.AttachMdfPath = MdfFileAttachLocations.MergeDirectoryAndFileUsingAssumedDirectorySeparator(OverrideAttachMdfPath, _path); + } + else + { + _locations.AttachMdfPath = OverrideAttachMdfPath; + } + } + public override ExitCodeType Attach(IDataLoadJob job, GracefulCancellationToken cancellationToken) { //The location of .mdf files from the perspective of the database server @@ -66,11 +111,7 @@ public override ExitCodeType Attach(IDataLoadJob job, GracefulCancellationToken _locations = new MdfFileAttachLocations(LoadDirectory.ForLoading, databaseDirectory, OverrideMDFFileCopyDestination); - if (!string.IsNullOrWhiteSpace(OverrideAttachMdfPath)) - _locations.AttachMdfPath = OverrideAttachMdfPath; - - if (!string.IsNullOrWhiteSpace(OverrideAttachLdfPath)) - _locations.AttachLdfPath = OverrideAttachLdfPath; + GetFileNames(); job.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, $"Identified the MDF file:{_locations.OriginLocationMdf} and corresponding LDF file:{_locations.OriginLocationLdf}")); diff --git a/Rdmp.Core/DataLoad/Modules/Attachers/MdfFileAttachLocations.cs b/Rdmp.Core/DataLoad/Modules/Attachers/MdfFileAttachLocations.cs index e8a72d3550..a282cfe83d 100644 --- a/Rdmp.Core/DataLoad/Modules/Attachers/MdfFileAttachLocations.cs +++ b/Rdmp.Core/DataLoad/Modules/Attachers/MdfFileAttachLocations.cs @@ -7,6 +7,7 @@ using System; using System.IO; using System.Linq; +using MathNet.Numerics.Statistics; using Rdmp.Core.DataLoad.Modules.Exceptions; namespace Rdmp.Core.DataLoad.Modules.Attachers; @@ -48,23 +49,8 @@ public MdfFileAttachLocations(DirectoryInfo originDirectory, CopyToMdf = Path.Combine(copyToDirectory, Path.GetFileName(OriginLocationMdf)); CopyToLdf = Path.Combine(copyToDirectory, Path.GetFileName(OriginLocationLdf)); - - if (databaseDirectoryFromPerspectiveOfDatabaseServer.Contains('/')) - { - // Unix-style paths - AttachMdfPath = - $"{databaseDirectoryFromPerspectiveOfDatabaseServer.TrimEnd('/')}/{Path.GetFileName(OriginLocationMdf)}"; - AttachLdfPath = - $"{databaseDirectoryFromPerspectiveOfDatabaseServer.TrimEnd('/')}/{Path.GetFileName(OriginLocationLdf)}"; - } - else - { - // DOS-style paths - AttachMdfPath = - $"{databaseDirectoryFromPerspectiveOfDatabaseServer.TrimEnd('\\')}\\{Path.GetFileName(OriginLocationMdf)}"; - AttachLdfPath = - $"{databaseDirectoryFromPerspectiveOfDatabaseServer.TrimEnd('\\')}\\{Path.GetFileName(OriginLocationLdf)}"; - } + AttachMdfPath = MergeDirectoryAndFileUsingAssumedDirectorySeparator(databaseDirectoryFromPerspectiveOfDatabaseServer, OriginLocationMdf); + AttachLdfPath = MergeDirectoryAndFileUsingAssumedDirectorySeparator(databaseDirectoryFromPerspectiveOfDatabaseServer, OriginLocationLdf); } public string OriginLocationMdf { get; set; } @@ -75,4 +61,18 @@ public MdfFileAttachLocations(DirectoryInfo originDirectory, public string AttachMdfPath { get; set; } public string AttachLdfPath { get; set; } + + + private static char GetCorrectDirectorySeparatorCharBasedOnString(string partialPath) + { + bool containUnixStyle = partialPath.Contains('/'); + bool containsNTFSStyle = partialPath.Contains('\\'); + if (containUnixStyle && containsNTFSStyle) throw new Exception("Override partial path contains both '/' and '\\', unable to correctly guess which file system is in use "); + return containUnixStyle ? '/' : '\\'; + } + + public static string MergeDirectoryAndFileUsingAssumedDirectorySeparator(string directory, string file) { + var directorySeparator = GetCorrectDirectorySeparatorCharBasedOnString(directory); + return directory.TrimEnd(directorySeparator) + directorySeparator + Path.GetFileName(file); + } } \ No newline at end of file From 903c93a0d0021ace2c63e10199b62843462e2022 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 7 Feb 2024 08:15:18 +0000 Subject: [PATCH 20/27] tidy up imports --- .../ExecuteCommandChangeExtractionCategoryTests.cs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandChangeExtractionCategoryTests.cs b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandChangeExtractionCategoryTests.cs index 6abab54986..fa341fcc6b 100644 --- a/Rdmp.Core.Tests/CommandExecution/ExecuteCommandChangeExtractionCategoryTests.cs +++ b/Rdmp.Core.Tests/CommandExecution/ExecuteCommandChangeExtractionCategoryTests.cs @@ -8,14 +8,7 @@ using Rdmp.Core.CommandExecution; using Rdmp.Core.CommandExecution.AtomicCommands; using Rdmp.Core.Curation.Data; -using Rdmp.Core.Repositories; -using System.Linq; using Tests.Common; -using NSubstitute; -using NSubstitute.Extensions; -using Rdmp.Core.DataExport.Data; -using Rdmp.Core.Curation.Data.Aggregation; -using static Org.BouncyCastle.Math.EC.ECCurve; namespace Rdmp.Core.Tests.CommandExecution; public class ExecuteCommandChangeExtractionCategoryTests : DatabaseTests From 0a4e8fe97901edd494f7770f351267c808e8e141 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Feb 2024 01:04:57 +0000 Subject: [PATCH 21/27] Bump NunitXml.TestLogger from 3.1.15 to 3.1.20 Bumps [NunitXml.TestLogger](https://github.com/spekt/nunit.testlogger) from 3.1.15 to 3.1.20. - [Release notes](https://github.com/spekt/nunit.testlogger/releases) - [Changelog](https://github.com/spekt/nunit.testlogger/blob/master/CHANGELOG.md) - [Commits](https://github.com/spekt/nunit.testlogger/compare/v3.1.15...v3.1.20) --- updated-dependencies: - dependency-name: NunitXml.TestLogger dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- Rdmp.Core.Tests/Rdmp.Core.Tests.csproj | 2 +- Rdmp.UI.Tests/Rdmp.UI.Tests.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj b/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj index 457640b807..f7cb48e6d2 100644 --- a/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj +++ b/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj @@ -80,7 +80,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Rdmp.UI.Tests/Rdmp.UI.Tests.csproj b/Rdmp.UI.Tests/Rdmp.UI.Tests.csproj index 45425c90ab..2020c1da5f 100644 --- a/Rdmp.UI.Tests/Rdmp.UI.Tests.csproj +++ b/Rdmp.UI.Tests/Rdmp.UI.Tests.csproj @@ -27,7 +27,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + From abcff70b0218fdeae2decd6b78f4bebd24eed8c5 Mon Sep 17 00:00:00 2001 From: tznind Date: Sat, 10 Feb 2024 10:05:20 +0000 Subject: [PATCH 22/27] Fix vanishing menu --- Tools/rdmp/CommandLine/Gui/ConsoleMainWindow.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Tools/rdmp/CommandLine/Gui/ConsoleMainWindow.cs b/Tools/rdmp/CommandLine/Gui/ConsoleMainWindow.cs index 5ea15d9d70..521b7aa156 100644 --- a/Tools/rdmp/CommandLine/Gui/ConsoleMainWindow.cs +++ b/Tools/rdmp/CommandLine/Gui/ConsoleMainWindow.cs @@ -157,12 +157,13 @@ internal void SetUp(Toplevel top) Application.RootMouseEvent = OnRootMouseEvent; + + + _treeView.ObjectActivationButton = _rightClick; - _treeView.ObjectActivated += _treeView_ObjectActivated; _treeView.KeyPress += treeView_KeyPress; _treeView.SelectionChanged += _treeView_SelectionChanged; _treeView.AspectGetter = AspectGetter; - var statusBar = new StatusBar(new StatusItem[] { new(Key.Q | Key.CtrlMask, "~^Q~ Quit", () => Quit()), @@ -363,6 +364,10 @@ private void treeView_KeyPress(View.KeyEventEventArgs obj) { switch (obj.KeyEvent.Key) { + case Key.Enter: + _treeView_ObjectActivated(null); + obj.Handled = true; + break; case Key.DeleteChar: var many = _treeView.GetAllSelectedObjects().ToArray(); obj.Handled = true; From ea6db8eb8cce1a1ee8c839a229efb49108201236 Mon Sep 17 00:00:00 2001 From: JBaird00183 <109660435+JBaird00183@users.noreply.github.com> Date: Tue, 13 Feb 2024 08:39:36 +0000 Subject: [PATCH 23/27] Bug fix/rdmp 135 timeout variable not being passed (#1744) * Changes made in CohortIdentificationConfigurationUI to pass user defined Timeout variable * More timeout changes * tidy up code --------- Co-authored-by: James Friel --- ...hortIdentificationConfigurationUICommon.cs | 23 ++++++++++--------- .../CohortIdentificationConfigurationUI.cs | 4 ++-- ...eGuiCohortIdentificationConfigurationUI.cs | 4 ++-- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/Rdmp.Core/CohortCreation/CohortIdentificationConfigurationUICommon.cs b/Rdmp.Core/CohortCreation/CohortIdentificationConfigurationUICommon.cs index 587de38d50..ebcef5d8a7 100644 --- a/Rdmp.Core/CohortCreation/CohortIdentificationConfigurationUICommon.cs +++ b/Rdmp.Core/CohortCreation/CohortIdentificationConfigurationUICommon.cs @@ -1,4 +1,4 @@ -// Copyright (c) The University of Dundee 2018-2019 +// Copyright (c) The University of Dundee 2018-2024 // This file is part of the Research Data Management Platform (RDMP). // RDMP is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. @@ -157,12 +157,12 @@ public void SetShowCumulativeTotals(bool show) RecreateAllTasks(); } - private void OrderActivity(Operation operation, IMapsDirectlyToDatabaseTable o) + private void OrderActivity(Operation operation, IMapsDirectlyToDatabaseTable o, int? userDefinedTimeout) { switch (operation) { case Operation.Execute: - StartThisTaskOnly(o); + StartThisTaskOnly(o, userDefinedTimeout); break; case Operation.Cancel: Cancel(o); @@ -177,7 +177,7 @@ private void OrderActivity(Operation operation, IMapsDirectlyToDatabaseTable o) } } - private void StartThisTaskOnly(IMapsDirectlyToDatabaseTable configOrContainer) + private void StartThisTaskOnly(IMapsDirectlyToDatabaseTable configOrContainer, int? userDefinedTimeout) { var task = Compiler.AddTask(configOrContainer, _globals); if (task.State == CompilationState.Crashed) @@ -194,7 +194,7 @@ private void StartThisTaskOnly(IMapsDirectlyToDatabaseTable configOrContainer) task = Compiler.AddTask(configOrContainer, _globals); //Task is now in state NotScheduled, so we can start it - Compiler.LaunchSingleTask(task, Timeout, true); + Compiler.LaunchSingleTask(task, userDefinedTimeout ?? Timeout, true); } public void Cancel(IMapsDirectlyToDatabaseTable o) @@ -328,7 +328,8 @@ public bool ConsultAboutClosing() /// the appropriate message to the user /// /// - public void ExecuteOrCancel(object o) + /// + public void ExecuteOrCancel(object o, int? userDefinedTimeout) { Task.Run(() => { @@ -339,19 +340,19 @@ public void ExecuteOrCancel(object o) var joinable = aggregate.JoinableCohortAggregateConfiguration; if (joinable != null) - OrderActivity(GetNextOperation(GetState(joinable)), joinable); + OrderActivity(GetNextOperation(GetState(joinable)), joinable, userDefinedTimeout); else - OrderActivity(GetNextOperation(GetState(aggregate)), aggregate); + OrderActivity(GetNextOperation(GetState(aggregate)), aggregate, userDefinedTimeout); break; } case CohortAggregateContainer container: - OrderActivity(GetNextOperation(GetState(container)), container); + OrderActivity(GetNextOperation(GetState(container)), container, userDefinedTimeout); break; } }); } - public void StartAll(Action afterDelegate, EventHandler onRunnerPhaseChanged) + public void StartAll(Action afterDelegate, EventHandler onRunnerPhaseChanged, int? userDefinedTimeout) { //only allow starting all if we are not mid execution already if (IsExecutingGlobalOperations()) @@ -360,7 +361,7 @@ public void StartAll(Action afterDelegate, EventHandler onRunnerPhaseChanged) _cancelGlobalOperations = new CancellationTokenSource(); - Runner = new CohortCompilerRunner(Compiler, Timeout); + Runner = new CohortCompilerRunner(Compiler, userDefinedTimeout ?? Timeout); Runner.PhaseChanged += onRunnerPhaseChanged; Task.Run(() => { diff --git a/Rdmp.UI/SubComponents/CohortIdentificationConfigurationUI.cs b/Rdmp.UI/SubComponents/CohortIdentificationConfigurationUI.cs index 8a7ae3fde4..975fae2fd1 100644 --- a/Rdmp.UI/SubComponents/CohortIdentificationConfigurationUI.cs +++ b/Rdmp.UI/SubComponents/CohortIdentificationConfigurationUI.cs @@ -286,14 +286,14 @@ public override void ConsultAboutClosing(object sender, FormClosingEventArgs e) private void tlvCic_ButtonClick(object sender, CellClickEventArgs e) { - Common.ExecuteOrCancel(e.Model); + Common.ExecuteOrCancel(e.Model, _timeoutControls.Timeout); } public void StartAll() { lblExecuteAllPhase.Enabled = true; - Common.StartAll(RebuildClearCacheCommand, RunnerOnPhaseChanged); + Common.StartAll(RebuildClearCacheCommand, RunnerOnPhaseChanged, _timeoutControls.Timeout); } private void RunnerOnPhaseChanged(object sender, EventArgs eventArgs) diff --git a/Tools/rdmp/CommandLine/Gui/ConsoleGuiCohortIdentificationConfigurationUI.cs b/Tools/rdmp/CommandLine/Gui/ConsoleGuiCohortIdentificationConfigurationUI.cs index 0ab327b9e5..d25ef2f160 100644 --- a/Tools/rdmp/CommandLine/Gui/ConsoleGuiCohortIdentificationConfigurationUI.cs +++ b/Tools/rdmp/CommandLine/Gui/ConsoleGuiCohortIdentificationConfigurationUI.cs @@ -58,7 +58,7 @@ public ConsoleGuiCohortIdentificationConfigurationUI(IBasicActivateItems activat if (int.TryParse(tbTimeout.Text.ToString(), out var t)) Common.Timeout = t; }; - btnRun.Clicked += () => { Common.StartAll(() => { }, RunnerOnPhaseChanged); }; + btnRun.Clicked += () => { Common.StartAll(() => { }, RunnerOnPhaseChanged, Common.Timeout); }; btnClose.Clicked += () => { if (!Common.ConsultAboutClosing()) @@ -171,7 +171,7 @@ private void Tableview1_CellActivated(TableView.CellActivatedEventArgs obj) } } - if (col.ColumnName.Equals("Execute")) Common.ExecuteOrCancel(o); + if (col.ColumnName.Equals("Execute")) Common.ExecuteOrCancel(o, Common.Timeout); } private bool IsValidSelection(int col, int row) From ef2eddbe04cf3cb52915ac8d8b556f3a497dfd84 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 13 Feb 2024 09:41:10 +0000 Subject: [PATCH 24/27] Bump Microsoft.NET.Test.Sdk from 17.8.0 to 17.9.0 (#1743) Bumps [Microsoft.NET.Test.Sdk](https://github.com/microsoft/vstest) from 17.8.0 to 17.9.0. - [Release notes](https://github.com/microsoft/vstest/releases) - [Changelog](https://github.com/microsoft/vstest/blob/main/docs/releases.md) - [Commits](https://github.com/microsoft/vstest/compare/v17.8.0...v17.9.0) --- updated-dependencies: - dependency-name: Microsoft.NET.Test.Sdk dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: James Friel --- Rdmp.Core.Tests/Rdmp.Core.Tests.csproj | 2 +- Rdmp.UI.Tests/Rdmp.UI.Tests.csproj | 2 +- Tests.Common/Tests.Common.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj b/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj index f7cb48e6d2..84c8df6f7d 100644 --- a/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj +++ b/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj @@ -69,7 +69,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Rdmp.UI.Tests/Rdmp.UI.Tests.csproj b/Rdmp.UI.Tests/Rdmp.UI.Tests.csproj index 2020c1da5f..e2cd176a97 100644 --- a/Rdmp.UI.Tests/Rdmp.UI.Tests.csproj +++ b/Rdmp.UI.Tests/Rdmp.UI.Tests.csproj @@ -16,7 +16,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Tests.Common/Tests.Common.csproj b/Tests.Common/Tests.Common.csproj index e657ff479e..1bdf4aaeb9 100644 --- a/Tests.Common/Tests.Common.csproj +++ b/Tests.Common/Tests.Common.csproj @@ -41,7 +41,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + \ No newline at end of file From cc2b39db5c60b52fcab1a7349025ef7df6a58486 Mon Sep 17 00:00:00 2001 From: James Friel Date: Wed, 14 Feb 2024 09:54:43 +0000 Subject: [PATCH 25/27] bump to 8.1.4 (#1749) --- CHANGELOG.md | 2 +- SharedAssemblyInfo.cs | 2 +- rdmp-client.xml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dff50145c1..d07c4d6db9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [8.1.4] - Unreleased +## [8.1.4] - 2024-02-19 ## Changed diff --git a/SharedAssemblyInfo.cs b/SharedAssemblyInfo.cs index 4345b6c353..f1858b55a7 100644 --- a/SharedAssemblyInfo.cs +++ b/SharedAssemblyInfo.cs @@ -12,4 +12,4 @@ [assembly: AssemblyVersion("8.1.4")] [assembly: AssemblyFileVersion("8.1.4")] -[assembly: AssemblyInformationalVersion("8.1.4-rc1")] \ No newline at end of file +[assembly: AssemblyInformationalVersion("8.1.4")] \ No newline at end of file diff --git a/rdmp-client.xml b/rdmp-client.xml index 38e14e0382..9ead7676a0 100644 --- a/rdmp-client.xml +++ b/rdmp-client.xml @@ -1,7 +1,7 @@ - 8.1.3.1 - https://github.com/HicServices/RDMP/releases/download/v8.1.4-rc1/rdmp-8.1.4-rc1-client.zip + 8.1.4.0 + https://github.com/HicServices/RDMP/releases/download/v8.1.4/rdmp-8.1.4-client.zip https://github.com/HicServices/RDMP/blob/main/CHANGELOG.md#7 true \ No newline at end of file From c496cd420445086cc416069930e8212ecb75c6f1 Mon Sep 17 00:00:00 2001 From: James Friel Date: Thu, 15 Feb 2024 12:19:48 +0000 Subject: [PATCH 26/27] fix excel attacher bug --- Rdmp.Core/DataLoad/Modules/Attachers/ExcelAttacher.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Rdmp.Core/DataLoad/Modules/Attachers/ExcelAttacher.cs b/Rdmp.Core/DataLoad/Modules/Attachers/ExcelAttacher.cs index b3d3dea382..c41a96df9b 100644 --- a/Rdmp.Core/DataLoad/Modules/Attachers/ExcelAttacher.cs +++ b/Rdmp.Core/DataLoad/Modules/Attachers/ExcelAttacher.cs @@ -55,6 +55,7 @@ public class ExcelAttacher : FlatFileAttacher private int ConvertColumnOffsetToInt() { + if (ColumnOffset is null) return 0; if(int.TryParse(ColumnOffset,out var result)) return result; if (ColumnOffset.Length == 1 && char.IsLetter(ColumnOffset[0])) return char.ToUpper(ColumnOffset[0]) - 65;//would be 64, but we index from zero here throw new Exception("Column offset is not a valid number or letter"); From 36ca4985e9dc3bafac2580c9e33e4659b044b526 Mon Sep 17 00:00:00 2001 From: James Friel Date: Fri, 16 Feb 2024 17:05:42 +0000 Subject: [PATCH 27/27] Task/rdmp-140 Migrate to centrally managed dependancies (#1753) * centrally managed dependacies * add directory.project.props file --- .../ResearchDataManagementPlatform.csproj | 10 +- Directory.Packages.props | 47 ++ HIC.DataManagementPlatform.sln | 1 + Rdmp.Core.Tests/Rdmp.Core.Tests.csproj | 14 +- .../PackageListIsCorrectTests.cs | 11 +- Rdmp.Core/Rdmp.Core.csproj | 764 +++++++++--------- Rdmp.UI.Tests/Rdmp.UI.Tests.csproj | 16 +- Rdmp.UI/Rdmp.UI.csproj | 14 +- Tests.Common/Tests.Common.csproj | 8 +- Tools/rdmp/rdmp.csproj | 98 ++- 10 files changed, 516 insertions(+), 467 deletions(-) create mode 100644 Directory.Packages.props diff --git a/Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj b/Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj index e78dd26e59..034b5c9746 100644 --- a/Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj +++ b/Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj @@ -1,4 +1,4 @@ - + {550988FD-F1FA-41D8-BE0F-00B4DE47D320} WinExe @@ -29,10 +29,10 @@ - - - - + + + + diff --git a/Directory.Packages.props b/Directory.Packages.props new file mode 100644 index 0000000000..b40ebb9954 --- /dev/null +++ b/Directory.Packages.props @@ -0,0 +1,47 @@ + + + true + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/HIC.DataManagementPlatform.sln b/HIC.DataManagementPlatform.sln index 9601ae9ef9..c088c8a388 100644 --- a/HIC.DataManagementPlatform.sln +++ b/HIC.DataManagementPlatform.sln @@ -16,6 +16,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution deadlinksconfig.json = deadlinksconfig.json directory.build.props = directory.build.props .github\workflows\links.yml = .github\workflows\links.yml + Directory.Packages.props = Directory.Packages.props NoteForNewDevelopers.md = NoteForNewDevelopers.md Documentation\CodeTutorials\Packages.md = Documentation\CodeTutorials\Packages.md rdmp-client.xml = rdmp-client.xml diff --git a/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj b/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj index 84c8df6f7d..6917c59ac6 100644 --- a/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj +++ b/Rdmp.Core.Tests/Rdmp.Core.Tests.csproj @@ -65,22 +65,22 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - - + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Rdmp.Core.Tests/ReusableCodeTests/PackageListIsCorrectTests.cs b/Rdmp.Core.Tests/ReusableCodeTests/PackageListIsCorrectTests.cs index 882630995a..ffa4e0968b 100644 --- a/Rdmp.Core.Tests/ReusableCodeTests/PackageListIsCorrectTests.cs +++ b/Rdmp.Core.Tests/ReusableCodeTests/PackageListIsCorrectTests.cs @@ -24,10 +24,11 @@ public class PackageListIsCorrectTests { RecurseSubdirectories = true, MatchCasing = MatchCasing.CaseInsensitive, IgnoreInaccessible = true }; // - private static readonly Regex RPackageRef = - new(@" RPackageRef.Matches(s)) - .Select(m => m.Groups[1].Value).ToHashSet(StringComparer.InvariantCultureIgnoreCase); + var usedPackages = GetCsprojFiles(root).Select(File.ReadAllText) + .SelectMany(s => RPackageRefNoVersion.Matches(s)) + .Select(m => m.Groups[1].Value) + .ToHashSet(StringComparer.InvariantCultureIgnoreCase); // Then subtract those listed in PACKAGES.md (should be empty) var undocumentedPackages = usedPackages.Except(packagesMarkdown).Select(BuildRecommendedMarkdownLine).ToList(); diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index 00edd93938..d1273bb7bd 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -1,384 +1,384 @@  - - HIC.RDMP.Plugin - $(version) - HIC.RDMP.Plugin - Health Informatics Centre, University of Dundee - Health Informatics Centre, University of Dundee - https://raw.githubusercontent.com/HicServices/RDMP/master/LICENSE - https://github.com/HicServices/RDMP - - https://raw.githubusercontent.com/HicServices/RDMP/master/Application/ResearchDataManagementPlatform/Icon/main.png - - false - GPL-3.0-or-later - Core package for plugin development - Copyright 2018-2019 - net7.0 - false - true - net7.0 - true - 1701;1702;CS1591;SCS0018 - embedded - true - $(NoWarn);NU5104 - - - Rdmp.Core.DataLoad.Modules.DataProvider.WebServiceConfiguration - - - true - latest - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Never - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - - - - - - - - - - - Never - - - - - - - - - True - True - GlobalStrings.resx - - - True - True - ChecksAndProgressIcons.resx - - - True - True - Images.resx - - - DatabaseProviderIcons.resx - True - True - - - - - PublicResXFileCodeGenerator - GlobalStrings.Designer.cs - - - PublicResXFileCodeGenerator - - - - - - PublicResXFileCodeGenerator - ChecksAndProgressIcons.Designer.cs - - - Designer - PublicResXFileCodeGenerator - Images.Designer.cs - - - Designer - DatabaseProviderIcons.Designer.cs - PublicResXFileCodeGenerator - - + + HIC.RDMP.Plugin + $(version) + HIC.RDMP.Plugin + Health Informatics Centre, University of Dundee + Health Informatics Centre, University of Dundee + https://raw.githubusercontent.com/HicServices/RDMP/master/LICENSE + https://github.com/HicServices/RDMP + + https://raw.githubusercontent.com/HicServices/RDMP/master/Application/ResearchDataManagementPlatform/Icon/main.png + + false + GPL-3.0-or-later + Core package for plugin development + Copyright 2018-2019 + net7.0 + false + true + net7.0 + true + 1701;1702;CS1591;SCS0018 + embedded + true + $(NoWarn);NU5104 + + + Rdmp.Core.DataLoad.Modules.DataProvider.WebServiceConfiguration + + + true + latest + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Never + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + + + + + Never + + + + + + + + + True + True + GlobalStrings.resx + + + True + True + ChecksAndProgressIcons.resx + + + True + True + Images.resx + + + DatabaseProviderIcons.resx + True + True + + + + + PublicResXFileCodeGenerator + GlobalStrings.Designer.cs + + + PublicResXFileCodeGenerator + + + + + + PublicResXFileCodeGenerator + ChecksAndProgressIcons.Designer.cs + + + Designer + PublicResXFileCodeGenerator + Images.Designer.cs + + + Designer + DatabaseProviderIcons.Designer.cs + PublicResXFileCodeGenerator + + \ No newline at end of file diff --git a/Rdmp.UI.Tests/Rdmp.UI.Tests.csproj b/Rdmp.UI.Tests/Rdmp.UI.Tests.csproj index e2cd176a97..4df2463615 100644 --- a/Rdmp.UI.Tests/Rdmp.UI.Tests.csproj +++ b/Rdmp.UI.Tests/Rdmp.UI.Tests.csproj @@ -1,4 +1,4 @@ - + net7.0-windows false @@ -12,22 +12,22 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - - + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Rdmp.UI/Rdmp.UI.csproj b/Rdmp.UI/Rdmp.UI.csproj index 3b3f0493ab..66fb7e55de 100644 --- a/Rdmp.UI/Rdmp.UI.csproj +++ b/Rdmp.UI/Rdmp.UI.csproj @@ -42,16 +42,16 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - + + + + + diff --git a/Tests.Common/Tests.Common.csproj b/Tests.Common/Tests.Common.csproj index 1bdf4aaeb9..b27732f98c 100644 --- a/Tests.Common/Tests.Common.csproj +++ b/Tests.Common/Tests.Common.csproj @@ -36,12 +36,12 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + \ No newline at end of file diff --git a/Tools/rdmp/rdmp.csproj b/Tools/rdmp/rdmp.csproj index a7d23a30e5..c0025723ce 100644 --- a/Tools/rdmp/rdmp.csproj +++ b/Tools/rdmp/rdmp.csproj @@ -1,52 +1,50 @@ - - {A6107DDC-8268-4902-A994-233B00480113} - Exe - net7.0 - true - false - rdmp - rdmp - Copyright © 2019 - - Rdmp.Core - embedded - true - true - true - net7.0 - - - true - - - - - - - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - - - - - - - - - - - + + {A6107DDC-8268-4902-A994-233B00480113} + Exe + net7.0 + true + false + rdmp + rdmp + Copyright © 2019 + + Rdmp.Core + embedded + true + true + true + net7.0 + + + true + + + + + + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + + + + + + + + +