Merge branch 'develop' of https://github.com/HicServices/RDMP

.github/workflows/build.yml

@@ -204,7 +204,7 @@ jobs:
 
 
       - name: Install Perl dependencies
-        uses: shogo82148/actions-setup-perl@v1.27.0
+        uses: shogo82148/actions-setup-perl@v1.28.0
         with:
           install-modules-with: cpanm
           install-modules: Archive::Zip Archive::Tar
@@ -264,7 +264,7 @@ jobs:
             dist
       - name: Cache Build
         id: cache-build
-        uses: actions/cache/save@v3
+        uses: actions/cache/save@v4
         with:
           path: ${{ github.workspace }}/
           key: ${{ github.sha }}-your-cache-key-bundled
@@ -275,7 +275,7 @@ jobs:
     needs: ['bundle','tests_db','tests_file_system']
     if: contains(github.ref, 'refs/tags/v')
     steps:
-      - uses: actions/cache/restore@v3
+      - uses: actions/cache/restore@v4
         id: restore-build
         with:
           path: ${{ github.workspace }}/

.github/workflows/docker.yml

@@ -21,7 +21,7 @@ jobs:
         with:
           dotnet-version: 6.0.x
       - name: Cache Nuget
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.nuget/packages
           key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj') }}

Application/ResearchDataManagementPlatform/ResearchDataManagementPlatform.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <ProjectGuid>{550988FD-F1FA-41D8-BE0F-00B4DE47D320}</ProjectGuid>
     <OutputType>WinExe</OutputType>
@@ -29,10 +29,10 @@
     <EmbeddedResource Remove="Updates\UpdaterUI.resx" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="ConsoleControl" Version="1.3.0" />
-    <PackageReference Include="Autoupdater.NET.Official" Version="1.8.4" />
-    <PackageReference Include="DockPanelSuite.ThemeVS2015" Version="3.1.0" />
-    <PackageReference Include="System.Resources.Extensions" Version="7.0.0" />
+    <PackageReference Include="ConsoleControl"/>
+    <PackageReference Include="Autoupdater.NET.Official"/>
+    <PackageReference Include="DockPanelSuite.ThemeVS2015" />
+    <PackageReference Include="System.Resources.Extensions" />
   </ItemGroup>
   <ItemGroup>
     <Compile Include="..\..\SharedAssemblyInfo.cs" Link="SharedAssemblyInfo.cs" />

CHANGELOG.md

@@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [8.1.4] - 2024-02-19
+
+## Changed
+
+- Add ability to use Extraction Category with Project Specific Catalogues
+- Allow arbitrary MDF files from foreign file systems to work with the MDF Attacher, see [MDFAttacher](Documentation\DataLoadEngine\MDFAttacher.md)
+- Update Excel Attacher to read data from arbitrary start points within sheets
+- Add Time based filtering of remote table and database attachers
+
 ## [8.1.3] - 2024-01-15
 
 ### Changed

Directory.Packages.props

@@ -0,0 +1,47 @@
+<Project>
+	<PropertyGroup>
+		<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
+	</PropertyGroup>
+	<ItemGroup>
+		<PackageVersion Include="CommandLineParser" Version="2.9.1"/>
+		<PackageVersion Include="CsvHelper" Version="30.0.1"/>
+		<PackageVersion Include="Equ" Version="2.3.0"/>
+		<PackageVersion Include="ExcelNumberFormat" Version="1.1.0"/>
+		<PackageVersion Include="FluentFTP" Version="49.0.1"/>
+		<PackageVersion Include="HIC.BadMedicine" Version="1.1.2"/>
+		<PackageVersion Include="HIC.FAnsiSql" Version="3.1.1"/>
+		<PackageVersion Include="LibArchive.Net" Version="0.1.4"/>
+		<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0">
+			<PrivateAssets>all</PrivateAssets>
+			<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+		</PackageVersion>
+		<PackageVersion Include="Microsoft.XmlSerializer.Generator" Version="8.0.0"/>
+		<PackageVersion Include="MongoDB.Driver" Version="2.23.1"/>
+		<PackageVersion Include="Newtonsoft.Json" Version="13.0.3"/>
+		<PackageVersion Include="NPOI" Version="2.6.2"/>
+		<PackageVersion Include="NLog" Version="5.2.8"/>
+		<PackageVersion Include="SixLabors.ImageSharp" Version="3.1.2"/>
+		<PackageVersion Include="SixLabors.ImageSharp.Drawing" Version="2.1.0"/>
+		<PackageVersion Include="Spectre.Console" Version="0.48.0"/>
+		<PackageVersion Include="SSH.NET" Version="2023.0.1"/>
+		<PackageVersion Include="Terminal.Gui" Version="1.15.1"/>
+		<PackageVersion Include="YamlDotNet" Version="15.1.1"/>
+		<PackageVersion Include="ConsoleControl" Version="1.3.0"/>
+		<PackageVersion Include="Autoupdater.NET.Official" Version="1.8.4"/>
+		<PackageVersion Include="DockPanelSuite.ThemeVS2015" Version="3.1.0"/>
+		<PackageVersion Include="System.Threading.ThreadPool" Version="4.3.0"/>
+		<PackageVersion Include="WeCantSpell.Hunspell" Version="5.0.0"/>
+		<PackageVersion Include="NUnit" Version="4.0.1"/>
+		<PackageVersion Include="ObjectListView.Repack.NET6Plus" Version="2.9.4"/>
+		<PackageVersion Include="Scintilla.NET" Version="5.3.2.9"/>
+		<PackageVersion Include="System.Resources.Extensions" Version="8.0.0"/>
+		<PackageVersion Include="VPKSoft.ScintillaLexers.NET" Version="1.1.16"/>
+		<PackageVersion Include="HIC.System.Windows.Forms.DataVisualization" Version="1.0.1"/>
+		<PackageVersion Include="NunitXml.TestLogger" Version="3.1.20"/>
+		<PackageVersion Include="NUnit3TestAdapter" Version="4.5.0"/>
+		<PackageVersion Include="NUnit.Analyzers" Version="4.0.1"/>
+		<PackageVersion Include="NSubstitute" Version="5.1.0"/>
+		<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.9.0"/>
+		<PackageVersion Include="coverlet.collector" Version="6.0.0"/>
+	</ItemGroup>
+</Project>

Documentation/DataLoadEngine/ExcelAttacher.md

@@ -0,0 +1,28 @@
+## Excel Attacher
+The Excel attacher is a convenient way to load your data from .xlsx files into RDMP.
+It works out of the box with a single-sheet workbook.
+
+### Simple Example
+For example, the following Excel sheet would be transformed into a single database table with columns "COLUMN_1" and "COLUMN_2"
+With a single entry ("some data","some other data").
+
+|   | A         | B               |
+|---|-----------|-----------------|
+| 1 | COLUMN_1  | COLUMN_2        |
+| 2 | some data | some other data |
+
+
+## Configurable options
+* Work Sheet Name - the name of the sheet you wish to import, only required if your Excel workbook has multiple sheets
+* Add Filename Column Named - (Optional) If you want to store the source file location in the database, add the name of a new column to store this information in
+* Force Replacement Headers - A comma separated list of headers to replace those found in your sheet
+* Allow Extra Columns In Target Without Complaining of Column Mismatch - Allows for missing columns in the source file
+* File Pattern - the file you wish to load
+* Table To Load / Table Name - Which Table you wish to populate
+* Send Load Not Required If File Not Found - If there is no file, we can skip the load
+* Delay Load Failures - Wait till load complete to fail about missing files
+* Culture / Explicit Date Time Format - Set the time format  of your data
+* Row Offset - If your data doesn't start on the first row, set the Row Offset to the row that contains your data's headers. The first row is 1.
+* Column Offset - If your data doesn't start on the first column, set the Column Offset to the column that contains your data's headers. The first Column is 'A' or '0'.
+
+

Documentation/DataLoadEngine/MDFAttacher.md

@@ -0,0 +1,21 @@
+# MDF Attacher
+
+The MDF attacher is used for loading a detached database file into RAW.  
+This attacher does not load RAW tables normally (like AnySeparatorFileAttacher etc) instead it specifies that it is itself going to act as RAW.
+Using this component requires that the computer running the data load has file system access to the RAW SQL Server data directory (and that the MDF and LDF files exist in the same directory).
+
+## Attach Strategies
+The MDF Attacher offers two attach strategies
+### Attach with Connection String
+AttachWithConnectionString attempts to do the attaching as part of connection by specifying the AttachDBFilename keyword in the connection string
+### Execute Create  Database For Attach SQL
+ ExecuteCreateDatabaseForAttachSql attempts to connect to 'master' and execute CREATE DATABASE SQL with the FILENAME property set to your MDF file in the DATA directory of your database server
+## Attaching an MDF from another File system type
+You might want to load an MDF file from a Linux system into a windows installation of RDMP.
+You will run into issues with the MDF file looking for a location like '/var/opt/mssql/data/my_db.mdf' while the file is on your file system at 'C:\Users\me\my_db.mdf'.
+In order to be able to use this MDF, you will need to
+* Use the 'ExecuteCreateDatabaseForAttachSql' attach strategy
+* Specify an 'Override Attach MDF Path' - this can be an absolute path to your MDF file on your file system, or to the directory, assuming the MDF filename has not changed
+* Specify an 'Override Attach LDF Path' - this can be an absolute path to your LDF file on your file system, or to the directory, assuming the LDF filename has not changed
+
+If using the directory based overrides, it is assumed that both the MDF and LDF files are in the same directory.