From b462f0a6ad241863de3ac3c86aaa5c2c04c2d176 Mon Sep 17 00:00:00 2001 From: James Friel Date: Mon, 9 Dec 2024 16:00:21 +0000 Subject: [PATCH 01/12] add catalogue overview start --- Rdmp.Core/Curation/Data/CatalogueOverview.cs | 102 +++++ .../Data/CatalogueOverviewDataPoint.cs | 41 ++ Rdmp.Core/Curation/Data/ICatalogueOverview.cs | 21 + .../Data/ICatalogueOverviewDataPoint.cs | 15 + .../Curation/Data/Overview/OverviewModel.cs | 407 ++++++++++-------- .../up/089_AddCatalogueOverview.sql | 39 ++ Rdmp.Core/Rdmp.Core.csproj | 1 + .../ViewCatalogueOverviewUI.Designer.cs | 30 +- Rdmp.UI/Overview/ViewCatalogueOverviewUI.cs | 105 +---- Rdmp.UI/Overview/ViewCatalogueOverviewUI.resx | 33 +- Rdmp.UI/SimpleDialogs/SelectDialog`1.resx | 63 --- SharedAssemblyInfo.cs | 6 +- 12 files changed, 496 insertions(+), 367 deletions(-) create mode 100644 Rdmp.Core/Curation/Data/CatalogueOverview.cs create mode 100644 Rdmp.Core/Curation/Data/CatalogueOverviewDataPoint.cs create mode 100644 Rdmp.Core/Curation/Data/ICatalogueOverview.cs create mode 100644 Rdmp.Core/Curation/Data/ICatalogueOverviewDataPoint.cs create mode 100644 Rdmp.Core/Databases/CatalogueDatabase/up/089_AddCatalogueOverview.sql delete mode 100644 Rdmp.UI/SimpleDialogs/SelectDialog`1.resx diff --git a/Rdmp.Core/Curation/Data/CatalogueOverview.cs b/Rdmp.Core/Curation/Data/CatalogueOverview.cs new file mode 100644 index 0000000000..18057becd6 --- /dev/null +++ b/Rdmp.Core/Curation/Data/CatalogueOverview.cs @@ -0,0 +1,102 @@ +using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes; +using Rdmp.Core.MapsDirectlyToDatabaseTable; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Rdmp.Core.ReusableLibraryCode.Annotations; +using Rdmp.Core.Repositories; +using System.Data.Common; +using Rdmp.Core.Curation.Data.Defaults; +using Microsoft.IdentityModel.Tokens; + +namespace Rdmp.Core.Curation.Data; + +public class CatalogueOverview : DatabaseEntity, ICatalogueOverview +{ + + private int _catalogue_ID; + private DateTime? _lastDataLoad; + private DateTime? _lastExtractionTime; + private int _numberOfRecords; + private int _numberOfPeople; + private DateTime? _startDate; + private DateTime? _endDate; + + + [Unique] + [NotNull] + [DoNotImportDescriptions] + public int Catalogue_ID + { + get => _catalogue_ID; + protected set => SetField(ref _catalogue_ID, value); + } + + [DoNotImportDescriptions] + public DateTime? LastDataLoad + { + get => _lastDataLoad; + set => SetField(ref _lastDataLoad, value); + } + + [DoNotImportDescriptions] + public DateTime? LastExtractionTime + { + get => _lastExtractionTime; + set => SetField(ref _lastExtractionTime, value); + } + + [DoNotImportDescriptions] + public int NumberOfRecords + { + get => _numberOfRecords; + set => SetField(ref _numberOfRecords, value); + } + + [DoNotImportDescriptions] + public int NumberOfPeople + { + get => _numberOfPeople; + set => SetField(ref _numberOfPeople, value); + } + + [DoNotImportDescriptions] + public DateTime? StartDate + { + get => _startDate; + set => SetField(ref _startDate, value); + } + + [DoNotImportDescriptions] + public DateTime? EndDate + { + get => _endDate; + set => SetField(ref _endDate, value); + } + + public CatalogueOverview() { } + + internal CatalogueOverview(ICatalogueRepository repository, DbDataReader r) + : base(repository, r) + { + Catalogue_ID = int.Parse(r["Catalogue_ID"].ToString()); + LastDataLoad = !string.IsNullOrEmpty(r["LastDataLoad"].ToString()) ? DateTime.Parse(r["LastDataLoad"].ToString()) : null; + LastExtractionTime = !string.IsNullOrEmpty(r["LastExtractionTime"].ToString())? DateTime.Parse(r["LastExtractionTime"].ToString()) : null; + NumberOfRecords = int.Parse(r["NumberOfRecords"].ToString()); + NumberOfPeople = int.Parse(r["NumberOfPeople"].ToString()); + StartDate = !string.IsNullOrEmpty(r["StartDate"].ToString())? DateTime.Parse(r["StartDate"].ToString()) : null; + EndDate = !string.IsNullOrEmpty(r["EndDate"].ToString())? DateTime.Parse(r["EndDate"].ToString()) : null; + } + + public CatalogueOverview(ICatalogueRepository repository, int catalogueID, int numberOfRecords, int numberOfPeople) + { + repository.InsertAndHydrate(this, new Dictionary + { + {"Catalogue_ID", catalogueID }, + { "NumberOfRecords", numberOfRecords}, + {"NumberOfPeople", numberOfPeople } + }); + } +} \ No newline at end of file diff --git a/Rdmp.Core/Curation/Data/CatalogueOverviewDataPoint.cs b/Rdmp.Core/Curation/Data/CatalogueOverviewDataPoint.cs new file mode 100644 index 0000000000..85cb2cdd36 --- /dev/null +++ b/Rdmp.Core/Curation/Data/CatalogueOverviewDataPoint.cs @@ -0,0 +1,41 @@ +using Rdmp.Core.Repositories; +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data; + +public class CatalogueOverviewDataPoint : DatabaseEntity, ICatalogueOverviewDataPoint +{ + + private int _catalogueOverview_ID; + private DateTime _date; + private int _count; + public int CatalogueOverview_ID { get => _catalogueOverview_ID; protected set => SetField(ref _catalogueOverview_ID, value); } + + public DateTime Date { get => _date; set => SetField(ref _date, value); } + public int Count { get => _count; set => SetField(ref _count, value); } + + public CatalogueOverviewDataPoint() { } + + public CatalogueOverviewDataPoint(ICatalogueRepository repository, DbDataReader r) + : base(repository, r) + { + _catalogueOverview_ID = int.Parse(r["CatalogueOverview_ID"].ToString()); + Date = DateTime.Parse(r["Date"].ToString()); + Count = int.Parse(r["Count"].ToString()); + } + + public CatalogueOverviewDataPoint(ICatalogueRepository repository, int catalogueOverview_ID, DateTime date, int count) + { + repository.InsertAndHydrate(this, new Dictionary + { + {"CatalogueOverview_ID", catalogueOverview_ID}, + { "Date", date}, + {"Count", count} + }); + } +} diff --git a/Rdmp.Core/Curation/Data/ICatalogueOverview.cs b/Rdmp.Core/Curation/Data/ICatalogueOverview.cs new file mode 100644 index 0000000000..aa7e03827b --- /dev/null +++ b/Rdmp.Core/Curation/Data/ICatalogueOverview.cs @@ -0,0 +1,21 @@ +using Rdmp.Core.MapsDirectlyToDatabaseTable.Attributes; +using Rdmp.Core.MapsDirectlyToDatabaseTable; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Rdmp.Core.ReusableLibraryCode.Annotations; + +namespace Rdmp.Core.Curation.Data; + +public interface ICatalogueOverview +{ + int Catalogue_ID { get; } + DateTime? LastDataLoad { get; set; } + DateTime? LastExtractionTime { get; set; } + int NumberOfRecords { get; set; } + int NumberOfPeople { get; set; } + DateTime? StartDate { get; set; } + DateTime? EndDate { get; set; } +} diff --git a/Rdmp.Core/Curation/Data/ICatalogueOverviewDataPoint.cs b/Rdmp.Core/Curation/Data/ICatalogueOverviewDataPoint.cs new file mode 100644 index 0000000000..20766cf11e --- /dev/null +++ b/Rdmp.Core/Curation/Data/ICatalogueOverviewDataPoint.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Rdmp.Core.Curation.Data; + +public interface ICatalogueOverviewDataPoint +{ + int CatalogueOverview_ID { get; } + DateTime Date { get; set; } + int Count { get;set; } + +} diff --git a/Rdmp.Core/Curation/Data/Overview/OverviewModel.cs b/Rdmp.Core/Curation/Data/Overview/OverviewModel.cs index a058df3cf8..59866e399a 100644 --- a/Rdmp.Core/Curation/Data/Overview/OverviewModel.cs +++ b/Rdmp.Core/Curation/Data/Overview/OverviewModel.cs @@ -4,11 +4,14 @@ // RDMP is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. // You should have received a copy of the GNU General Public License along with RDMP. If not, see . +using MongoDB.Driver; using Rdmp.Core.CommandExecution; +using Rdmp.Core.Curation.Data.DataLoad; using Rdmp.Core.DataExport.Data; using Rdmp.Core.DataLoad.Triggers; using Rdmp.Core.DataViewing; using Rdmp.Core.Logging; +using Rdmp.Core.MapsDirectlyToDatabaseTable; using Rdmp.Core.QueryBuilding; using Rdmp.Core.Repositories; using Rdmp.Core.ReusableLibraryCode.DataAccess; @@ -17,6 +20,7 @@ using System.Collections.Generic; using System.Data; using System.Linq; +using System.Threading.Tasks; namespace Rdmp.Core.Curation.Data.Overview; @@ -28,246 +32,287 @@ public class OverviewModel private readonly ICatalogue _catalogue; private readonly IBasicActivateItems _activator; - - private DataTable _dataLoads; - - private int _numberOfPeople; - private int _numberOfRecords; + private CatalogueOverview _catalogueOverview; public OverviewModel(IBasicActivateItems activator, ICatalogue catalogue) { _activator = activator; _catalogue = catalogue; - if (catalogue != null) + _catalogueOverview = activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("Catalogue_ID", catalogue.ID).FirstOrDefault(); + } + + public void Generate() + { + var recordCount = GetRecordCount(); + var peopleCount = GetPeopleCount(); + if (_catalogueOverview is null) { - Regen(""); + _catalogueOverview = new CatalogueOverview(_activator.RepositoryLocator.CatalogueRepository, _catalogue.ID, recordCount, peopleCount); } + //data load date + _catalogueOverview.LastDataLoad = GetDataLoadDate(); + //extraction date + _catalogueOverview.LastExtractionTime = GetExtractionTime(); + //date range + var dates = GetDates(); + _catalogueOverview.StartDate = dates.Item1; + _catalogueOverview.EndDate = dates.Item2; + _catalogueOverview.SaveToDatabase(); + + GetCountsByDate(); + //todo generate graph data } - public void Regen(string whereClause) + private int GetRecordCount() { - DataTable dt = new(); - bool hasExtractionIdentifier = true; var column = _catalogue.CatalogueItems.Where(ci => ci.ExtractionInformation.IsExtractionIdentifier).FirstOrDefault(); if (column is null) { column = _catalogue.CatalogueItems.FirstOrDefault(); - hasExtractionIdentifier = false; } - if (column is null) return; + if (column is null) return 0; + var discoveredColumn = column.ColumnInfo.Discover(DataAccessContext.InternalDataProcessing); + return discoveredColumn.Table.GetRowCount(); + } + + private int GetPeopleCount() + { + var column = _catalogue.CatalogueItems.Where(ci => ci.ExtractionInformation.IsExtractionIdentifier).FirstOrDefault(); + if (column is null) return 0; var discoveredColumn = column.ColumnInfo.Discover(DataAccessContext.InternalDataProcessing); + var qb = new QueryBuilder("DISTINCT", null, null); + var memRepo = new MemoryRepository(); + qb.AddColumn(new ColumnInfoToIColumn(memRepo, column.ColumnInfo)); var server = discoveredColumn.Table.Database.Server; using var con = server.GetConnection(); con.Open(); - string populatedWhere = !string.IsNullOrWhiteSpace(whereClause) ? $"WHERE {whereClause}" : ""; - var sql = $"SELECT {column.ColumnInfo.GetRuntimeName()} FROM {discoveredColumn.Table.GetRuntimeName()} {populatedWhere}"; - using var cmd = server.GetCommand(sql, con); - cmd.CommandTimeout = 30000; - using var da = server.GetDataAdapter(cmd); - dt.BeginLoadData(); - da.Fill(dt); - dt.EndLoadData(); - con.Dispose(); - _numberOfRecords = dt.Rows.Count; - _numberOfPeople = hasExtractionIdentifier ? dt.DefaultView.ToTable(true, column.ColumnInfo.GetRuntimeName()).Rows.Count : 0; - GetDataLoads(); - dt.Dispose(); + using var cmd = server.GetCommand($"SELECT count(*) from ({qb.SQL})as dt", con); + return int.Parse(cmd.ExecuteScalar().ToString()); } - public int GetNumberOfRecords() + private DateTime? GetDataLoadDate() { - return _numberOfRecords; + var column = _catalogue.CatalogueItems.Where(c => c.Name == SpecialFieldNames.ValidFrom).FirstOrDefault(); + if (column is null) return null; + var discoveredColumn = column.ColumnInfo.Discover(DataAccessContext.InternalDataProcessing); + var qb = new QueryBuilder("DISTINCT", null, null); + var memRepo = new MemoryRepository(); + qb.AddColumn(new ColumnInfoToIColumn(memRepo, column.ColumnInfo)); + var server = discoveredColumn.Table.Database.Server; + using var con = server.GetConnection(); + con.Open(); + using var cmd = server.GetCommand(qb.SQL, con); + using var da = server.GetDataAdapter(cmd); + var dt = new DataTable(); + da.Fill(dt); + var rows = dt.AsEnumerable().Where(r => !string.IsNullOrEmpty(r[0].ToString())); + if (!rows.Any()) return null; + return rows.Select(r => DateTime.Parse(r[0].ToString())).Max(); } - public int GetNumberOfPeople() + private DateTime? GetExtractionTime() { - return _numberOfPeople; + var datasets = _activator.RepositoryLocator.DataExportRepository.GetAllObjectsWhere("Catalogue_ID", _catalogue.ID).Select(d => d.ID); + var results = _activator.RepositoryLocator.DataExportRepository.GetAllObjects().Where(result => datasets.Contains(result.ExtractableDataSet_ID)).ToList(); + if (!results.Any()) return null; + return results.Select(r => r.DateOfExtraction).Max(); + } - public Tuple GetStartEndDates(ColumnInfo dateColumn, string whereClause) - { - DataTable dt = new(); - var discoveredColumn = _catalogue.CatalogueItems.First().ColumnInfo.Discover(DataAccessContext.InternalDataProcessing); + private Tuple GetDates() + { + var column = _catalogue.CatalogueItems.Where(c => c.ColumnInfo.Data_type == "datetime2").FirstOrDefault();//temp + if (column == null) return null; + if (column is null) return null; + var discoveredColumn = column.ColumnInfo.Discover(DataAccessContext.InternalDataProcessing); + var qb = new QueryBuilder("DISTINCT", null, null); + var memRepo = new MemoryRepository(); + qb.AddColumn(new ColumnInfoToIColumn(memRepo, column.ColumnInfo)); var server = discoveredColumn.Table.Database.Server; - var populatedWhereClause = !string.IsNullOrWhiteSpace(whereClause) ? $"WHERE {whereClause}" : ""; using var con = server.GetConnection(); con.Open(); - if (server.DatabaseType == FAnsi.DatabaseType.MicrosoftSQLServer) - { - var sql = $@" - select min({dateColumn.GetRuntimeName()}) as min, max({dateColumn.GetRuntimeName()}) as max - from - (select {dateColumn.GetRuntimeName()}, - count(1) over (partition by year({dateColumn.GetRuntimeName()})) as occurs - from {discoveredColumn.Table.GetRuntimeName()} {populatedWhereClause}) as t - where occurs >1 - "; - - using var cmd = server.GetCommand(sql, con); - cmd.CommandTimeout = 30000; - using var da = server.GetDataAdapter(cmd); - dt.BeginLoadData(); - da.Fill(dt); - dt.EndLoadData(); - } - else - { - var repo = new MemoryCatalogueRepository(); - var qb = new QueryBuilder(null, null); - qb.AddColumn(new ColumnInfoToIColumn(repo, dateColumn)); - qb.AddCustomLine($"{dateColumn.Name} IS NOT NULL", FAnsi.Discovery.QuerySyntax.QueryComponent.WHERE); - var cmd = server.GetCommand(qb.SQL, con); - using var da = server.GetDataAdapter(cmd); - dt.BeginLoadData(); - da.Fill(dt); - var latest = dt.AsEnumerable() - .Max(r => r.Field(dateColumn.Name)); - var earliest = dt.AsEnumerable() - .Min(r => r.Field(dateColumn.Name)); - dt = new(); - dt.Rows.Add([earliest, latest]); - } - con.Dispose(); - return new Tuple(DateTime.Parse(dt.Rows[0].ItemArray[0].ToString()), DateTime.Parse(dt.Rows[0].ItemArray[1].ToString())); + using var cmd = server.GetCommand(qb.SQL, con); + using var da = server.GetDataAdapter(cmd); + var dt = new DataTable(); + da.Fill(dt); + var rows = dt.AsEnumerable().Where(r => !string.IsNullOrEmpty(r[0].ToString())); + if (!rows.Any()) return null; + var dateTimeRows = rows.Select(r => DateTime.Parse(r[0].ToString())); + var max = dateTimeRows.Max(); + var min = dateTimeRows.Min(); + return new Tuple(min, max); } + //public void Regen(string whereClause) + //{ + // DataTable dt = new(); + // bool hasExtractionIdentifier = true; + // var column = _catalogue.CatalogueItems.Where(ci => ci.ExtractionInformation.IsExtractionIdentifier).FirstOrDefault(); + // if (column is null) + // { + // column = _catalogue.CatalogueItems.FirstOrDefault(); + // hasExtractionIdentifier = false; + // } + // if (column is null) return; + // var discoveredColumn = column.ColumnInfo.Discover(DataAccessContext.InternalDataProcessing); + // var server = discoveredColumn.Table.Database.Server; + // using var con = server.GetConnection(); + // con.Open(); + // string populatedWhere = !string.IsNullOrWhiteSpace(whereClause) ? $"WHERE {whereClause}" : ""; + // var sql = $"SELECT {column.ColumnInfo.GetRuntimeName()} FROM {discoveredColumn.Table.GetRuntimeName()} {populatedWhere}"; + // using var cmd = server.GetCommand(sql, con); + // cmd.CommandTimeout = 30000; + // using var da = server.GetDataAdapter(cmd); + // dt.BeginLoadData(); + // da.Fill(dt); + // dt.EndLoadData(); + // con.Dispose(); + // _numberOfRecords = dt.Rows.Count; + // _numberOfPeople = hasExtractionIdentifier ? dt.DefaultView.ToTable(true, column.ColumnInfo.GetRuntimeName()).Rows.Count : 0; + // GetDataLoads(); + // dt.Dispose(); + //} - - public static DataTable GetCountsByDatePeriod(ColumnInfo dateColumn, string datePeriod, string optionalWhere = "") + private void GetCountsByDate() { + var column = _catalogue.CatalogueItems.Where(c => c.ColumnInfo.Data_type == "datetime2").FirstOrDefault();//temp + var dateString = "yyyy-MM"; DataTable dt = new(); - if (!(new[] { "Day", "Month", "Year" }).Contains(datePeriod)) - { - throw new Exception("Invalid Date period"); - } - var discoveredColumn = dateColumn.Discover(DataAccessContext.InternalDataProcessing); + var discoveredColumn = column.ColumnInfo.Discover(DataAccessContext.InternalDataProcessing); var server = discoveredColumn.Table.Database.Server; using var con = server.GetConnection(); con.Open(); - var dateString = "yyyy-MM"; - switch (datePeriod) - { - case "Day": - dateString = "yyyy-MM-dd"; - break; - case "Month": - dateString = "yyyy-MM"; - break; - case "Year": - dateString = "yyyy"; - break; - } - if (server.DatabaseType == FAnsi.DatabaseType.MicrosoftSQLServer) - { - var sql = @$" - SELECT format({dateColumn.GetRuntimeName()}, '{dateString}') as YearMonth, count(*) as '# Records' - FROM {discoveredColumn.Table.GetRuntimeName()} - WHERE {dateColumn.GetRuntimeName()} IS NOT NULL - {(optionalWhere != "" ? "AND" : "")} {optionalWhere.Replace('"', '\'')} - GROUP BY format({dateColumn.GetRuntimeName()}, '{dateString}') - ORDER BY 1 - "; - - using var cmd = server.GetCommand(sql, con); - cmd.CommandTimeout = 30000; - using var da = server.GetDataAdapter(cmd); - dt.BeginLoadData(); - da.Fill(dt); - dt.EndLoadData(); - } - else + var repo = new MemoryCatalogueRepository(); + var qb = new QueryBuilder(null, null); + qb.AddColumn(new ColumnInfoToIColumn(repo, column.ColumnInfo)); + qb.AddCustomLine($"{column.ColumnInfo.Name} IS NOT NULL", FAnsi.Discovery.QuerySyntax.QueryComponent.WHERE); + var cmd = server.GetCommand(qb.SQL, con); + using var da = server.GetDataAdapter(cmd); + dt.BeginLoadData(); + da.Fill(dt); + dt.EndLoadData(); + con.Dispose(); + Dictionary counts = []; + foreach (var key in dt.AsEnumerable().Select(row => DateTime.Parse(row[0].ToString()).ToString(dateString))) { - var repo = new MemoryCatalogueRepository(); - var qb = new QueryBuilder(null, null); - qb.AddColumn(new ColumnInfoToIColumn(repo, dateColumn)); - qb.AddCustomLine($"{dateColumn.Name} IS NOT NULL", FAnsi.Discovery.QuerySyntax.QueryComponent.WHERE); - var cmd = server.GetCommand(qb.SQL, con); - using var da = server.GetDataAdapter(cmd); - dt.BeginLoadData(); - da.Fill(dt); - Dictionary counts = []; - foreach (var key in dt.AsEnumerable().Select(row => DateTime.Parse(row.ItemArray[0].ToString()).ToString(dateString))) + if (counts.TryGetValue(key, out var count)) { counts[key]++; } - dt = new DataTable(); - foreach (var item in counts) + else { - DataRow dr = dt.NewRow(); - dr["YearMonth"] = item.Key; - dr["# Records"] = item.Value; - dt.Rows.Add(dr); + counts[key] = 1; } - dt.EndLoadData(); + } + //this is stupidly slow + foreach (CatalogueOverviewDataPoint dp in _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("CatalogueOverview_ID", _catalogueOverview.ID).ToList()) + { + dp.DeleteInDatabase(); + } + foreach (var item in counts) + { + var dp = new CatalogueOverviewDataPoint(_activator.RepositoryLocator.CatalogueRepository, _catalogueOverview.ID, DateTime.Parse(item.Key), item.Value); + dp.SaveToDatabase(); } - con.Dispose(); - return dt; } - private void GetDataLoads() + public int GetNumberOfRecords() { - _dataLoads = new(); - var repo = new MemoryCatalogueRepository(); - var qb = new QueryBuilder(null, null); - var columnInfo = _catalogue.CatalogueItems.Where(c => c.Name == SpecialFieldNames.DataLoadRunID).Select(c => c.ColumnInfo).FirstOrDefault(); - if (columnInfo != null) - { - qb.AddColumn(new ColumnInfoToIColumn(repo, columnInfo)); - qb.AddCustomLine($"{columnInfo.Name} IS NOT NULL", FAnsi.Discovery.QuerySyntax.QueryComponent.WHERE); - var sql = qb.SQL; - var server = columnInfo.Discover(DataAccessContext.InternalDataProcessing).Table.Database.Server; - using var con = server.GetConnection(); - con.Open(); + return _catalogueOverview.NumberOfRecords; + } - using var cmd = server.GetCommand(sql, con); - cmd.CommandTimeout = 30000; - using var da = server.GetDataAdapter(cmd); - _dataLoads.BeginLoadData(); - da.Fill(_dataLoads); - _dataLoads.EndLoadData(); - } + public int GetNumberOfPeople() + { + return _catalogueOverview.NumberOfPeople; + } + public string GetLatestExtraction() + { + return _catalogueOverview.LastExtractionTime != null ? _catalogueOverview.LastExtractionTime.ToString() : null; } - public DataTable GetMostRecentDataLoad() + public string GetLatestDataLoad() { - if (_dataLoads == null) GetDataLoads(); - if (_dataLoads.Rows.Count == 0) return null; - var maxDataLoadId = _dataLoads.AsEnumerable().Select(r => int.Parse(r[0].ToString())).Distinct().Max(); - var loggingServers = _activator.RepositoryLocator.CatalogueRepository.GetAllObjectsWhere("CreatedByAssembly", "Rdmp.Core/Databases.LoggingDatabase"); - var columnInfo = _catalogue.CatalogueItems.Where(c => c.Name == SpecialFieldNames.DataLoadRunID).Select(c => c.ColumnInfo).First(); - var server = columnInfo.Discover(DataAccessContext.InternalDataProcessing).Table.Database.Server; + return _catalogueOverview.LastDataLoad != null ? _catalogueOverview.LastDataLoad.ToString() : null; + } - DataTable dt = new(); - foreach (var loggingServer in loggingServers) - { - var logCollection = new ViewLogsCollection(loggingServer, new LogViewerFilter(LoggingTables.DataLoadRun)); - var dataLoadRunSql = $"{logCollection.GetSql()} WHERE ID={maxDataLoadId}"; - var logServer = loggingServer.Discover(DataAccessContext.InternalDataProcessing).Server; - using var loggingCon = logServer.GetConnection(); - loggingCon.Open(); - using var loggingCmd = logServer.GetCommand(dataLoadRunSql, loggingCon); - loggingCmd.CommandTimeout = 30000; - using var loggingDa = server.GetDataAdapter(loggingCmd); - dt.BeginLoadData(); - loggingDa.Fill(dt); - dt.EndLoadData(); - loggingCon.Dispose(); - if (dt.Rows.Count > 0) - { - break; - } - } - return dt; + public Tuple GetStartEndDates() + { + return new Tuple(_catalogueOverview.StartDate, _catalogueOverview.EndDate); } - public List GetExtractions() + + public static DataTable GetCountsByDatePeriod(ColumnInfo dateColumn, string datePeriod, string optionalWhere = "") { - var datasets = _activator.RepositoryLocator.DataExportRepository.GetAllObjectsWhere("Catalogue_ID", _catalogue.ID).Select(d => d.ID); - var results = _activator.RepositoryLocator.DataExportRepository.GetAllObjects().Where(result => datasets.Contains(result.ExtractableDataSet_ID)).ToList(); - return results; + DataTable dt = new(); + //if (!(new[] { "Day", "Month", "Year" }).Contains(datePeriod)) + //{ + // throw new Exception("Invalid Date period"); + //} + //var discoveredColumn = dateColumn.Discover(DataAccessContext.InternalDataProcessing); + //var server = discoveredColumn.Table.Database.Server; + //using var con = server.GetConnection(); + //con.Open(); + //var dateString = "yyyy-MM"; + //switch (datePeriod) + //{ + // case "Day": + // dateString = "yyyy-MM-dd"; + // break; + // case "Month": + // dateString = "yyyy-MM"; + // break; + // case "Year": + // dateString = "yyyy"; + // break; + //} + //if (server.DatabaseType == FAnsi.DatabaseType.MicrosoftSQLServer) + //{ + // var sql = @$" + //SELECT format({dateColumn.GetRuntimeName()}, '{dateString}') as YearMonth, count(*) as '# Records' + //FROM {discoveredColumn.Table.GetRuntimeName()} + //WHERE {dateColumn.GetRuntimeName()} IS NOT NULL + //{(optionalWhere != "" ? "AND" : "")} {optionalWhere.Replace('"', '\'')} + //GROUP BY format({dateColumn.GetRuntimeName()}, '{dateString}') + //ORDER BY 1 + //"; + // using var cmd = server.GetCommand(sql, con); + // cmd.CommandTimeout = 30000; + // using var da = server.GetDataAdapter(cmd); + // dt.BeginLoadData(); + // da.Fill(dt); + // dt.EndLoadData(); + //} + //else + //{ + // var repo = new MemoryCatalogueRepository(); + // var qb = new QueryBuilder(null, null); + // qb.AddColumn(new ColumnInfoToIColumn(repo, dateColumn)); + // qb.AddCustomLine($"{dateColumn.Name} IS NOT NULL", FAnsi.Discovery.QuerySyntax.QueryComponent.WHERE); + // var cmd = server.GetCommand(qb.SQL, con); + // using var da = server.GetDataAdapter(cmd); + // dt.BeginLoadData(); + // da.Fill(dt); + // Dictionary counts = []; + // foreach (var key in dt.AsEnumerable().Select(row => DateTime.Parse(row.ItemArray[0].ToString()).ToString(dateString))) + // { + // counts[key]++; + // } + // dt = new DataTable(); + // foreach (var item in counts) + // { + // DataRow dr = dt.NewRow(); + // dr["YearMonth"] = item.Key; + // dr["# Records"] = item.Value; + // dt.Rows.Add(dr); + // } + // dt.EndLoadData(); + + //} + //con.Dispose(); + return dt; } } diff --git a/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddCatalogueOverview.sql b/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddCatalogueOverview.sql new file mode 100644 index 0000000000..20d9489e61 --- /dev/null +++ b/Rdmp.Core/Databases/CatalogueDatabase/up/089_AddCatalogueOverview.sql @@ -0,0 +1,39 @@ +--Version: 8.4.1 +--Description: Add catalogue Overview + + +if not exists (select 1 from sys.tables where name = 'CatalogueOverview') +BEGIN +CREATE TABLE [dbo].[CatalogueOverview]( + [ID] [int] IDENTITY(1,1) NOT NULL, + [Catalogue_ID] [int] NOT NULL, + [LastDataLoad] [datetime] NULL, + [LastExtractionTime] [datetime] NULL, + [NumberOfRecords] [int] NOT NULL, + [NumberOfPeople] [int] NOT NULL, + [StartDate] [datetime] NULL, + [EndDate] [datetime] NULL, + CONSTRAINT FK_Catalogue_Overview_Catalogue FOREIGN KEY (Catalogue_ID) REFERENCES Catalogue(ID) ON DELETE CASCADE, +CONSTRAINT PK_CatalogueOverview PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +END +GO + +if not exists (select 1 from sys.tables where name = 'CatalogueOverviewDataPoint') +BEGIN +CREATE TABLE [dbo].[CatalogueOverviewDataPoint]( + [ID] [int] IDENTITY(1,1) NOT NULL, + [CatalogueOverview_ID] [int] NOT NULL, + [Date] [datetime] NOT NULL, + [Count] int NOT NULL, + CONSTRAINT FK_Catalogue_Overview_DataPoint_Catalogue_Overview FOREIGN KEY (CatalogueOverview_ID) REFERENCES CatalogueOverview(ID) ON DELETE CASCADE, + CONSTRAINT PK_CatalogueOverviewDataPoin2t PRIMARY KEY CLUSTERED +( + [ID] ASC +)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON) ON [PRIMARY] +) ON [PRIMARY] +END +GO diff --git a/Rdmp.Core/Rdmp.Core.csproj b/Rdmp.Core/Rdmp.Core.csproj index 2de063bb79..2838b907e7 100644 --- a/Rdmp.Core/Rdmp.Core.csproj +++ b/Rdmp.Core/Rdmp.Core.csproj @@ -257,6 +257,7 @@ + diff --git a/Rdmp.UI/Overview/ViewCatalogueOverviewUI.Designer.cs b/Rdmp.UI/Overview/ViewCatalogueOverviewUI.Designer.cs index 2a702eb12b..c717cac0ed 100644 --- a/Rdmp.UI/Overview/ViewCatalogueOverviewUI.Designer.cs +++ b/Rdmp.UI/Overview/ViewCatalogueOverviewUI.Designer.cs @@ -34,6 +34,7 @@ private void InitializeComponent() System.ComponentModel.ComponentResourceManager resources = new System.ComponentModel.ComponentResourceManager(typeof(ViewCatalogueOverviewUI)); panel1 = new System.Windows.Forms.Panel(); splitContainer1 = new System.Windows.Forms.SplitContainer(); + btnGenerate = new System.Windows.Forms.Button(); btnSettings = new System.Windows.Forms.Button(); lblDescription = new System.Windows.Forms.Label(); lblName = new System.Windows.Forms.Label(); @@ -59,7 +60,6 @@ private void InitializeComponent() panel5 = new System.Windows.Forms.Panel(); label3 = new System.Windows.Forms.Label(); lblRecords = new System.Windows.Forms.Label(); - areaChart1 = new CatalogueSummary.DataQualityReporting.AreaChartUI(OnTabChange); panel1.SuspendLayout(); ((System.ComponentModel.ISupportInitialize)splitContainer1).BeginInit(); splitContainer1.Panel1.SuspendLayout(); @@ -87,6 +87,7 @@ private void InitializeComponent() // // splitContainer1.Panel1 // + splitContainer1.Panel1.Controls.Add(btnGenerate); splitContainer1.Panel1.Controls.Add(btnSettings); splitContainer1.Panel1.Controls.Add(lblDescription); splitContainer1.Panel1.Controls.Add(lblName); @@ -94,6 +95,16 @@ private void InitializeComponent() splitContainer1.SplitterDistance = 259; splitContainer1.TabIndex = 0; // + // btnGenerate + // + btnGenerate.Image = (System.Drawing.Image)resources.GetObject("btnGenerate.Image"); + btnGenerate.Location = new System.Drawing.Point(198, 3); + btnGenerate.Name = "btnGenerate"; + btnGenerate.Size = new System.Drawing.Size(26, 23); + btnGenerate.TabIndex = 5; + btnGenerate.UseVisualStyleBackColor = true; + btnGenerate.Click += btnGenerate_Click; + // // btnSettings // btnSettings.Image = (System.Drawing.Image)resources.GetObject("btnSettings.Image"); @@ -111,7 +122,6 @@ private void InitializeComponent() lblDescription.Size = new System.Drawing.Size(240, 285); lblDescription.TabIndex = 3; lblDescription.Text = "label1"; - lblDescription.Click += lblDescription_Click; // // lblName // @@ -144,7 +154,6 @@ private void InitializeComponent() lblLatestExtraction.TabIndex = 3; lblLatestExtraction.Text = "label2"; lblLatestExtraction.TextAlign = System.Drawing.ContentAlignment.MiddleCenter; - lblLatestExtraction.Click += lblLatestExtraction_Click; // // label2 // @@ -155,7 +164,6 @@ private void InitializeComponent() label2.Size = new System.Drawing.Size(169, 30); label2.TabIndex = 2; label2.Text = "Latest Extraction"; - label2.Click += label2_Click; // // label1 // @@ -177,7 +185,6 @@ private void InitializeComponent() lblLastDataLoad.TabIndex = 1; lblLastDataLoad.Text = "label2"; lblLastDataLoad.TextAlign = System.Drawing.ContentAlignment.MiddleCenter; - lblLastDataLoad.Click += lblLastDataLoad_Click; // // lblTime // @@ -203,7 +210,6 @@ private void InitializeComponent() tbMainWhere.Name = "tbMainWhere"; tbMainWhere.Size = new System.Drawing.Size(402, 23); tbMainWhere.TabIndex = 7; - tbMainWhere.TextChanged += tbMainWhere_TextChanged; // // cbTimeColumns // @@ -234,13 +240,11 @@ private void InitializeComponent() // // button1 // - button1.Image = (System.Drawing.Image)resources.GetObject("button1.Image"); button1.Location = new System.Drawing.Point(723, 6); button1.Name = "button1"; button1.Size = new System.Drawing.Size(26, 23); button1.TabIndex = 5; button1.UseVisualStyleBackColor = false; - button1.Click += button1_Click; // // panel7 // @@ -353,13 +357,6 @@ private void InitializeComponent() lblRecords.TabIndex = 3; lblRecords.Text = "label2"; lblRecords.TextAlign = System.Drawing.ContentAlignment.MiddleCenter; - splitContainer1.Panel2.Controls.Add(areaChart1); - areaChart1.Location = new System.Drawing.Point(4, 0); - areaChart1.Margin = new System.Windows.Forms.Padding(4, 3, 4, 3); - areaChart1.Name = "areaChart1"; - areaChart1.Size = new System.Drawing.Size(1075, 400); - areaChart1.TabIndex = 0; - areaChart1.Load += areaChart1_Load; // // ViewCatalogueOverviewUI // @@ -399,7 +396,7 @@ private void InitializeComponent() private System.Windows.Forms.Label label1; private System.Windows.Forms.Label label2; private System.Windows.Forms.Label lblLatestExtraction; - private CatalogueSummary.DataQualityReporting.AreaChartUI areaChart1; + //private CatalogueSummary.DataQualityReporting.AreaChartUI areaChart1; private System.Windows.Forms.ComboBox cbTimeColumns; private System.Windows.Forms.Panel panel3; private System.Windows.Forms.Label lblTime; @@ -418,5 +415,6 @@ private void InitializeComponent() private System.Windows.Forms.Label label6; private System.Windows.Forms.Label lblPeople; private System.Windows.Forms.Button button1; + private System.Windows.Forms.Button btnGenerate; } } \ No newline at end of file diff --git a/Rdmp.UI/Overview/ViewCatalogueOverviewUI.cs b/Rdmp.UI/Overview/ViewCatalogueOverviewUI.cs index 5ec04c1c47..16f1fa8b44 100644 --- a/Rdmp.UI/Overview/ViewCatalogueOverviewUI.cs +++ b/Rdmp.UI/Overview/ViewCatalogueOverviewUI.cs @@ -10,6 +10,7 @@ using System.Data; using System.Drawing; using System.Linq; +using System.Threading.Tasks; using System.Windows.Forms; using BrightIdeasSoftware; using Rdmp.Core; @@ -45,7 +46,6 @@ public partial class ViewCatalogueOverviewUI : ViewCatalogueOverviewUI_Design { private Catalogue _catalogue; private OverviewModel _overview; - private List _dateColumns; private static readonly string[] frequencies = new[] { "Day", "Month", "Year" }; public ViewCatalogueOverviewUI() @@ -58,46 +58,17 @@ private void UpdateCatalogueData() lblName.Text = _catalogue.Name; lblDescription.Text = _catalogue.Description; - var latestDataLoad = _overview.GetMostRecentDataLoad(); - lblLastDataLoad.Text = latestDataLoad != null ? latestDataLoad.Rows[0][3].ToString() : "No Successful DataLoads"; - var extractions = _overview.GetExtractions(); - if (extractions.Any()) - { - var latestExtractionDate = extractions.AsEnumerable().Select(r => r.DateOfExtraction).Distinct().Max(); - lblLatestExtraction.Text = latestExtractionDate.ToString(); - } - else - { - lblLatestExtraction.Text = "Catalogue has not been extracted"; - } - try - { - var syntaxHelper = _catalogue.GetDistinctLiveDatabaseServer(DataAccessContext.InternalDataProcessing, false)?.GetQuerySyntaxHelper(); - var dateTypeString = syntaxHelper.TypeTranslater.GetSQLDBTypeForCSharpType(new TypeGuesser.DatabaseTypeRequest(typeof(DateTime))); - - _dateColumns = _catalogue.CatalogueItems.Where(ci => ci.ColumnInfo.Data_type == dateTypeString).ToList(); - } - catch - { - return; - } - cbTimeColumns.Items.Clear(); - cbTimeColumns.Items.AddRange(_dateColumns.ToArray()); - var pks = _dateColumns.Where(ci => ci.ColumnInfo.IsPrimaryKey).ToList(); - - if (pks.Any()) - { - cbTimeColumns.SelectedIndex = _dateColumns.IndexOf(pks[0]); - } - else if (_dateColumns.Any()) - { - cbTimeColumns.SelectedIndex = 0; - } + var latestDataLoad = _overview.GetLatestDataLoad(); + lblLastDataLoad.Text = latestDataLoad ?? "No Successful DataLoads"; + var extraction = _overview.GetLatestExtraction(); + lblLatestExtraction.Text = extraction ?? "Catalogue has not been extracted"; lblRecords.Text = _overview.GetNumberOfRecords().ToString(); - var dates = _overview.GetStartEndDates(_dateColumns[cbTimeColumns.SelectedIndex].ColumnInfo, tbMainWhere.Text); + var dates = _overview.GetStartEndDates(); lblDateRange.Text = $"{dates.Item1} - {dates.Item2}"; lblPeople.Text = $"{_overview.GetNumberOfPeople()}"; + //areaChart1.GenerateChart(dt, $"Records per {cbFrequency.SelectedItem}"); + } private void cbFrequency_SelectedIndexChanged(object sender, EventArgs e) @@ -126,8 +97,7 @@ private int OnTabChange(int tabIndex) private void cbTimeColumns_SelectedIndexChanged(object sender, EventArgs e) { - var dt = OverviewModel.GetCountsByDatePeriod(_dateColumns[cbTimeColumns.SelectedIndex].ColumnInfo, cbFrequency.SelectedItem.ToString(), tbMainWhere.Text); - areaChart1.GenerateChart(dt, $"Records per {cbFrequency.SelectedItem}"); + UpdateCatalogueData(); } public override void SetDatabaseObject(IActivateItems activator, Catalogue databaseObject) @@ -143,60 +113,27 @@ public override void SetDatabaseObject(IActivateItems activator, Catalogue datab public override string GetTabName() => $"{_catalogue.Name} Overview"; - private void lblLastDataLoad_Click(object sender, EventArgs e) - { - - } - - private void label2_Click(object sender, EventArgs e) - { - - } - - private void lblDescription_Click(object sender, EventArgs e) - { - - } - - private void lblLatestExtraction_Click(object sender, EventArgs e) - { - - } - - private void dataGridView1_CellContentClick(object sender, DataGridViewCellEventArgs e) - { - - } - - private void areaChart1_Load(object sender, EventArgs e) - { - - } - - private void tbMainWhere_TextChanged(object sender, EventArgs e) - { - } - private void btnSettings_Click(object sender, EventArgs e) { Activator.Activate(_catalogue); } - private void button1_Click(object sender, EventArgs e) + private void label4_Click(object sender, EventArgs e) { - _overview.Regen(tbMainWhere.Text); - lblRecords.Text = _overview.GetNumberOfRecords().ToString(); - lblPeople.Text = _overview.GetNumberOfPeople().ToString(); - var dates = _overview.GetStartEndDates(_dateColumns[cbTimeColumns.SelectedIndex].ColumnInfo, tbMainWhere.Text); - lblDateRange.Text = $"{dates.Item1} - {dates.Item2}"; - lblPeople.Text = $"{_overview.GetNumberOfPeople()}"; - var dt = OverviewModel.GetCountsByDatePeriod(_dateColumns[cbTimeColumns.SelectedIndex].ColumnInfo, cbFrequency.SelectedItem.ToString(), tbMainWhere.Text); - areaChart1.GenerateChart(dt, $"Records per {cbFrequency.SelectedItem}"); + } - private void label4_Click(object sender, EventArgs e) + private void btnGenerate_Click(object sender, EventArgs e) { - + _overview.Generate(); + UpdateCatalogueData(); + //Task.Run(() => + //{ + // _overview.Generate(); + //}).ContinueWith((task) => + //{ + // UpdateCatalogueData(); + //}, TaskScheduler.FromCurrentSynchronizationContext()); } } [TypeDescriptionProvider(typeof(AbstractControlDescriptionProvider))] diff --git a/Rdmp.UI/Overview/ViewCatalogueOverviewUI.resx b/Rdmp.UI/Overview/ViewCatalogueOverviewUI.resx index 6d81376dc8..1cf02dddc7 100644 --- a/Rdmp.UI/Overview/ViewCatalogueOverviewUI.resx +++ b/Rdmp.UI/Overview/ViewCatalogueOverviewUI.resx @@ -1,7 +1,7 @@