diff --git a/src/BUTR.CrashReportServer/Controllers/CrashUploadController.cs b/src/BUTR.CrashReportServer/Controllers/CrashUploadController.cs index 1f88da1..58ed0cb 100644 --- a/src/BUTR.CrashReportServer/Controllers/CrashUploadController.cs +++ b/src/BUTR.CrashReportServer/Controllers/CrashUploadController.cs @@ -87,7 +87,7 @@ public async Task CrashUploadAsync(CancellationToken ct) Converters = { new JsonStringEnumConverter() } }), ct); - idEntity = new IdEntity { FileId = default!, CrashReportId = id, Created = DateTimeOffset.UtcNow, }; + idEntity = new IdEntity { FileId = default!, CrashReportId = id, Version = version, Created = DateTimeOffset.UtcNow, }; await _dbContext.Set().AddAsync(idEntity, ct); await _dbContext.Set().AddAsync(new FileEntity { Id = idEntity, DataCompressed = compressedHtmlStream.ToArray(), }, ct); if (version >= 13) await _dbContext.Set().AddAsync(new JsonEntity { Id = idEntity, CrashReportCompressed = compressedJsonStream.ToArray(), }, ct); @@ -118,7 +118,7 @@ public async Task CrashUploadAsync([FromBody] CrashReportUploadBo await using var compressedHtmlStream = await _gZipCompressor.CompressAsync(html.AsStream(), ct); await using var compressedJsonStream = await _gZipCompressor.CompressAsync(json.AsStream(), ct); - idEntity = new IdEntity { FileId = default!, CrashReportId = body.CrashReport.Id, Created = DateTimeOffset.UtcNow, }; + idEntity = new IdEntity { FileId = default!, CrashReportId = body.CrashReport.Id, Version = body.CrashReport.Version, Created = DateTimeOffset.UtcNow, }; await _dbContext.Set().AddAsync(idEntity, ct); await _dbContext.Set().AddAsync(new JsonEntity { Id = idEntity, CrashReportCompressed = compressedJsonStream.ToArray(), }, ct); await _dbContext.Set().AddAsync(new FileEntity { Id = idEntity, DataCompressed = compressedHtmlStream.ToArray(), }, ct); diff --git a/src/BUTR.CrashReportServer/Controllers/ReportController.cs b/src/BUTR.CrashReportServer/Controllers/ReportController.cs index 24255c0..6440156 100644 --- a/src/BUTR.CrashReportServer/Controllers/ReportController.cs +++ b/src/BUTR.CrashReportServer/Controllers/ReportController.cs @@ -144,19 +144,19 @@ public async Task Delete(string filename) public ActionResult> GetAllFilenames() => Ok(_dbContext.Set().Select(x => x.FileId)); [Authorize] - [HttpPost("GetFilenameDates")] - [ProducesResponseType(typeof(FilenameDate[]), StatusCodes.Status200OK, "application/json")] + [HttpPost("GetMetadata")] + [ProducesResponseType(typeof(FileMetadata[]), StatusCodes.Status200OK, "application/json")] [ProducesResponseType(typeof(void), StatusCodes.Status500InternalServerError, "application/problem+json")] [ProducesResponseType(typeof(TLSError), StatusCodes.Status400BadRequest, "application/json")] [HttpsProtocol(Protocol = SslProtocols.Tls13)] - public ActionResult> GetFilenameDates(ICollection filenames, CancellationToken ct) + public ActionResult> GetFilenameDates(ICollection filenames, CancellationToken ct) { var filenamesWithExtension = filenames.Select(Path.GetFileNameWithoutExtension).ToImmutableArray(); return Ok(_dbContext.Set() .Where(x => filenamesWithExtension.Contains(x.FileId)) - .Select(x => new { x.FileId, x.Created }) + .Select(x => new { x.FileId, x.Version, x.Created }) .AsAsyncEnumerable() - .Select(x => new FilenameDate(x.FileId, x.Created.ToUniversalTime().ToString("O")))); + .Select(x => new FileMetadata(x.FileId, x.Version, x.Created.ToUniversalTime()))); } } \ No newline at end of file diff --git a/src/BUTR.CrashReportServer/Migrations/20231002112904_IdEntityVersion.Designer.cs b/src/BUTR.CrashReportServer/Migrations/20231002112904_IdEntityVersion.Designer.cs new file mode 100644 index 0000000..6482e01 --- /dev/null +++ b/src/BUTR.CrashReportServer/Migrations/20231002112904_IdEntityVersion.Designer.cs @@ -0,0 +1,107 @@ +// +using System; +using BUTR.CrashReportServer.Contexts; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace BUTR.CrashReportServer.Migrations +{ + [DbContext(typeof(AppDbContext))] + [Migration("20231002112904_IdEntityVersion")] + partial class IdEntityVersion + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "7.0.11"); + + modelBuilder.Entity("BUTR.CrashReportServer.Models.Database.FileEntity", b => + { + b.Property("FileId") + .HasColumnType("TEXT") + .HasColumnName("file_id"); + + b.Property("DataCompressed") + .IsRequired() + .HasColumnType("BLOB") + .HasColumnName("data_compressed"); + + b.HasKey("FileId") + .HasName("file_entity_pkey"); + + b.ToTable("file_entity", (string)null); + }); + + modelBuilder.Entity("BUTR.CrashReportServer.Models.Database.IdEntity", b => + { + b.Property("FileId") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT") + .HasColumnName("file_id") + .HasDefaultValueSql("hex(randomblob(3))"); + + b.Property("CrashReportId") + .HasColumnType("TEXT") + .HasColumnName("crash_report_id"); + + b.Property("Created") + .HasColumnType("TEXT") + .HasColumnName("created"); + + b.Property("Version") + .HasColumnType("INTEGER") + .HasColumnName("version"); + + b.HasKey("FileId"); + + b.HasIndex("CrashReportId"); + + b.ToTable("id_entity", (string)null); + }); + + modelBuilder.Entity("BUTR.CrashReportServer.Models.Database.JsonEntity", b => + { + b.Property("FileId") + .HasColumnType("TEXT") + .HasColumnName("file_id"); + + b.Property("CrashReportCompressed") + .IsRequired() + .HasColumnType("BLOB") + .HasColumnName("data_compressed"); + + b.HasKey("FileId"); + + b.ToTable("json_entity", (string)null); + }); + + modelBuilder.Entity("BUTR.CrashReportServer.Models.Database.FileEntity", b => + { + b.HasOne("BUTR.CrashReportServer.Models.Database.IdEntity", "Id") + .WithOne() + .HasForeignKey("BUTR.CrashReportServer.Models.Database.FileEntity", "FileId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Id"); + }); + + modelBuilder.Entity("BUTR.CrashReportServer.Models.Database.JsonEntity", b => + { + b.HasOne("BUTR.CrashReportServer.Models.Database.IdEntity", "Id") + .WithOne() + .HasForeignKey("BUTR.CrashReportServer.Models.Database.JsonEntity", "FileId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Id"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/BUTR.CrashReportServer/Migrations/20231002112904_IdEntityVersion.cs b/src/BUTR.CrashReportServer/Migrations/20231002112904_IdEntityVersion.cs new file mode 100644 index 0000000..3bd53ec --- /dev/null +++ b/src/BUTR.CrashReportServer/Migrations/20231002112904_IdEntityVersion.cs @@ -0,0 +1,29 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace BUTR.CrashReportServer.Migrations +{ + /// + public partial class IdEntityVersion : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "version", + table: "id_entity", + type: "INTEGER", + nullable: false, + defaultValue: (byte)0); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "version", + table: "id_entity"); + } + } +} diff --git a/src/BUTR.CrashReportServer/Migrations/AppDbContextModelSnapshot.cs b/src/BUTR.CrashReportServer/Migrations/AppDbContextModelSnapshot.cs index 26b25ff..2bce0a0 100644 --- a/src/BUTR.CrashReportServer/Migrations/AppDbContextModelSnapshot.cs +++ b/src/BUTR.CrashReportServer/Migrations/AppDbContextModelSnapshot.cs @@ -15,7 +15,7 @@ partial class AppDbContextModelSnapshot : ModelSnapshot protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 - modelBuilder.HasAnnotation("ProductVersion", "7.0.4"); + modelBuilder.HasAnnotation("ProductVersion", "7.0.11"); modelBuilder.Entity("BUTR.CrashReportServer.Models.Database.FileEntity", b => { @@ -50,6 +50,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("TEXT") .HasColumnName("created"); + b.Property("Version") + .HasColumnType("INTEGER") + .HasColumnName("version"); + b.HasKey("FileId"); b.HasIndex("CrashReportId"); diff --git a/src/BUTR.CrashReportServer/Models/API/FileMetadata.cs b/src/BUTR.CrashReportServer/Models/API/FileMetadata.cs new file mode 100644 index 0000000..ffffee5 --- /dev/null +++ b/src/BUTR.CrashReportServer/Models/API/FileMetadata.cs @@ -0,0 +1,5 @@ +using System; + +namespace BUTR.CrashReportServer.Models.API; + +public sealed record FileMetadata(string File, byte Version, DateTimeOffset Date); \ No newline at end of file diff --git a/src/BUTR.CrashReportServer/Models/API/FilenameDate.cs b/src/BUTR.CrashReportServer/Models/API/FilenameDate.cs deleted file mode 100644 index c744d8b..0000000 --- a/src/BUTR.CrashReportServer/Models/API/FilenameDate.cs +++ /dev/null @@ -1,3 +0,0 @@ -namespace BUTR.CrashReportServer.Models.API; - -public record FilenameDate(string Filename, string Date); \ No newline at end of file diff --git a/src/BUTR.CrashReportServer/Models/Database/IdEntity.cs b/src/BUTR.CrashReportServer/Models/Database/IdEntity.cs index 80465a2..c433f0a 100644 --- a/src/BUTR.CrashReportServer/Models/Database/IdEntity.cs +++ b/src/BUTR.CrashReportServer/Models/Database/IdEntity.cs @@ -6,5 +6,6 @@ public sealed record IdEntity : IEntity { public required string FileId { get; set; } public required Guid CrashReportId { get; set; } + public required byte Version { get; set; } public required DateTimeOffset Created { get; set; } } \ No newline at end of file diff --git a/src/BUTR.CrashReportServer/Services/DatabaseMigrator.cs b/src/BUTR.CrashReportServer/Services/DatabaseMigrator.cs index 9018a93..87bf262 100644 --- a/src/BUTR.CrashReportServer/Services/DatabaseMigrator.cs +++ b/src/BUTR.CrashReportServer/Services/DatabaseMigrator.cs @@ -55,7 +55,7 @@ await Parallel.ForEachAsync(Enumerable.Range(0, 4), options, async (_, ct2) => { var dbContext = await dbContextPool.GetAsync(ct2); - var wrong = dbContext.Set().AsNoTracking().Where(x => x.Id.CrashReportId == Guid.Empty).Take(1000); + var wrong = dbContext.Set().AsNoTracking().Where(x => x.Id.Version == 0).Take(1000); while (true) { var entities = wrong.ToArray(); @@ -74,7 +74,7 @@ await Parallel.ForEachAsync(Enumerable.Range(0, 4), options, async (_, ct2) => { sb.AppendLine($""" UPDATE id_entity - SET crash_report_id = '{id}' + SET version = '{version}' WHERE file_id = '{entity.Id.FileId}'; """); } diff --git a/src/BUTR.CrashReportServer/Utils/CrashReportRawParser.cs b/src/BUTR.CrashReportServer/Utils/CrashReportRawParser.cs index 906b4e3..5eec80d 100644 --- a/src/BUTR.CrashReportServer/Utils/CrashReportRawParser.cs +++ b/src/BUTR.CrashReportServer/Utils/CrashReportRawParser.cs @@ -1,5 +1,7 @@ using BUTR.CrashReport.Models; +using SQLitePCL; + using System; using System.Buffers; using System.Buffers.Text; @@ -15,17 +17,18 @@ public static class CrashReportRawParser { private static readonly byte[] Newline = "\n"u8.ToArray(); private static readonly byte[] ReportMarkerStart = " TryReadCrashReportDataAsync(PipeReader reader) { var crashRreportId = Guid.Empty; - var crashRreportVersion = (byte) 0; + var crashReportVersion = (byte) 0; var crashReportModel = default(CrashReportModel); while (true) @@ -33,7 +36,7 @@ public static class CrashReportRawParser var result = await reader.ReadAsync(); var buffer = result.Buffer; - while (!result.IsCompleted && !TryReadCrashReportData(ref buffer, out crashRreportId, out crashRreportVersion)) + while (!result.IsCompleted && !TryReadCrashReportData(ref buffer, out crashRreportId, out crashReportVersion)) { reader.AdvanceTo(buffer.Start, buffer.End); } @@ -55,9 +58,10 @@ public static class CrashReportRawParser } var isValid = crashRreportId != Guid.Empty && - ((crashRreportVersion > 12 && crashReportModel is not null) || (crashRreportVersion <= 12 && crashReportModel is null)); + crashReportVersion > 0 && + ((crashReportVersion > 12 && crashReportModel is not null) || (crashReportVersion <= 12 && crashReportModel is null)); - return (isValid, crashRreportId, crashRreportVersion, crashReportModel); + return (isValid, crashRreportId, crashReportVersion, crashReportModel); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -72,6 +76,24 @@ private static bool TryReadCrashReportData(ref ReadOnlySequence buffer, ou buffer = buffer.Slice(reader.Position); if (line.IndexOf(ReportMarkerStart) is not (var idxIdStart and not -1)) return false; + var line2 = line.utf8_span_to_string(); + + if (line.Slice(idxIdStart + ReportMarkerStart.Length).IndexOf(ReportIdMarkerEnd) is not (var idxIdEnd and not -1)) return false; + + if (!Utf8Parser.TryParse(line.Slice(idxIdStart + ReportMarkerStart.Length, idxIdEnd), out crashReportId, out _)) return false; + + if (line.IndexOf(ReportVersionMarkerStart) is var idxVersionStart and not -1) + { + var idxEnd = -1; + if (line.Slice(idxVersionStart + ReportVersionMarkerStart.Length).IndexOf(ReportVersionMarkerEnd) is var idxVersionEnd and not -1) idxEnd = idxVersionEnd; + if (line.Slice(idxVersionStart + ReportVersionMarkerStart.Length).IndexOf(ReportVersionMarkerEnd2) is var idxVersionEnd2 and not -1) idxEnd = idxVersionEnd2; + if (idxEnd == -1) return false; + + if (!Utf8Parser.TryParse(line.Slice(idxVersionStart + ReportVersionMarkerStart.Length, idxEnd), out version, out _)) return false; + + return true; + } + if (line.Slice(idxIdStart + ReportMarkerStart.Length).IndexOf(ReportIdNoVersionMarkerEnd) is var idxNoVersionStart and not -1) { if (!Utf8Parser.TryParse(line.Slice(idxIdStart + ReportMarkerStart.Length, idxNoVersionStart), out crashReportId, out _)) return false; @@ -80,16 +102,7 @@ private static bool TryReadCrashReportData(ref ReadOnlySequence buffer, ou return true; } - if (line.Slice(idxIdStart + ReportMarkerStart.Length).IndexOf(ReportIdMarkerEnd) is not (var idxIdEnd and not -1)) return false; - - if (!Utf8Parser.TryParse(line.Slice(idxIdStart + ReportMarkerStart.Length, idxIdEnd), out crashReportId, out _)) return false; - - if (line.IndexOf(ReportVersionMarkerStart) is not (var idxVersionStart and not -1)) return false; - if (line.Slice(idxVersionStart + ReportVersionMarkerStart.Length).IndexOf(ReportVersionMarkerEnd) is not (var idxVersionEnd and not -1)) return false; - - if (!Utf8Parser.TryParse(line.Slice(idxVersionStart + ReportVersionMarkerStart.Length, idxVersionEnd), out version, out _)) return false; - - return true; + return false; } [MethodImpl(MethodImplOptions.AggressiveInlining)]