Skip to content

Commit

Permalink
Fixed Exposed Mods pagination
Browse files Browse the repository at this point in the history 
Added a hacky solution to have both items and the count via one round trip
  • Loading branch information
@Aragas
Aragas committed Jun 19, 2024
1 parent db8d25a commit 13e3d8b
Show file tree
Hide file tree
Showing 8 changed files with 66 additions and 22 deletions.
2 changes: 1 addition & 1 deletion ....NexusMods.Server.ValueObjects.Vogen/BUTR.Site.NexusMods.Server.ValueObjects.Vogen.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
<PackageReference Include="BUTR.CrashReport.Models" Version="13.0.0.81" />
<PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.0" />
<PackageReference Include="Swashbuckle.AspNetCore.SwaggerGen" Version="6.5.0" />
<PackageReference Include="Vogen" Version="4.0.6" />
<PackageReference Include="Vogen" Version="4.0.7" />
</ItemGroup>

<ItemGroup>
Expand Down
22 changes: 11 additions & 11 deletions src/BUTR.Site.NexusMods.Server/BUTR.Site.NexusMods.Server.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,46 +17,46 @@
<ItemGroup>
<PackageReference Include="Aragas.Extensions.Options.FluentValidation" Version="1.1.0.14" />
<PackageReference Include="AsmResolver.DotNet" Version="5.5.1" />
<PackageReference Include="Bannerlord.ModuleManager" Version="5.0.222" />
<PackageReference Include="Bannerlord.ModuleManager" Version="5.0.226" />
<PackageReference Include="BUTR.Authentication.NexusMods" Version="1.0.0.9" />
<PackageReference Include="BUTR.CrashReport.Models" Version="13.0.0.81" />
<PackageReference Include="BUTR.CrashReport.Bannerlord.Parser" Version="13.0.0.81" />
<PackageReference Include="Community.Microsoft.Extensions.Caching.PostgreSql" Version="4.0.4" />
<PackageReference Include="CsvHelper" Version="32.0.3" />
<PackageReference Include="CsvHelper" Version="33.0.0" />
<PackageReference Include="EFCore.BulkExtensions.PostgreSql" Version="8.0.4" />
<PackageReference Include="ICSharpCode.Decompiler" Version="9.0.0.7618-preview1" />
<PackageReference Include="JetBrains.Annotations" Version="2023.3.0" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.8.0" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="4.8.0" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.5">
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.6">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Http.Polly" Version="8.0.5" />
<PackageReference Include="Microsoft.Extensions.Http.Polly" Version="8.0.6" />
<PackageReference Include="Nerdbank.Streams" Version="2.11.74" />
<PackageReference Include="Npgsql" Version="8.0.3" />
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.4" />
<PackageReference Include="Npgsql.OpenTelemetry" Version="8.0.3" />
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.8.1" />
<PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.8.1" />
<PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.9.0" />
<PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.9.0" />
<PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.8.1" />
<PackageReference Include="OpenTelemetry.Instrumentation.EntityFrameworkCore" Version="1.0.0-beta.11" />
<PackageReference Include="OpenTelemetry.Instrumentation.GrpcNetClient" Version="1.8.0-beta.1" />
<PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.8.1" />
<PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.9.0" />
<PackageReference Include="OpenTelemetry.Instrumentation.Process" Version="0.5.0-beta.5" />
<PackageReference Include="OpenTelemetry.Instrumentation.Quartz" Version="1.0.0-beta.2" />
<PackageReference Include="OpenTelemetry.Instrumentation.Runtime" Version="1.8.1" />
<PackageReference Include="OpenTelemetry.Instrumentation.Runtime" Version="1.9.0" />
<PackageReference Include="OpenTelemetry.ResourceDetectors.Container" Version="1.0.0-beta.7" />
<PackageReference Include="Polly" Version="8.4.0" />
<PackageReference Include="Polly.Contrib.WaitAndRetry" Version="1.1.1" />
<PackageReference Include="Quartz" Version="3.9.0" />
<PackageReference Include="Quartz.Extensions.DependencyInjection" Version="3.9.0" />
<PackageReference Include="Quartz.Extensions.Hosting" Version="3.9.0" />
<PackageReference Include="RateLimiter" Version="2.2.0" />
<PackageReference Include="Serilog" Version="3.1.1" />
<PackageReference Include="Serilog" Version="4.0.0" />
<PackageReference Include="Serilog.AspNetCore" Version="8.0.1" />
<PackageReference Include="Serilog.Sinks.Async" Version="1.5.0" />
<PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
<PackageReference Include="Serilog.Sinks.Async" Version="2.0.0" />
<PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
<PackageReference Include="SharpCompress" Version="0.37.2" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.6.2" />
<PackageReference Include="Swashbuckle.AspNetCore.Annotations" Version="6.6.2" />
Expand Down
4 changes: 2 additions & 2 deletions src/BUTR.Site.NexusMods.Server/Controllers/ExposedModsController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public ExposedModsController(ILogger<ExposedModsController> logger, IUnitOfWorkF
[HttpPost("Paginated")]
public async Task<ApiResult<PagingData<LinkedByExposureNexusModsModModelsModel>?>> GetPaginatedAsync([FromBody, Required] PaginatedQuery query, CancellationToken ct)
{
await using var unitOfRead = _unitOfWorkFactory.CreateUnitOfRead(TenantId.None);
await using var unitOfRead = _unitOfWorkFactory.CreateUnitOfRead();

var paginated = await unitOfRead.NexusModsModModules.GetExposedPaginatedAsync(query, ct);

Expand All @@ -40,7 +40,7 @@ public ExposedModsController(ILogger<ExposedModsController> logger, IUnitOfWorkF
[HttpGet("Autocomplete/ModuleIds")]
public async Task<ApiResult<IList<string>?>> GetAutocompleteModuleIdsAsync([FromQuery, Required] ModuleId moduleId)
{
await using var unitOfRead = _unitOfWorkFactory.CreateUnitOfRead(TenantId.None);
await using var unitOfRead = _unitOfWorkFactory.CreateUnitOfRead();

var moduleIds = await unitOfRead.Autocompletes.AutocompleteStartsWithAsync<NexusModsModToModuleEntity, ModuleId>(x => x.Module.ModuleId, moduleId, CancellationToken.None);

Expand Down
44 changes: 44 additions & 0 deletions src/BUTR.Site.NexusMods.Server/Extensions/QueryableExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ public static async Task<Paging<TEntity>> PaginatedAsync<TEntity>(this IQueryabl
where TEntity : class
{
var startTime = Stopwatch.GetTimestamp();

var count = await queryable.CountAsync(ct);

return new()
Expand All @@ -89,6 +90,49 @@ public static async Task<Paging<TEntity>> PaginatedAsync<TEntity>(this IQueryabl
}
};
}

public static Task<Paging<TEntity>> PaginatedGroupedAsync<TEntity>(this IQueryable<TEntity> queryable, PaginatedQuery query, uint maxPageSize = 20, Sorting? defaultSorting = default, CancellationToken ct = default)
where TEntity : class
{
var page = query.Page;
var pageSize = Math.Max(Math.Min(query.PageSize, maxPageSize), 5);
var filters = query.Filters ?? Enumerable.Empty<Filtering>();
var sortings = query.Sortings is null || query.Sortings.Count == 0
? defaultSorting == null ? Array.Empty<Sorting>() : new List<Sorting> { defaultSorting }
: query.Sortings;

return queryable
.WithFilter(filters)
.WithSort(sortings)
.PaginatedGroupedAsync(page, pageSize, ct);
}

public static async Task<Paging<TEntity>> PaginatedGroupedAsync<TEntity>(this IQueryable<TEntity> queryable, uint page, uint pageSize, CancellationToken ct = default)
where TEntity : class
{
var startTime = Stopwatch.GetTimestamp();

var response = await queryable.GroupBy(_ => 1)
.Select(x => new
{
PageItems = x.Skip((int) ((page - 1) * pageSize)).Take((int) pageSize).ToList(),
Total = x.Count()
})
.FirstAsync(cancellationToken: ct);

return new()
{
StartTime = startTime,
Items = response.PageItems.ToAsyncEnumerable(),
Metadata = new()
{
PageSize = pageSize,
CurrentPage = page,
TotalCount = (uint) response.Total,
TotalPages = (uint) Math.Floor((double) response.Total / (double) pageSize),
}
};
}

public static Task<ImmutableArray<TSource>> ToImmutableArrayAsync<TSource>(this IQueryable<TSource> source, CancellationToken ct = default)
{
Expand Down
2 changes: 1 addition & 1 deletion src/BUTR.Site.NexusMods.Server/Repositories/CrashReportEntityRepository.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,6 @@ IQueryable<UserCrashReportModel> DbQueryBase(Expression<Func<CrashReportEntity,
x.ModuleInfos.Any(y => moduleIds.Contains(y.Module.ModuleId)) ||
x.ModuleInfos.Where(y => y.NexusModsMod != null).Any(y => nexusModsModIds.Contains(y.NexusModsMod!.NexusModsModId)));

return await dbQuery.PaginatedAsync(query.Page, query.PageSize, ct);
return await dbQuery.PaginatedGroupedAsync(query.Page, query.PageSize, ct);
}
}
4 changes: 2 additions & 2 deletions src/BUTR.Site.NexusMods.Server/Repositories/IRepository.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,12 +129,12 @@ public async Task<IList<TProjection>> GetAllAsync<TProjection>(Expression<Func<T
.ToListAsync(ct);

public Task<Paging<TEntity>> PaginatedAsync(PaginatedQuery query, uint maxPageSize = 20, Sorting? defaultSorting = default, CancellationToken ct = default) => InternalQuery
.PaginatedAsync(query, maxPageSize, defaultSorting, ct);
.PaginatedGroupedAsync(query, maxPageSize, defaultSorting, ct);

public Task<Paging<TProjection>> PaginatedAsync<TProjection>(Expression<Func<TEntity, TProjection>> projection, PaginatedQuery query, uint maxPageSize = 20, Sorting? defaultSorting = default, CancellationToken ct = default)
where TProjection : class => InternalQuery
.Select(projection)
.PaginatedAsync(query, maxPageSize, defaultSorting, ct);
.PaginatedGroupedAsync(query, maxPageSize, defaultSorting, ct);


public virtual void Add(TEntity entity) => _dbContext.Set<TEntity>()
Expand Down
8 changes: 4 additions & 4 deletions src/BUTR.Site.NexusMods.Server/Repositories/NexusModsModToModuleEntityRepository.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,10 @@ public NexusModsModToModuleEntityRepository(IAppDbContextProvider appDbContextPr

public async Task<Paging<LinkedByStaffModuleNexusModsModsModel>> GetByStaffPaginatedAsync(PaginatedQuery query, CancellationToken ct) => await _dbContext.NexusModsModModules
.Where(x => x.LinkType == NexusModsModToModuleLinkType.ByStaff)
.GroupBy(x => x.Module.ModuleId)
.GroupBy(x => new { x.Module.ModuleId })
.Select(x => new LinkedByStaffModuleNexusModsModsModel
{
ModuleId = x.Key,
ModuleId = x.Key.ModuleId,
NexusModsMods = x.Select(y => new LinkedByStaffNexusModsModModel
{
NexusModsModId = y.NexusModsMod.NexusModsModId,
Expand All @@ -71,10 +71,10 @@ public async Task<Paging<LinkedByStaffModuleNexusModsModsModel>> GetByStaffPagin

public async Task<Paging<LinkedByExposureNexusModsModModelsModel>> GetExposedPaginatedAsync(PaginatedQuery query, CancellationToken ct) => await _dbContext.NexusModsModModules
.Where(x => x.LinkType == NexusModsModToModuleLinkType.ByUnverifiedFileExposure)
.GroupBy(x => x.NexusModsMod.NexusModsModId)
.GroupBy(x => new { x.NexusModsMod.NexusModsModId })
.Select(x => new LinkedByExposureNexusModsModModelsModel
{
NexusModsModId = x.Key,
NexusModsModId = x.Key.NexusModsModId,
Modules = x.Select(y => new LinkedByExposureModuleModel
{
ModuleId = y.Module.ModuleId,
Expand Down
2 changes: 1 addition & 1 deletion src/BUTR.Site.NexusMods.Server/Repositories/NexusModsUserRepository.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ public async Task<Paging<UserLinkedModModel>> GetNexusModsModsPaginatedAsync(Nex
});

return await availableModsByNexusModsModLinkage
.PaginatedAsync(query, 20, new() { Property = nameof(UserLinkedModModel.NexusModsModId), Type = SortingType.Ascending }, ct);
.PaginatedGroupedAsync(query, 20, new() { Property = nameof(UserLinkedModModel.NexusModsModId), Type = SortingType.Ascending }, ct);
}

public async Task<Paging<UserAvailableModModel>> GetAvailableModsPaginatedAsync(NexusModsUserId userId, PaginatedQuery query, CancellationToken ct)
Expand Down

0 comments on commit 13e3d8b

Please sign in to comment.