Skip to content

Commit

Permalink
Query optimizations
Browse files Browse the repository at this point in the history
  • Loading branch information
@Aragas
Aragas committed Oct 16, 2024
1 parent c0c8829 commit 71e2383
Show file tree
Hide file tree
Showing 19 changed files with 64 additions and 64 deletions.
12 changes: 6 additions & 6 deletions src/BUTR.Site.NexusMods.Server/Controllers/CrashReportsAnalyzerController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,16 +126,16 @@ private async IAsyncEnumerable<ModuleUpdate> GetModuleUpdatesForBannerlordAsync(

// SMAPI uses different update providers - Chucklefish, NexusMods, GitHub
// We curectly will only use NexusMods
//var updateInfoEntries = _dbContextRead.NexusModsModToModuleInfoHistory.Where(x => moduleIds.Contains(x.Module.ModuleId));
//var entries = _dbContextRead.NexusModsModToModuleInfoHistory.Where(x => moduleIds.Contains(x.Module.ModuleId));
var historicEntriesBasedOnModuleId = await unitOfRead.NexusModsModToModuleInfoHistory.GetAllAsync(x => currentModuleIdsWithoutAnyData.Contains(x.Module.ModuleId), null, ct);
var historicEntriesBasedOnNexusModsId = await unitOfRead.NexusModsModToModuleInfoHistory.GetAllAsync(x => currentNexusModsIds.Contains(x.NexusModsMod.NexusModsModId), null, ct);
var historicEntriesBasedOnUpdateInfo = await unitOfRead.NexusModsModToModuleInfoHistory.GetAllAsync(x => currentMexusModsUpdateInfos.Contains(x.NexusModsMod.NexusModsModId), null, ct);
//var updateInfoEntries = _dbContextRead.NexusModsModToModuleInfoHistory.Where(x => moduleIds.Contains(x.ModuleId));
//var entries = _dbContextRead.NexusModsModToModuleInfoHistory.Where(x => moduleIds.Contains(x.ModuleId));
var historicEntriesBasedOnModuleId = await unitOfRead.NexusModsModToModuleInfoHistory.GetAllAsync(x => currentModuleIdsWithoutAnyData.Contains(x.ModuleId), null, ct);
var historicEntriesBasedOnNexusModsId = await unitOfRead.NexusModsModToModuleInfoHistory.GetAllAsync(x => currentNexusModsIds.Contains(x.NexusModsModId), null, ct);
var historicEntriesBasedOnUpdateInfo = await unitOfRead.NexusModsModToModuleInfoHistory.GetAllAsync(x => currentMexusModsUpdateInfos.Contains(x.NexusModsModId), null, ct);
var allHistoricEntries = historicEntriesBasedOnModuleId.Concat(historicEntriesBasedOnNexusModsId).Concat(historicEntriesBasedOnUpdateInfo);

var historicEntriesCompatibleWithGameVersion = allHistoricEntries.Select(x => new
{
ModuleId = x.Module.ModuleId,
ModuleId = x.ModuleId,
ModuleVersion = ApplicationVersion.TryParse(x.ModuleVersion.Value, out var v) ? v : ApplicationVersion.Empty,
ModuleInfo = ModuleInfoModel.Create(x.ModuleInfo),
}).Where(x => x.ModuleInfo.DependentModuleMetadatas.Any(y =>
Expand Down
4 changes: 2 additions & 2 deletions src/BUTR.Site.NexusMods.Server/Controllers/CrashReportsController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ public CrashReportsController(ILogger<CrashReportsController> logger, IUnitOfWor
await using var unitOfWrite = _unitOfWorkFactory.CreateUnitOfWrite();

var existingEntity = await unitOfWrite.NexusModsUserToCrashReports.FirstOrDefaultAsync(
x => x.TenantId == tenant && x.NexusModsUser.NexusModsUserId == userId && x.CrashReportId == crashReportId,
x => x.TenantId == tenant && x.NexusModsUserId == userId && x.CrashReportId == crashReportId,
null, CancellationToken.None);
var entity = new NexusModsUserToCrashReportEntity
{
Expand Down Expand Up @@ -137,7 +137,7 @@ public async Task<StreamingMultipartResult> GetPaginatedStreamingAsync([FromBody
{
await using var unitOfRead = _unitOfWorkFactory.CreateUnitOfRead();

var moduleIds = await unitOfRead.Autocompletes.AutocompleteStartsWithAsync<CrashReportToModuleMetadataEntity, ModuleId>(x => x.Module.ModuleId, moduleId, CancellationToken.None);
var moduleIds = await unitOfRead.Autocompletes.AutocompleteStartsWithAsync<CrashReportToModuleMetadataEntity, ModuleId>(x => x.ModuleId, moduleId, CancellationToken.None);

return ApiResult(moduleIds);
}
Expand Down
2 changes: 1 addition & 1 deletion src/BUTR.Site.NexusMods.Server/Controllers/NexusModsArticleController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ public NexusModsArticleController(ILogger<NexusModsArticleController> logger, IU
{
NexusModsArticleId = x.NexusModsArticleId,
Title = x.Title,
NexusModsUserId = x.NexusModsUser.NexusModsUserId,
NexusModsUserId = x.NexusModsUserId,
Author = x.NexusModsUser.Name != null ? x.NexusModsUser.Name.Name : NexusModsUserName.Empty,
CreateDate = x.CreateDate
},
Expand Down
2 changes: 1 addition & 1 deletion src/BUTR.Site.NexusMods.Server/Controllers/NexusModsModController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ public NexusModsModController(ILogger<NexusModsModController> logger, INexusMods
await using var unitOfWrite = _unitOfWorkFactory.CreateUnitOfWrite();

unitOfWrite.NexusModsModModules
.Remove(x => x.Module.ModuleId == moduleId && x.NexusModsMod.NexusModsModId == modId && x.LinkType == ModToModuleLinkType.ByStaff);
.Remove(x => x.ModuleId == moduleId && x.NexusModsModId == modId && x.LinkType == ModToModuleLinkType.ByStaff);

await unitOfWrite.SaveChangesAsync(CancellationToken.None);
return ApiResult("Unlinked successful!");
Expand Down
10 changes: 5 additions & 5 deletions src/BUTR.Site.NexusMods.Server/Controllers/NexusModsUserController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ public NexusModsUserController(ILogger<NexusModsUserController> logger, INexusMo
await using var unitOfWrite = _unitOfWorkFactory.CreateUnitOfWrite();

unitOfWrite.NexusModsUserToNexusModsMods
.Remove(x => x.NexusModsUser.NexusModsUserId == nexusModsUserId && x.NexusModsMod.NexusModsModId == modId && x.LinkType == NexusModsUserToModLinkType.ByOwner);
.Remove(x => x.NexusModsUserId == nexusModsUserId && x.NexusModsModId == modId && x.LinkType == NexusModsUserToModLinkType.ByOwner);

await unitOfWrite.SaveChangesAsync(CancellationToken.None);
return ApiResult("Unlinked successful!");
Expand Down Expand Up @@ -273,7 +273,7 @@ public NexusModsUserController(ILogger<NexusModsUserController> logger, INexusMo


[HttpPost("SteamWorkshopModsLinks")]
public async Task<ApiResult<string?>> AddSteamWorkshopModsLinkAsync([FromQuery, Required] SteamWorkshopModId modId, [FromQuery] NexusModsUserId? userId, [FromQuery] NexusModsUserName? username, [BindTenant] TenantId tenant, CancellationToken ct)
public async Task<ApiResult<string?>> AddSteamWorkshopModLinkAsync([FromQuery, Required] SteamWorkshopModId modId, [FromQuery] NexusModsUserId? userId, [FromQuery] NexusModsUserName? username, [BindTenant] TenantId tenant, CancellationToken ct)
{
var nexusModsUserId = await GetUserIdAsync(userId, username, ct);
if (nexusModsUserId == NexusModsUserId.None)
Expand Down Expand Up @@ -389,7 +389,7 @@ public NexusModsUserController(ILogger<NexusModsUserController> logger, INexusMo
await using var unitOfWrite = _unitOfWorkFactory.CreateUnitOfWrite();

unitOfWrite.NexusModsUserToSteamWorkshopMods
.Remove(x => x.NexusModsUser.NexusModsUserId == nexusModsUserId && x.SteamWorkshopMod.SteamWorkshopModId == modId && x.LinkType == NexusModsUserToModLinkType.ByOwner);
.Remove(x => x.NexusModsUserId == nexusModsUserId && x.SteamWorkshopModId == modId && x.LinkType == NexusModsUserToModLinkType.ByOwner);

await unitOfWrite.SaveChangesAsync(CancellationToken.None);
return ApiResult("Unlinked successful!");
Expand Down Expand Up @@ -440,7 +440,7 @@ public NexusModsUserController(ILogger<NexusModsUserController> logger, INexusMo
await using var unitOfWrite = _unitOfWorkFactory.CreateUnitOfWrite();

unitOfWrite.NexusModsUserToModules
.Remove(x => x.NexusModsUser.NexusModsUserId == nexusModsUserId && x.Module.ModuleId == moduleId && x.LinkType == NexusModsUserToModuleLinkType.ByStaff);
.Remove(x => x.NexusModsUserId == nexusModsUserId && x.ModuleId == moduleId && x.LinkType == NexusModsUserToModuleLinkType.ByStaff);

await unitOfWrite.SaveChangesAsync(CancellationToken.None);
return ApiResult("Disallowed successful!");
Expand Down Expand Up @@ -537,7 +537,7 @@ public NexusModsUserController(ILogger<NexusModsUserController> logger, INexusMo
await using var unitOfWrite = _unitOfWorkFactory.CreateUnitOfWrite();

unitOfWrite.NexusModsUserToNexusModsMods
.Remove(x => x.NexusModsUser.NexusModsUserId == nexusModsUserId && x.NexusModsMod.NexusModsModId == modId && x.LinkType == NexusModsUserToModLinkType.ByOwner);
.Remove(x => x.NexusModsUserId == nexusModsUserId && x.NexusModsModId == modId && x.LinkType == NexusModsUserToModLinkType.ByOwner);

await unitOfWrite.SaveChangesAsync(CancellationToken.None);
return ApiResult("Disallowed successful!");
Expand Down
2 changes: 1 addition & 1 deletion src/BUTR.Site.NexusMods.Server/Controllers/StatisticsController.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ public StatisticsController(ILogger<StatisticsController> logger, IUnitOfWorkFac
{
await using var unitOfRead = _unitOfWorkFactory.CreateUnitOfRead();

var moduleIds = await unitOfRead.Autocompletes.AutocompleteStartsWithAsync<CrashReportToModuleMetadataEntity, ModuleId>(x => x.Module.ModuleId, moduleId, CancellationToken.None);
var moduleIds = await unitOfRead.Autocompletes.AutocompleteStartsWithAsync<CrashReportToModuleMetadataEntity, ModuleId>(x => x.ModuleId, moduleId, CancellationToken.None);

return ApiResult(moduleIds);
}
Expand Down
6 changes: 3 additions & 3 deletions src/BUTR.Site.NexusMods.Server/Jobs/AutocompleteProcessorProcessorJob.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,14 @@ private sealed record AutocompleteGroupingEntry(string Name, Func<IUnitOfRead, I
private static readonly AutocompleteEntry[] ToAutocomplete =
[
new(GenerateName<CrashReportEntity, GameVersion>(x => x.GameVersion), x => x.CrashReports.GetAllGameVersions().Select(y => y.Value)),
new(GenerateName<CrashReportToModuleMetadataEntity, ModuleId>(x => x.Module.ModuleId), x => x.CrashReportModuleInfos.GetAllModuleIds().Select(y => y.Value)),
new(GenerateName<CrashReportToModuleMetadataEntity, ModuleId>(x => x.ModuleId), x => x.CrashReportModuleInfos.GetAllModuleIds().Select(y => y.Value)),
new(GenerateName<NexusModsArticleEntity, NexusModsUserName>(x => x.NexusModsUser.Name!.Name), x => x.NexusModsArticles.GetAllUserNames().Select(y => y.Value))
];
private static readonly AutocompleteGroupingEntry[] ToAutocompleteGrouping =
[
new(GenerateName<CrashReportToModuleMetadataEntity, ModuleId>(x => x.Module.ModuleId),
x => x.CrashReportModuleInfos.GroupBy(y => y.Module.ModuleId).Select(y => new GroupingEntry { Key = y.Key, Values = y.Select(z => z.Version).Distinct().ToArray() })),
new(GenerateName<CrashReportToModuleMetadataEntity, ModuleId>(x => x.ModuleId),
x => x.CrashReportModuleInfos.GroupBy(y => y.ModuleId).Select(y => new GroupingEntry { Key = y.Key, Values = y.Select(z => z.Version).Distinct().ToArray() })),
];
*/

Expand Down
2 changes: 1 addition & 1 deletion src/BUTR.Site.NexusMods.Server/Jobs/NexusModsModFileUpdatesProcessorJob.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ public async Task Execute(IJobExecutionContext context)
var latestFileUpdateDate = DateTimeOffset.FromUnixTimeSeconds(x.LatestFileUpdateTimestamp).ToUniversalTime();
if (latestFileUpdateDate < dateOneWeekAgo) return false;
var found = updatesStoredWithinWeek.FirstOrDefault(y => y.NexusModsMod.NexusModsModId == x.Id);
var found = updatesStoredWithinWeek.FirstOrDefault(y => y.NexusModsModId == x.Id);
return found is null || found.LastCheckedDate < latestFileUpdateDate;
}).ToList();

Expand Down
14 changes: 7 additions & 7 deletions src/BUTR.Site.NexusMods.Server/Repositories/CrashReportEntityRepository.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,11 +55,11 @@ public async Task GenerateAutoCompleteForGameVersionsAsync(CancellationToken ct)

public async Task<Paging<UserCrashReportModel>> GetCrashReportsPaginatedAsync(NexusModsUserEntity user, PaginatedQuery query, ApplicationRole applicationRole, CancellationToken ct)
{
var nexusModsModIds = user.ToNexusModsMods.Select(x => x.NexusModsMod.NexusModsModId).ToHashSet();
var steamWorkshopModIds = user.ToSteamWorkshopMods.Select(x => x.SteamWorkshopMod.SteamWorkshopModId).ToHashSet();
var nexusModsModIds = user.ToNexusModsMods.Select(x => x.NexusModsModId).ToHashSet();
var steamWorkshopModIds = user.ToSteamWorkshopMods.Select(x => x.SteamWorkshopModId).ToHashSet();
var moduleIds = _dbContext.NexusModsModModules.Where(x => nexusModsModIds.Contains(x.NexusModsModId)).Select(x => x.ModuleId)
.Concat(_dbContext.SteamWorkshopModModules.Where(x => steamWorkshopModIds.Contains(x.SteamWorkshopModId)).Select(x => x.ModuleId))
.Concat(user.ToModules.Select(x => x.Module.ModuleId));
.Concat(user.ToModules.Select(x => x.ModuleId));

IQueryable<UserCrashReportModel> DbQueryBase(Expression<Func<CrashReportEntity, bool>> predicate) => _dbContext.CrashReports
.Include(x => x.ToUsers).ThenInclude(x => x.NexusModsUser)
Expand All @@ -86,17 +86,17 @@ IQueryable<UserCrashReportModel> DbQueryBase(Expression<Func<CrashReportEntity,
TopInvolvedModuleId = x.ModuleInfos.OrderBy(y => y.InvolvedPosition).Where(z => z.IsInvolved).Select(y => y.Module).Select(y => y.ModuleId).Cast<ModuleId?>().FirstOrDefault(),
InvolvedModuleIds = x.ModuleInfos.OrderBy(y => y.InvolvedPosition).Where(z => z.IsInvolved).Select(y => y.Module).Select(y => y.ModuleId).ToArray(),
//NexusModsModIds = x.ModuleInfos.Select(y => y.NexusModsMod).Where(y => y != null).Select(y => y!.NexusModsModId).ToArray(),
Status = x.ToUsers.Where(y => y.NexusModsUser.NexusModsUserId == user.NexusModsUserId).Select(y => y.Status).FirstOrDefault(),
Comment = x.ToUsers.Where(y => y.NexusModsUser.NexusModsUserId == user.NexusModsUserId).Select(y => y.Comment).FirstOrDefault(),
Status = x.ToUsers.Where(y => y.NexusModsUserId == user.NexusModsUserId).Select(y => y.Status).FirstOrDefault(),
Comment = x.ToUsers.Where(y => y.NexusModsUserId == user.NexusModsUserId).Select(y => y.Comment).FirstOrDefault(),
})
.WithFilter(query.Filters ?? [])
.WithSort(query.Sortings ?? []);

var dbQuery = applicationRole == ApplicationRoles.Administrator || applicationRole == ApplicationRoles.Moderator
? DbQueryBase(x => true)
: DbQueryBase(x => x.ToUsers.Any(y => y.NexusModsUser.NexusModsUserId == user.NexusModsUserId) ||
: DbQueryBase(x => x.ToUsers.Any(y => y.NexusModsUserId == user.NexusModsUserId) ||
x.ModuleInfos.Any(y => moduleIds.Contains(y.Module.ModuleId)) ||
x.ModuleInfos.Any(y => moduleIds.Contains(y.ModuleId)) ||
x.ModuleInfos.Any(y => nexusModsModIds.Contains(y.NexusModsMod!.NexusModsModId)) ||
Expand Down
12 changes: 6 additions & 6 deletions src/BUTR.Site.NexusMods.Server/Repositories/CrashReportToModuleMetadataEntityRepository.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,27 +33,27 @@ public CrashReportToModuleMetadataEntityRepository(IAppDbContextProvider appDbCo
public async Task<IList<StatisticsCrashReport>> GetAllStatisticsAsync(CancellationToken ct)
{
var allModVersionsQuery = _dbContext.CrashReportModuleInfos
.GroupBy(x => new { x.Module.ModuleId, x.Version })
.GroupBy(x => new { x.ModuleId, x.Version })
.Select(x => new { x.Key.ModuleId, x.Key.Version })
.Distinct();

var modCountsQuery = _dbContext.CrashReportModuleInfos
.Include(x => x.ToCrashReport!)
.GroupBy(x => new { x.ToCrashReport!.GameVersion, x.Module.ModuleId, x.Version })
.GroupBy(x => new { x.ToCrashReport!.GameVersion, x.ModuleId, x.Version })
.Select(x => new { x.Key.GameVersion, x.Key.ModuleId, x.Key.Version, Count = x.Count() })
.Distinct();

var involvedModCountsQuery = _dbContext.CrashReportModuleInfos
.Include(x => x.ToCrashReport!)
.Where(x => x.IsInvolved)
.GroupBy(x => new { x.ToCrashReport!.GameVersion, x.Module.ModuleId, x.Version })
.GroupBy(x => new { x.ToCrashReport!.GameVersion, x.ModuleId, x.Version })
.Select(x => new { x.Key.GameVersion, x.Key.ModuleId, x.Key.Version, Count = x.Count() })
.Distinct();

var notInvolvedModCountsQuery = _dbContext.CrashReportModuleInfos
.Include(x => x.ToCrashReport!)
.Where(x => !x.IsInvolved)
.GroupBy(x => new { x.ToCrashReport!.GameVersion, x.Module.ModuleId, x.Version })
.GroupBy(x => new { x.ToCrashReport!.GameVersion, x.ModuleId, x.Version })
.Select(x => new { x.Key.GameVersion, x.Key.ModuleId, x.Key.Version, Count = x.Count() })
.Distinct();

Expand All @@ -80,11 +80,11 @@ from allModVersios in allModVersionsQuery
public async Task GenerateAutoCompleteForModuleIdsAsync(CancellationToken ct)
{
var tenant = _tenantContextAccessor.Current;
var key = AutocompleteProcessorProcessorJob.GenerateName<CrashReportToModuleMetadataEntity, ModuleId>(x => x.Module.ModuleId);
var key = AutocompleteProcessorProcessorJob.GenerateName<CrashReportToModuleMetadataEntity, ModuleId>(x => x.ModuleId);

await _dbContext.Autocompletes.Where(x => x.Type == key).ExecuteDeleteAsync(ct);

var data = await _dbContext.CrashReportModuleInfos.Select(y => y.Module.ModuleId.Value).Distinct().Select(x => new AutocompleteEntity
var data = await _dbContext.CrashReportModuleInfos.Select(y => y.ModuleId.Value).Distinct().Select(x => new AutocompleteEntity
{
AutocompleteId = default,
TenantId = tenant,
Expand Down
8 changes: 4 additions & 4 deletions src/BUTR.Site.NexusMods.Server/Repositories/NexusModsModToModuleEntityRepository.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,27 +25,27 @@ public NexusModsModToModuleEntityRepository(IAppDbContextProvider appDbContextPr

public async Task<Paging<LinkedByStaffModuleNexusModsModsModel>> GetByStaffPaginatedAsync(PaginatedQuery query, CancellationToken ct) => await _dbContext.NexusModsModModules
.Where(x => x.LinkType == ModToModuleLinkType.ByStaff)
.GroupBy(x => new { x.Module.ModuleId })
.GroupBy(x => new { x.ModuleId })
.Select(x => new LinkedByStaffModuleNexusModsModsModel
{
ModuleId = x.Key.ModuleId,
NexusModsMods = x.Select(y => new LinkedByStaffNexusModsModModel
{
NexusModsModId = y.NexusModsMod.NexusModsModId,
NexusModsModId = y.NexusModsModId,
LastCheckedDate = y.LastUpdateDate.ToUniversalTime(),
}).ToArray()
})
.PaginatedAsync(query, 100, new() { Property = nameof(LinkedByStaffModuleNexusModsModsModel.ModuleId), Type = SortingType.Ascending }, ct);

public async Task<Paging<LinkedByExposureNexusModsModModelsModel>> GetExposedPaginatedAsync(PaginatedQuery query, CancellationToken ct) => await _dbContext.NexusModsModModules
.Where(x => x.LinkType == ModToModuleLinkType.ByUnverifiedFileExposure)
.GroupBy(x => new { x.NexusModsMod.NexusModsModId })
.GroupBy(x => new { x.NexusModsModId })
.Select(x => new LinkedByExposureNexusModsModModelsModel
{
NexusModsModId = x.Key.NexusModsModId,
Modules = x.Select(y => new LinkedByExposureModuleModel
{
ModuleId = y.Module.ModuleId,
ModuleId = y.ModuleId,
LastCheckedDate = y.LastUpdateDate.ToUniversalTime(),
}).ToArray()
})
Expand Down
Loading

0 comments on commit 71e2383

Please sign in to comment.