Merge pull request #720 from LykosAI/main

v2.11.3

.backportrc.json

@@ -0,0 +1,8 @@
+{
+  "sourceBranch": "dev",
+  "targetBranch": "main",
+  "mainline": 1,
+  "fork": false,
+  "targetPRLabels": ["backport"],
+  "prTitle": "[{{sourceBranch}} to {{targetBranch}}] backport: {{sourcePullRequest.title}} ({{sourcePullRequest.number}})"
+}

.github/workflows/backport.yml

@@ -10,14 +10,21 @@ jobs:
     name: Backport PR
     runs-on: ubuntu-latest
     steps:
+      # Get the merge target branch to decide mainline number
+      # git cherry-pick mainline is 1 for merge to 'dev', else 2
+      - name: Get target branch
+        run: echo "CP_MAINLINE=$(if [ '${{ github.event.pull_request.base.ref }}' == 'dev' ]; then echo 1; else echo 2; fi)" >> $GITHUB_ENV
+
       - name: Write json
         id: create-json
         uses: jsdaniell/[email protected]
         with:
           name: ".backportrc.json"
           json: |
             {
-              "targetPRLabels": "backport",
+              "targetPRLabels": ["backport"],
+              "mainline": ${{ env.CP_MAINLINE }},
+              "commitConflicts": "true",
               "prTitle": "[{{sourceBranch}} to {{targetBranch}}] backport: {{sourcePullRequest.title}} ({{sourcePullRequest.number}})"
             }
         

CHANGELOG.md

@@ -5,6 +5,23 @@ All notable changes to Stability Matrix will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2.0.0.html).
 
+
+## v2.11.3
+### Changed
+- Base Python install will now use `pip>=23.3.2,<24.1` for compatibility with `torchsde`.Individual Packages can upgrade as required.
+- Added default `PIP_DISABLE_PIP_VERSION_CHECK=1` environment variable to suppress notices about pip version checks.
+  - As with other default environment variables, this can be overridden by setting your own value in `Settings > Environment Variables [Edit]`.
+### Fixed
+- Fooocus Package - Added `pip>=23.3.2,<24.1` specifier before install, fixes potential install errors due to deprecated requirement spec used by `torchsde`.
+- Fixed error when launching SwarmUI when installed to a path with spaces
+- Fixed issue where model folders were being created too late in certain cases
+- Fixed [#683](https://github.com/LykosAI/StabilityMatrix/issues/683) - Model indexing causing LiteDB errors after upgrading from older versions due to updated enum values
+### Supporters
+#### Visionaries
+- Huge thanks to our Visionary-tier supporters on Patreon, **Scopp Mcdee** and **Waterclouds**! Your support helps us continue to improve Stability Matrix!
+#### Pioneers
+- Thank you to our Pioneer-tier supporters on Patreon, **tankfox** and **tanangular**! Your support is greatly appreciated!
+
 ## v2.11.2
 ### Changed
 - StableSwarmUI installs will be migrated to SwarmUI by mcmonkeyprojects the next time the package is updated

StabilityMatrix.Avalonia/ViewModels/CheckpointsPageViewModel.cs

@@ -295,15 +295,6 @@ or nameof(SortConnectedModelsFirst)
             true
         );
 
-        try
-        {
-            SharedFolders.SetupSharedModelFolders(settingsManager.ModelsDirectory);
-        }
-        catch (Exception e)
-        {
-            logger.LogError(e, @"Failed to setup shared model folders");
-        }
-
         Refresh().SafeFireAndForget();
 
         EventManager.Instance.ModelIndexChanged += (_, _) =>

StabilityMatrix.Avalonia/ViewModels/MainWindowViewModel.cs

@@ -5,6 +5,7 @@
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
+using AsyncAwaitBestPractices;
 using Avalonia.Controls;
 using Avalonia.Controls.Primitives;
 using Avalonia.Threading;
@@ -115,6 +116,12 @@ protected override async Task OnInitialLoadedAsync()
             return;
         }
 
+        Task.Run(() => SharedFolders.SetupSharedModelFolders(settingsManager.ModelsDirectory))
+            .SafeFireAndForget(ex =>
+            {
+                Logger.Error(ex, "Error setting up shared model folders");
+            });
+
         try
         {
             await modelDownloadLinkHandler.Value.StartListening();

StabilityMatrix.Core/Database/ILiteDbContext.cs

@@ -21,4 +21,9 @@ public interface ILiteDbContext : IDisposable
     Task<bool> UpsertCivitModelQueryCacheEntryAsync(CivitModelQueryCacheEntry entry);
     Task<GithubCacheEntry?> GetGithubCacheEntry(string cacheKey);
     Task<bool> UpsertGithubCacheEntry(GithubCacheEntry cacheEntry);
+
+    /// <summary>
+    /// Clear all Collections that store re-fetchable cache type data.
+    /// </summary>
+    Task ClearAllCacheCollectionsAsync();
 }

StabilityMatrix.Core/Database/LiteDbContext.cs

@@ -174,6 +174,30 @@ public async Task<bool> UpsertCivitModelQueryCacheEntryAsync(CivitModelQueryCach
     public Task<bool> UpsertGithubCacheEntry(GithubCacheEntry cacheEntry) =>
         GithubCache.UpsertAsync(cacheEntry);
 
+    /// <summary>
+    /// Clear all Collections that store re-fetchable cache type data.
+    /// </summary>
+    public async Task ClearAllCacheCollectionsAsync()
+    {
+        var collectionNames = new List<string>
+        {
+            nameof(CivitModels),
+            nameof(CivitModelVersions),
+            nameof(CivitModelQueryCache),
+            nameof(GithubCache),
+            nameof(LocalModelFiles),
+            nameof(LocalImageFiles)
+        };
+
+        logger.LogInformation("Clearing all cache collections: [{@Names}]", collectionNames);
+
+        foreach (var name in collectionNames)
+        {
+            var collection = Database.GetCollection(name);
+            await collection.DeleteAllAsync().ConfigureAwait(false);
+        }
+    }
+
     public void Dispose()
     {
         if (lazyDatabase.IsValueCreated)

StabilityMatrix.Core/Models/Packages/Fooocus.cs

@@ -273,6 +273,9 @@ public override async Task InstallPackage(
 
         progress?.Report(new ProgressReport(-1f, "Installing requirements...", isIndeterminate: true));
 
+        // Pip version 24.1 deprecated numpy requirement spec used by torchsde 0.2.5
+        await venvRunner.PipInstall(["pip>=23.3.2,<24.1"], onConsoleOutput).ConfigureAwait(false);
+
         var pipArgs = new PipInstallArgs();
 
         if (torchVersion == TorchVersion.DirectMl)

StabilityMatrix.Core/Models/Packages/StableSwarm.cs

@@ -280,7 +280,7 @@ void HandleConsoleOutput(ProcessOutput s)
 
         dotnetProcess = await prerequisiteHelper
             .RunDotnet(
-                args: $"{Path.Combine(releaseFolder, dllName)} {arguments.TrimEnd()}",
+                args: [Path.Combine(releaseFolder, dllName), arguments.TrimEnd()],
                 workingDirectory: installedPackagePath,
                 envVars: aspEnvVars,
                 onProcessOutput: HandleConsoleOutput,

StabilityMatrix.Core/Models/Settings/Settings.cs

@@ -127,7 +127,13 @@ public InstalledPackage? PreferredWorkflowPackage
 
     [JsonIgnore]
     public Dictionary<string, string> DefaultEnvironmentVariables { get; } =
-        new() { ["SETUPTOOLS_USE_DISTUTILS"] = "stdlib" };
+        new()
+        {
+            // Fixes potential setuptools error on Portable Windows Python
+            ["SETUPTOOLS_USE_DISTUTILS"] = "stdlib",
+            // Suppresses 'A new release of pip is available' messages
+            ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
+        };
 
     [JsonPropertyName("EnvironmentVariables")]
     public Dictionary<string, string>? UserEnvironmentVariables { get; set; }

StabilityMatrix.Core/Python/PyRunner.cs

@@ -116,10 +116,17 @@ public async Task SetupPip()
             throw new FileNotFoundException("get-pip not found", GetPipPath);
         }
 
-        var result = await ProcessRunner
-            .GetProcessResultAsync(PythonExePath, "-m get-pip")
+        await ProcessRunner
+            .GetProcessResultAsync(PythonExePath, ["-m", "get-pip"])
+            .EnsureSuccessExitCode()
+            .ConfigureAwait(false);
+
+        // Pip version 24.1 deprecated numpy star requirement spec used by some packages
+        // So make the base pip less than that for compatibility, venvs can upgrade themselves if needed
+        await ProcessRunner
+            .GetProcessResultAsync(PythonExePath, ["-m", "pip", "install", "pip>=23.3.2,<24.1"])
+            .EnsureSuccessExitCode()
             .ConfigureAwait(false);
-        result.EnsureSuccessExitCode();
     }
 
     /// <summary>

StabilityMatrix.Core/Services/ModelIndexService.cs

@@ -2,10 +2,11 @@
 using System.Collections.Immutable;
 using System.Diagnostics;
 using System.Text;
-using System.Text.Json;
 using AsyncAwaitBestPractices;
 using AutoCtor;
 using KGySoft.CoreLibraries;
+using LiteDB;
+using LiteDB.Async;
 using Microsoft.Extensions.Logging;
 using StabilityMatrix.Core.Attributes;
 using StabilityMatrix.Core.Database;
@@ -15,6 +16,7 @@
 using StabilityMatrix.Core.Models.Api;
 using StabilityMatrix.Core.Models.Database;
 using StabilityMatrix.Core.Models.FileInterfaces;
+using JsonSerializer = System.Text.Json.JsonSerializer;
 
 namespace StabilityMatrix.Core.Services;
 
@@ -103,9 +105,34 @@ private async Task LoadFromDbAsync()
 
         logger.LogInformation("Loading models from database...");
 
-        var allModels = (
-            await liteDbContext.LocalModelFiles.IncludeAll().FindAllAsync().ConfigureAwait(false)
-        ).ToImmutableArray();
+        ImmutableArray<LocalModelFile> allModels = [];
+        try
+        {
+            allModels =
+            [
+                ..(
+                    await liteDbContext.LocalModelFiles.IncludeAll().FindAllAsync().ConfigureAwait(false)
+                )
+            ];
+        }
+        catch (Exception e)
+        {
+            // Handle enum deserialize exceptions from changes
+            if (e is LiteException or LiteAsyncException && e.InnerException is ArgumentException inner)
+            {
+                logger.LogWarning(
+                    e,
+                    "LiteDb Deserialize error while fetching LocalModelFiles '{Inner}', cache collections will be cleared",
+                    inner.ToString()
+                );
+
+                await liteDbContext.ClearAllCacheCollectionsAsync().ConfigureAwait(false);
+            }
+            else
+            {
+                throw;
+            }
+        }
 
         ModelIndex = allModels.GroupBy(m => m.SharedFolderType).ToDictionary(g => g.Key, g => g.ToList());
 
@@ -479,12 +506,34 @@ private async Task RefreshIndexParallelCore()
 
             if (model.LatestModelInfo == null && model.HasConnectedModel)
             {
-                var civitModel = await liteDbContext
-                    .CivitModels.Include(m => m.ModelVersions)
-                    .FindByIdAsync(model.ConnectedModelInfo.ModelId)
-                    .ConfigureAwait(false);
+                try
+                {
+                    model.LatestModelInfo = await liteDbContext
+                        .CivitModels.Include(m => m.ModelVersions)
+                        .FindByIdAsync(model.ConnectedModelInfo.ModelId)
+                        .ConfigureAwait(false);
+                }
+                catch (Exception e)
+                {
+                    // Handle enum deserialize exceptions from changes
+                    if (
+                        e is LiteException or LiteAsyncException
+                        && e.InnerException is ArgumentException inner
+                    )
+                    {
+                        logger.LogWarning(
+                            e,
+                            "LiteDb Deserialize error while fetching CivitModels '{Inner}', cache collections will be cleared",
+                            inner.ToString()
+                        );
 
-                model.LatestModelInfo = civitModel;
+                        await liteDbContext.ClearAllCacheCollectionsAsync().ConfigureAwait(false);
+                    }
+                    else
+                    {
+                        throw;
+                    }
+                }
             }
             var list = newIndex.GetOrAdd(model.SharedFolderType);
             list.Add(model);