Merge pull request #1140 from ionite34/fixes-n-stuff

Fixed AI Toolkit error on job start, show at least 30 results in civit…

Author: mohnjiles

CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to Stability Matrix will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2.0.0.html).
 
+## v2.15.1
+### Fixed
+- Fixed [#1372](https://github.com/LykosAI/StabilityMatrix/issues/1372) - LiteAsyncException upon starting Stability Matrix v2.15.0
+- Fixed [#1391](https://github.com/LykosAI/StabilityMatrix/issues/1391) - "Failed to parse" error when upgrading pip packages with extra index url
+- Fixed "cannot access local variable 'job' where it is not associated with a value" error when running jobs in AI Toolkit
+- Fixed Civitai browser not always returning at least 30 results when possible on initial search
+
 ## v2.15.0
 ### Added
 - Added new package - [AI Toolkit](https://github.com/ostris/ai-toolkit/)

StabilityMatrix.Avalonia/ViewModels/CheckpointBrowser/CivitAiBrowserViewModel.cs

@@ -52,6 +52,8 @@ public sealed partial class CivitAiBrowserViewModel : TabViewModelBase, IInfinit
 
     private readonly SourceCache<OrderedValue<CivitModel>, int> modelCache = new(static ov => ov.Value.Id);
 
+    private const int TargetPageItemCount = 30;
+
     [ObservableProperty]
     private IObservableCollection<CheckpointBrowserCardViewModel> modelCards =
         new ObservableCollectionExtended<CheckpointBrowserCardViewModel>();
@@ -453,8 +455,46 @@ private async Task CivitModelQuery(CivitModelsRequest request, bool isInfiniteSc
             }
             else
             {
-                modelsResponse = await civitApi.GetModels(request);
-                models = modelsResponse.Items;
+                // Auto-paginate via cursor until we fill the target page size or run out
+                var collectedById = new HashSet<int>();
+                var targetCount = request.Limit ?? TargetPageItemCount;
+                var safetyGuard = 0;
+
+                while (true)
+                {
+                    var resp = await civitApi.GetModels(request);
+                    modelsResponse = resp;
+
+                    if (resp.Items != null)
+                    {
+                        foreach (var item in resp.Items)
+                        {
+                            if (collectedById.Add(item.Id))
+                            {
+                                models.Add(item);
+                            }
+                        }
+                    }
+
+                    // Check how many items survive local filtering
+                    var filteredCount = models
+                        .Where(m => m.Type.ConvertTo<SharedFolderType>() > 0)
+                        .Count(m => m.Mode == null);
+
+                    var next = resp.Metadata?.NextCursor;
+                    if (filteredCount >= targetCount || string.IsNullOrEmpty(next))
+                    {
+                        break;
+                    }
+
+                    request.Cursor = next;
+
+                    if (++safetyGuard >= 10)
+                    {
+                        // Avoid unbounded looping on unexpected cursors
+                        break;
+                    }
+                }
             }
 
             if (models is null)
@@ -634,6 +674,7 @@ private async Task SearchModels(bool isInfiniteScroll = false)
         // Build request
         var modelRequest = new CivitModelsRequest
         {
+            Limit = TargetPageItemCount + 20, // Fetch a few extra to account for local filtering
             Nsfw = "true", // Handled by local view filter
             Sort = SortMode,
             Period = SelectedPeriod,

StabilityMatrix.Avalonia/ViewModels/Dialogs/PythonPackagesViewModel.cs

@@ -239,7 +239,7 @@ private async Task UpgradePackageVersion(
 
         if (extraIndexUrl != null)
         {
-            args = args.AddArg(("--extra-index-url", extraIndexUrl));
+            args = args.AddArgs("--extra-index-url", extraIndexUrl);
         }
 
         var steps = new List<IPackageStep>

StabilityMatrix.Avalonia/ViewModels/Inference/ModelCardViewModel.cs

@@ -309,7 +309,7 @@ public override void LoadStateFromJsonObject(JsonObject state)
             ? HybridModelFile.None
             : ClientManager.ClipModels.FirstOrDefault(x => x.RelativePath == model.SelectedClip3Name);
 
-        SelectedClip4 = model.SelectedClip3Name is null
+        SelectedClip4 = model.SelectedClip4Name is null
             ? HybridModelFile.None
             : ClientManager.ClipModels.FirstOrDefault(x => x.RelativePath == model.SelectedClip4Name);
 

StabilityMatrix.Core/Database/LiteDbContext.cs

@@ -10,6 +10,7 @@
 using StabilityMatrix.Core.Models.Api;
 using StabilityMatrix.Core.Models.Configs;
 using StabilityMatrix.Core.Models.Database;
+using StabilityMatrix.Core.Models.FileInterfaces;
 using StabilityMatrix.Core.Services;
 
 namespace StabilityMatrix.Core.Database;
@@ -63,18 +64,25 @@ IOptions<DebugOptions> debugOptions
 
     private LiteDatabaseAsync CreateDatabase()
     {
-        LiteDatabaseAsync? db = null;
+        // Try at most twice:
+        //  - attempt 0: open/repair if needed
+        //  - on "Detected loop in FindAll": dispose, delete file, try once more
+        const int maxAttempts = 2;
+        var dbPath = Path.Combine(settingsManager.LibraryDir, "StabilityMatrix.db");
 
-        if (debugOptions.TempDatabase)
+        for (var attempt = 0; attempt < maxAttempts; attempt++)
         {
-            db = new LiteDatabaseAsync(":temp:");
-        }
-        else
-        {
-            // Attempt to create connection, might be in use
+            LiteDatabaseAsync? db = null;
+
             try
             {
-                var dbPath = Path.Combine(settingsManager.LibraryDir, "StabilityMatrix.db");
+                if (debugOptions.TempDatabase)
+                {
+                    db = new LiteDatabaseAsync(":temp:");
+                    RegisterRefs();
+                    return db;
+                }
+
                 db = new LiteDatabaseAsync(
                     new ConnectionString { Filename = dbPath, Connection = ConnectionType.Shared }
                 );
@@ -86,33 +94,73 @@ private LiteDatabaseAsync CreateDatabase()
                         "Database collation is not Ordinal ({SortOption}), rebuilding...",
                         sortOption
                     );
-
                     var options = new RebuildOptions
                     {
                         Collation = new Collation(CultureInfo.InvariantCulture.LCID, CompareOptions.Ordinal),
                     };
-
                     db.RebuildAsync(options).GetAwaiter().GetResult();
                 }
+
+                RegisterRefs();
+                return db;
             }
-            catch (IOException e)
+            catch (LiteAsyncException ex)
+                when (ex.InnerException is LiteException e
+                    && e.Message.Contains("Detected loop in FindAll", StringComparison.OrdinalIgnoreCase)
+                )
+            {
+                logger.LogWarning("Database corruption detected ({Message}), rebuilding...", e.Message);
+
+                try
+                {
+                    db?.Dispose();
+                }
+                catch
+                {
+                    // ignored
+                }
+
+                try
+                {
+                    // Backup then delete, in case we want to inspect later.
+                    var corruptPath = dbPath + ".old-" + DateTime.UtcNow.ToString("yyyyMMddHHmmss");
+                    if (File.Exists(dbPath))
+                    {
+                        File.Copy(dbPath, corruptPath, overwrite: false);
+                        File.Delete(dbPath);
+                    }
+                }
+                catch (Exception delEx)
+                {
+                    logger.LogWarning("Failed to delete corrupt DB: {Message}", delEx.Message);
+                    // If we can't delete, no point retrying; break to fallback.
+                    break;
+                }
+            }
+            catch (IOException ioEx)
             {
                 logger.LogWarning(
                     "Database in use or not accessible ({Message}), using temporary database",
-                    e.Message
+                    ioEx.Message
                 );
+                break; // fall through to temp
             }
         }
 
         // Fallback to temporary database
-        db ??= new LiteDatabaseAsync(":temp:");
-
-        // Register reference fields
-        LiteDBExtensions.Register<CivitModel, CivitModelVersion>(m => m.ModelVersions, "CivitModelVersions");
-        LiteDBExtensions.Register<CivitModelQueryCacheEntry, CivitModel>(e => e.Items, "CivitModels");
-        LiteDBExtensions.Register<LocalModelFile, CivitModel>(e => e.LatestModelInfo, "CivitModels");
+        var tempDb = new LiteDatabaseAsync(":temp:");
+        RegisterRefs();
+        return tempDb;
 
-        return db;
+        void RegisterRefs()
+        {
+            LiteDBExtensions.Register<CivitModel, CivitModelVersion>(
+                m => m.ModelVersions,
+                "CivitModelVersions"
+            );
+            LiteDBExtensions.Register<CivitModelQueryCacheEntry, CivitModel>(e => e.Items, "CivitModels");
+            LiteDBExtensions.Register<LocalModelFile, CivitModel>(e => e.LatestModelInfo, "CivitModels");
+        }
     }
 
     public async Task<(CivitModel?, CivitModelVersion?)> FindCivitModelFromFileHashAsync(string hashBlake3)

StabilityMatrix.Core/Models/Packages/AiToolkit.cs

@@ -183,6 +183,9 @@ await npmProcess
 
     private ImmutableDictionary<string, string> GetEnvVars(ImmutableDictionary<string, string> env)
     {
+        // set SETUPTOOLS_USE_DISTUTILS=setuptools to avoid job errors
+        env = env.SetItem("SETUPTOOLS_USE_DISTUTILS", "setuptools");
+
         var pathBuilder = new EnvPathBuilder();
 
         if (env.TryGetValue("PATH", out var value))