Merge pull request LykosAI#1146 from ionite34/dev-to-main

Dev to main

Author: mohnjiles

CHANGELOG.md

@@ -5,6 +5,15 @@ All notable changes to Stability Matrix will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2.0.0.html).
 
+## v2.15.1
+### Changed
+- Upgraded ComfyUI-Zluda and Stable Diffusion WebUI AMDGPU Forge packages to install and use HIP SDK 6.4.2
+### Fixed
+- Fixed [#1372](https://github.com/LykosAI/StabilityMatrix/issues/1372) - LiteAsyncException upon starting Stability Matrix v2.15.0
+- Fixed [#1391](https://github.com/LykosAI/StabilityMatrix/issues/1391) - "Failed to parse" error when upgrading pip packages with extra index url
+- Fixed "cannot access local variable 'job' where it is not associated with a value" error when running jobs in AI Toolkit
+- Fixed Civitai browser not always returning at least 30 results when possible on initial search
+
 ## v2.15.0
 ### Added
 - Added new package - [AI Toolkit](https://github.com/ostris/ai-toolkit/)

StabilityMatrix.Avalonia/Helpers/WindowsPrerequisiteHelper.cs

@@ -45,7 +45,7 @@ IPyInstallationManager pyInstallationManager
     private const string CppBuildToolsUrl = "https://aka.ms/vs/17/release/vs_BuildTools.exe";
 
     private const string HipSdkDownloadUrl =
-        "https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-24.Q4-Win10-Win11-For-HIP.exe";
+        "https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-25.Q3-Win10-Win11-For-HIP.exe";
     private const string PythonLibsDownloadUrl = "https://cdn.lykos.ai/python_libs_for_sage.zip";
 
     private const string UvWindowsDownloadUrl =
@@ -110,7 +110,7 @@ private string GetPythonLibraryZipPath(PyVersion version) =>
     private string HipSdkDownloadPath => Path.Combine(AssetsDir, "AMD-HIP-SDK.exe");
 
     private string HipInstalledPath =>
-        Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles), "AMD", "ROCm", "6.2");
+        Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles), "AMD", "ROCm", "6.4");
 
     private string UvDownloadPath => Path.Combine(AssetsDir, "uv.zip");
     private string UvExtractPath => Path.Combine(AssetsDir, "uv");
@@ -189,7 +189,7 @@ private async Task<string> RunNode(
 
         result.EnsureSuccessExitCode();
         return result.StandardOutput ?? result.StandardError ?? string.Empty;
-    }    
+    }
 
     public async Task RunNpm(
         ProcessArgs args,
@@ -997,15 +997,11 @@ private async Task PatchHipSdkIfNecessary(IProgress<ProgressReport>? progress)
 
         var zipFolderName = downloadUrl switch
         {
-            _ when downloadUrl.Contains("gfx1201") => null,
-            _ when downloadUrl.Contains("gfx1150") => "rocm gfx1150 for hip skd 6.2.4",
-            _ when downloadUrl.Contains("gfx1103.AMD") =>
-                "rocm gfx1103 AMD 780M phoenix V5.0 for hip skd 6.2.4",
-            _ when downloadUrl.Contains("gfx1034") => "rocm gfx1034-gfx1035-gfx1036 for hip sdk 6.2.4",
-            _ when downloadUrl.Contains("gfx1032") => "rocm gfx1032 for hip skd 6.2.4(navi21 logic)",
-            _ when downloadUrl.Contains("gfx1031") => "rocm gfx1031 for hip skd 6.2.4 (littlewu's logic)",
-            _ when downloadUrl.Contains("gfx1010") =>
-                "rocm gfx1010-xnack-gfx1011-xnack-gfx1012-xnack- for hip sdk 6.2.4",
+            _ when downloadUrl.Contains("gfx1103.for.hip") => "rocm gfx1103 for hip 6.4.2",
+            _ when downloadUrl.Contains("gfx1034") => "rocm gfx1034.gfx1035.gfx1036 for hip 6.4.2",
+            _ when downloadUrl.Contains("gfx1032") => "rocm gfx1032 for hip 6.4.2",
+            _ when downloadUrl.Contains("gfx1031") => "rocm gfx1031 for hip 6.4.2",
+            _ when downloadUrl.Contains("gfx1010") => "rocm gfx1010-xnack-gfx1012-xnack-.for.hip6.4.2",
             _ => null,
         };
 
@@ -1035,27 +1031,29 @@ _ when downloadUrl.Contains("gfx1010") =>
 
     private string? GetDownloadUrlFromGpuName(string name)
     {
+        // not used anymore but good to know for gfx -> name reference
         // gfx1201
-        if (name.Contains("9060") || name.Contains("9070"))
-        {
-            return "https://github.com/likelovewant/ROCmLibs-for-gfx1103-AMD780M-APU/releases/download/v0.6.2.4/rocm.gfx1201.for.hip.skd.6.2.4-no-optimized.7z";
-        }
+        // if (name.Contains("9060") || name.Contains("9070"))
+        // {
+        //     return "https://github.com/likelovewant/ROCmLibs-for-gfx1103-AMD780M-APU/releases/download/v0.6.2.4/rocm.gfx1201.for.hip.skd.6.2.4-no-optimized.7z";
+        // }
 
+        // not used anymore but good to know for gfx -> name reference
         // gfx1150
-        if (
-            name.Contains("8050S")
-            || name.Contains("8060S")
-            || name.Contains("880M")
-            || name.Contains("890M")
-        )
-        {
-            return "https://github.com/likelovewant/ROCmLibs-for-gfx1103-AMD780M-APU/releases/download/v0.6.2.4/rocm.gfx1150.for.hip.skd.6.2.4.7z";
-        }
+        // if (
+        //     name.Contains("8050S")
+        //     || name.Contains("8060S")
+        //     || name.Contains("880M")
+        //     || name.Contains("890M")
+        // )
+        // {
+        //     return "https://github.com/likelovewant/ROCmLibs-for-gfx1103-AMD780M-APU/releases/download/v0.6.2.4/rocm.gfx1150.for.hip.skd.6.2.4.7z";
+        // }
 
         // gfx1103
         if (name.Contains("740M") || name.Contains("760M") || name.Contains("780M") || name.Contains("Z1"))
         {
-            return "https://github.com/likelovewant/ROCmLibs-for-gfx1103-AMD780M-APU/releases/download/v0.6.2.4/rocm.gfx1103.AMD.780M.phoenix.V5.0.for.hip.sdk.6.2.4.7z";
+            return "https://github.com/likelovewant/ROCmLibs-for-gfx1103-AMD780M-APU/releases/download/v0.6.4.2/rocm.gfx1103.for.hip.6.4.2.7z";
         }
 
         // gfx1034, gfx1035, gfx1036
@@ -1070,7 +1068,7 @@ _ when downloadUrl.Contains("gfx1010") =>
             || name.Contains("Graphics 128SP")
         )
         {
-            return "https://github.com/likelovewant/ROCmLibs-for-gfx1103-AMD780M-APU/releases/download/v0.6.2.4/rocm.gfx1034-gfx1035-gfx1036.for.hip.sdk.6.2.4.7z";
+            return "https://github.com/likelovewant/ROCmLibs-for-gfx1103-AMD780M-APU/releases/download/v0.6.4.2/rocm.gfx1034.gfx1035.gfx1036.for.hip.6.4.2.7z";
         }
 
         // gfx1032
@@ -1081,7 +1079,7 @@ _ when downloadUrl.Contains("gfx1010") =>
             || name.Contains("6650")
         )
         {
-            return "https://github.com/likelovewant/ROCmLibs-for-gfx1103-AMD780M-APU/releases/download/v0.6.2.4/rocm.gfx1032.for.hip.sdk.6.2.4.navi21.logic.7z";
+            return "https://github.com/likelovewant/ROCmLibs-for-gfx1103-AMD780M-APU/releases/download/v0.6.4.2/rocm.gfx1032.for.hip.6.4.2.7z";
         }
 
         // gfx1031
@@ -1092,13 +1090,13 @@ _ when downloadUrl.Contains("gfx1010") =>
             || name.Contains("6850M")
         )
         {
-            return "https://github.com/likelovewant/ROCmLibs-for-gfx1103-AMD780M-APU/releases/download/v0.6.2.4/rocm.gfx1031.for.hip.sdk.6.2.4.littlewu.s.logic.7z";
+            return "https://github.com/likelovewant/ROCmLibs-for-gfx1103-AMD780M-APU/releases/download/v0.6.4.2/rocm.gfx1031.for.hip.6.4.2.7z";
         }
 
         // gfx1010/1012
         if (name.Contains("5700") || name.Contains("5600") || name.Contains("5300") || name.Contains("5500"))
         {
-            return "https://github.com/likelovewant/ROCmLibs-for-gfx1103-AMD780M-APU/releases/download/v0.6.2.4/rocm.gfx1010-xnack-gfx1011-xnack-gfx1012-xnack-.for.hip.sdk.6.2.4.7z";
+            return "https://github.com/likelovewant/ROCmLibs-for-gfx1103-AMD780M-APU/releases/download/v0.6.4.2/rocm.gfx1010-xnack-gfx1012-xnack-.for.hip6.4.2.7z";
         }
 
         return null;

StabilityMatrix.Avalonia/ViewModels/CheckpointBrowser/CivitAiBrowserViewModel.cs

@@ -52,6 +52,8 @@ public sealed partial class CivitAiBrowserViewModel : TabViewModelBase, IInfinit
 
     private readonly SourceCache<OrderedValue<CivitModel>, int> modelCache = new(static ov => ov.Value.Id);
 
+    private const int TargetPageItemCount = 30;
+
     [ObservableProperty]
     private IObservableCollection<CheckpointBrowserCardViewModel> modelCards =
         new ObservableCollectionExtended<CheckpointBrowserCardViewModel>();
@@ -453,8 +455,46 @@ private async Task CivitModelQuery(CivitModelsRequest request, bool isInfiniteSc
             }
             else
             {
-                modelsResponse = await civitApi.GetModels(request);
-                models = modelsResponse.Items;
+                // Auto-paginate via cursor until we fill the target page size or run out
+                var collectedById = new HashSet<int>();
+                var targetCount = request.Limit ?? TargetPageItemCount;
+                var safetyGuard = 0;
+
+                while (true)
+                {
+                    var resp = await civitApi.GetModels(request);
+                    modelsResponse = resp;
+
+                    if (resp.Items != null)
+                    {
+                        foreach (var item in resp.Items)
+                        {
+                            if (collectedById.Add(item.Id))
+                            {
+                                models.Add(item);
+                            }
+                        }
+                    }
+
+                    // Check how many items survive local filtering
+                    var filteredCount = models
+                        .Where(m => m.Type.ConvertTo<SharedFolderType>() > 0)
+                        .Count(m => m.Mode == null);
+
+                    var next = resp.Metadata?.NextCursor;
+                    if (filteredCount >= targetCount || string.IsNullOrEmpty(next))
+                    {
+                        break;
+                    }
+
+                    request.Cursor = next;
+
+                    if (++safetyGuard >= 10)
+                    {
+                        // Avoid unbounded looping on unexpected cursors
+                        break;
+                    }
+                }
             }
 
             if (models is null)
@@ -634,6 +674,7 @@ private async Task SearchModels(bool isInfiniteScroll = false)
         // Build request
         var modelRequest = new CivitModelsRequest
         {
+            Limit = TargetPageItemCount + 20, // Fetch a few extra to account for local filtering
             Nsfw = "true", // Handled by local view filter
             Sort = SortMode,
             Period = SelectedPeriod,

StabilityMatrix.Avalonia/ViewModels/Dialogs/PythonPackagesViewModel.cs

@@ -239,7 +239,7 @@ private async Task UpgradePackageVersion(
 
         if (extraIndexUrl != null)
         {
-            args = args.AddArg(("--extra-index-url", extraIndexUrl));
+            args = args.AddArgs("--extra-index-url", extraIndexUrl);
         }
 
         var steps = new List<IPackageStep>

StabilityMatrix.Avalonia/ViewModels/Inference/ModelCardViewModel.cs

@@ -309,7 +309,7 @@ public override void LoadStateFromJsonObject(JsonObject state)
             ? HybridModelFile.None
             : ClientManager.ClipModels.FirstOrDefault(x => x.RelativePath == model.SelectedClip3Name);
 
-        SelectedClip4 = model.SelectedClip3Name is null
+        SelectedClip4 = model.SelectedClip4Name is null
             ? HybridModelFile.None
             : ClientManager.ClipModels.FirstOrDefault(x => x.RelativePath == model.SelectedClip4Name);
 

StabilityMatrix.Core/Database/LiteDbContext.cs

@@ -10,6 +10,7 @@
 using StabilityMatrix.Core.Models.Api;
 using StabilityMatrix.Core.Models.Configs;
 using StabilityMatrix.Core.Models.Database;
+using StabilityMatrix.Core.Models.FileInterfaces;
 using StabilityMatrix.Core.Services;
 
 namespace StabilityMatrix.Core.Database;
@@ -63,18 +64,25 @@ IOptions<DebugOptions> debugOptions
 
     private LiteDatabaseAsync CreateDatabase()
     {
-        LiteDatabaseAsync? db = null;
+        // Try at most twice:
+        //  - attempt 0: open/repair if needed
+        //  - on "Detected loop in FindAll": dispose, delete file, try once more
+        const int maxAttempts = 2;
+        var dbPath = Path.Combine(settingsManager.LibraryDir, "StabilityMatrix.db");
 
-        if (debugOptions.TempDatabase)
+        for (var attempt = 0; attempt < maxAttempts; attempt++)
         {
-            db = new LiteDatabaseAsync(":temp:");
-        }
-        else
-        {
-            // Attempt to create connection, might be in use
+            LiteDatabaseAsync? db = null;
+
             try
             {
-                var dbPath = Path.Combine(settingsManager.LibraryDir, "StabilityMatrix.db");
+                if (debugOptions.TempDatabase)
+                {
+                    db = new LiteDatabaseAsync(":temp:");
+                    RegisterRefs();
+                    return db;
+                }
+
                 db = new LiteDatabaseAsync(
                     new ConnectionString { Filename = dbPath, Connection = ConnectionType.Shared }
                 );
@@ -86,33 +94,73 @@ private LiteDatabaseAsync CreateDatabase()
                         "Database collation is not Ordinal ({SortOption}), rebuilding...",
                         sortOption
                     );
-
                     var options = new RebuildOptions
                     {
                         Collation = new Collation(CultureInfo.InvariantCulture.LCID, CompareOptions.Ordinal),
                     };
-
                     db.RebuildAsync(options).GetAwaiter().GetResult();
                 }
+
+                RegisterRefs();
+                return db;
             }
-            catch (IOException e)
+            catch (LiteAsyncException ex)
+                when (ex.InnerException is LiteException e
+                    && e.Message.Contains("Detected loop in FindAll", StringComparison.OrdinalIgnoreCase)
+                )
+            {
+                logger.LogWarning("Database corruption detected ({Message}), rebuilding...", e.Message);
+
+                try
+                {
+                    db?.Dispose();
+                }
+                catch
+                {
+                    // ignored
+                }
+
+                try
+                {
+                    // Backup then delete, in case we want to inspect later.
+                    var corruptPath = dbPath + ".old-" + DateTime.UtcNow.ToString("yyyyMMddHHmmss");
+                    if (File.Exists(dbPath))
+                    {
+                        File.Copy(dbPath, corruptPath, overwrite: false);
+                        File.Delete(dbPath);
+                    }
+                }
+                catch (Exception delEx)
+                {
+                    logger.LogWarning("Failed to delete corrupt DB: {Message}", delEx.Message);
+                    // If we can't delete, no point retrying; break to fallback.
+                    break;
+                }
+            }
+            catch (IOException ioEx)
             {
                 logger.LogWarning(
                     "Database in use or not accessible ({Message}), using temporary database",
-                    e.Message
+                    ioEx.Message
                 );
+                break; // fall through to temp
             }
         }
 
         // Fallback to temporary database
-        db ??= new LiteDatabaseAsync(":temp:");
-
-        // Register reference fields
-        LiteDBExtensions.Register<CivitModel, CivitModelVersion>(m => m.ModelVersions, "CivitModelVersions");
-        LiteDBExtensions.Register<CivitModelQueryCacheEntry, CivitModel>(e => e.Items, "CivitModels");
-        LiteDBExtensions.Register<LocalModelFile, CivitModel>(e => e.LatestModelInfo, "CivitModels");
+        var tempDb = new LiteDatabaseAsync(":temp:");
+        RegisterRefs();
+        return tempDb;
 
-        return db;
+        void RegisterRefs()
+        {
+            LiteDBExtensions.Register<CivitModel, CivitModelVersion>(
+                m => m.ModelVersions,
+                "CivitModelVersions"
+            );
+            LiteDBExtensions.Register<CivitModelQueryCacheEntry, CivitModel>(e => e.Items, "CivitModels");
+            LiteDBExtensions.Register<LocalModelFile, CivitModel>(e => e.LatestModelInfo, "CivitModels");
+        }
     }
 
     public async Task<(CivitModel?, CivitModelVersion?)> FindCivitModelFromFileHashAsync(string hashBlake3)

StabilityMatrix.Core/Models/Packages/AiToolkit.cs

@@ -183,6 +183,9 @@ await npmProcess
 
     private ImmutableDictionary<string, string> GetEnvVars(ImmutableDictionary<string, string> env)
     {
+        // set SETUPTOOLS_USE_DISTUTILS=setuptools to avoid job errors
+        env = env.SetItem("SETUPTOOLS_USE_DISTUTILS", "setuptools");
+
         var pathBuilder = new EnvPathBuilder();
 
         if (env.TryGetValue("PATH", out var value))