diff --git a/Common.cs b/Common.cs index 7ec4eaa..d896361 100644 --- a/Common.cs +++ b/Common.cs @@ -198,6 +198,19 @@ private void ReleaseAndMaybeRemove(Guid key, SemaphoreSlim sem) } } +public static class StringExtensions +{ + public static string FirstCharToUpperInvariant(this string input) + { + if (string.IsNullOrEmpty(input)) + { + return input; + } + + return char.ToUpperInvariant(input[0]) + input[1..]; + } +} + public static class EnumMappingExtensions { public static StremioMediaType ToStremio(this BaseItemKind kind) diff --git a/Config/ConfigurationHelper.cs b/Config/ConfigurationHelper.cs new file mode 100644 index 0000000..e20a68d --- /dev/null +++ b/Config/ConfigurationHelper.cs @@ -0,0 +1,17 @@ +namespace Gelato.Config +{ + internal static class ConfigurationHelper + { + public static CatalogConfig? GetCatalogConfig(string id, string type) + { + return GelatoPlugin.Instance!.Configuration.Catalogs.FirstOrDefault(c => + c.Id == id && c.Type == type + ); + } + + public static PluginConfiguration GetConfig(Guid? userId = null) + { + return GelatoPlugin.Instance!.GetConfig(userId ?? Guid.Empty); + } + } +} diff --git a/GelatoManager.cs b/GelatoManager.cs index c5d0aff..a2de9dc 100644 --- a/GelatoManager.cs +++ b/GelatoManager.cs @@ -102,29 +102,15 @@ private static void SeedFolder(string path) public Folder? TryGetMovieFolder(Guid userId) { - return TryGetFolder( - GelatoPlugin.Instance!.Configuration.GetEffectiveConfig(userId).MoviePath - ); + return TryGetConfigFolder(GelatoPlugin.Instance!.Configuration.GetEffectiveConfig(userId).MoviePath); } public Folder? TryGetSeriesFolder(Guid userId) { - return TryGetFolder( - GelatoPlugin.Instance!.Configuration.GetEffectiveConfig(userId).SeriesPath - ); - } - - public Folder? TryGetMovieFolder(PluginConfiguration cfg) - { - return TryGetFolder(cfg.MoviePath); - } - - public Folder? TryGetSeriesFolder(PluginConfiguration cfg) - { - return TryGetFolder(cfg.SeriesPath); + return TryGetConfigFolder(GelatoPlugin.Instance!.Configuration.GetEffectiveConfig(userId).SeriesPath); } - private Folder? TryGetFolder(string path) + public Folder? TryGetConfigFolder(string path) { if (string.IsNullOrWhiteSpace(path)) { diff --git a/Plugin.cs b/Plugin.cs index 064d681..b709443 100644 --- a/Plugin.cs +++ b/Plugin.cs @@ -85,8 +85,19 @@ public PluginConfiguration GetConfig(Guid userId) } var stremio = _stremioFactory.Create(cfg); cfg.Stremio = stremio; - cfg.MovieFolder = _manager.TryGetMovieFolder(cfg); - cfg.SeriesFolder = _manager.TryGetSeriesFolder(cfg); + + cfg.MovieFolder = _manager.TryGetConfigFolder(cfg.MoviePath); + if (cfg.MovieFolder is null) + { + _log.LogError($"Unable to retrieve movie folder for user '{userId}', Gelato will not be able to function properly! Please add the path for defined in Gelato setting to a library, start a library scan and restart Jellyfin."); + } + + cfg.SeriesFolder = _manager.TryGetConfigFolder(cfg.SeriesPath); + if (cfg.SeriesFolder is null) + { + _log.LogError($"Unable to retrieve series folder for user '{userId}', Gelato will not be able to function properly! Please add the path for defined in Gelato setting to a library, start a library scan and restart Jellyfin."); + } + return cfg; } ); diff --git a/Services/CatalogImportService.cs b/Services/CatalogImportService.cs index a4fa19e..f425359 100644 --- a/Services/CatalogImportService.cs +++ b/Services/CatalogImportService.cs @@ -1,5 +1,3 @@ -using System.Collections.Concurrent; -using System.Diagnostics; using Gelato.Config; using Jellyfin.Data.Enums; using MediaBrowser.Controller.Collections; @@ -7,8 +5,7 @@ using MediaBrowser.Controller.Entities.Movies; using MediaBrowser.Controller.Library; using Microsoft.Extensions.Logging; - -// For BoxSet +using System.Diagnostics; namespace Gelato.Services; @@ -17,172 +14,219 @@ public class CatalogImportService( GelatoManager manager, CatalogService catalogService, ICollectionManager collectionManager, - ILibraryManager libraryManager -) + ILibraryManager libraryManager) { + private const string ProviderKey = "Stremio"; + public async Task ImportCatalogAsync( string catalogId, string type, CancellationToken ct, - IProgress? progress = null - ) + IProgress? progress = null) { - var catalogCfg = catalogService.GetCatalogConfig(catalogId, type); + var catalogCfg = ConfigurationHelper.GetCatalogConfig(catalogId, type); if (catalogCfg == null) { - logger.LogWarning("Catalog config not found for {Id} {Type}", catalogId, type); + logger.LogWarning("Catalog config not found for {CatalogId} {Type}", catalogId, type); return; } if (!catalogCfg.Enabled) { - logger.LogInformation("Catalog {Id} {Type} is disabled, skipping.", catalogId, type); + logger.LogInformation("Catalog {CatalogId} {Type} is disabled, skipping.", catalogId, type); return; } - var cfg = GelatoPlugin.Instance!.GetConfig(Guid.Empty); - var stremio = cfg.Stremio; - var seriesFolder = cfg.SeriesFolder; - var movieFolder = cfg.MovieFolder; - if (seriesFolder is null) + if (catalogCfg.MaxItems <= 0) { - logger.LogWarning("No series root folder found"); + logger.LogWarning( + "{MaxItemsName} for {Name} (ID: {CatalogId}) ({Type}) is {MaxItems}, skipping.", + nameof(catalogCfg.MaxItems), + catalogCfg.Name, + catalogId, + type, + catalogCfg.MaxItems); + return; } - if (movieFolder is null) + var cfg = ConfigurationHelper.GetConfig(); + var catalogName = $"{catalogCfg.Name} {catalogCfg.Type}"; + + var stremio = cfg.Stremio; + if (stremio == null) { - logger.LogWarning("No movie root folder found"); + logger.LogError("Unable to retrieve AIOStreams from configuration, aborting Import. Please check the AIOStreams URL in your settings."); + return; } - var maxItems = catalogCfg.MaxItems; - long done = 0; - - var stopwatch = Stopwatch.StartNew(); logger.LogInformation( - "Starting import for catalog {Name} ({Id}) - Limit: {Limit}", - catalogCfg.Name, - catalogId, - maxItems + "Starting import for catalog {CatalogName} ({CatalogId}) ({Type}) - Max Items: {MaxItems}", + catalogName, catalogId, type, catalogCfg.MaxItems ); + var stopwatch = Stopwatch.StartNew(); + + var skip = 0; + var processedItems = 0; + var maxItems = catalogCfg.MaxItems; + + var allSeenLibraryCatalogItems = new List(); + + var stuckPages = 0; + const int maxStuckPages = 2; + string[]? lastPageCatalogItemIds = null; + try { - var skip = 0; - var processedItems = 0; - var importedIds = new ConcurrentBag(); - while (processedItems < maxItems) { ct.ThrowIfCancellationRequested(); - var page = await stremio + var catalogItems = await stremio .GetCatalogMetasAsync(catalogId, type, search: null, skip: skip) .ConfigureAwait(false); - if (page.Count == 0) + if (catalogItems.Count == 0) { break; } - foreach (var meta in page) + var catalogPageItemIds = catalogItems + .Select(x => x.Id) + .Where(id => !string.IsNullOrWhiteSpace(id)) + .ToArray(); + + if (lastPageCatalogItemIds != null && lastPageCatalogItemIds.SequenceEqual(catalogPageItemIds)) + { + if (stuckPages >= maxStuckPages) + { + logger.LogWarning( + "Stopping import for {CatalogName} {CatalogId} ({Type}) to avoid loop.", + catalogName, catalogId, type); + break; + } + + stuckPages++; + + logger.LogWarning( + "Item retrieval for {CatalogName} {CatalogId} ({Type}) appears stuck, attemping retry {Attempt}/{MaxAttempt}.", + catalogName, catalogId, type, stuckPages, maxStuckPages); + + continue; + } + else + { + stuckPages = 0; + lastPageCatalogItemIds = catalogPageItemIds; + } + + foreach (var catalogItemMetadata in catalogItems) { ct.ThrowIfCancellationRequested(); + if (processedItems >= maxItems) { break; } - var mediaType = meta.Type; - var baseItemKind = mediaType.ToBaseItem(); + if (string.IsNullOrWhiteSpace(catalogItemMetadata.Id)) + { + continue; + } - // catalog can contain multiple types. + // catalog can contain multiple types + var (libraryFolder, baseItemKind) = GetLibraryFolder(catalogItemMetadata, cfg.SeriesFolder, cfg.MovieFolder); - var root = baseItemKind switch + if (libraryFolder is null) { - BaseItemKind.Series => seriesFolder, - BaseItemKind.Movie => movieFolder, - _ => null, - }; + continue; + } - if (root is not null) + try { - try - { - var (item, _) = await manager - .InsertMeta( - root, - meta, - null, - true, - true, - baseItemKind == BaseItemKind.Series, - ct - ) - .ConfigureAwait(false); - - if (item != null) - importedIds.Add(item.Id); - } - catch (Exception ex) + var (item, isNewLibraryItem) = await manager.InsertMeta( + parent: libraryFolder, + meta: catalogItemMetadata, + user: null, + allowRemoteRefresh: true, + refreshItem: true, + queueRefreshItem: baseItemKind == BaseItemKind.Series, + ct).ConfigureAwait(false); + + if (item is null) { - logger.LogError( - "{CatId}: insert meta failed for {Id}. Exception: {Message}\n{StackTrace}", - catalogId, - meta.Id, - ex.Message, - ex.StackTrace - ); + continue; } - } - processedItems++; - progress?.Report(processedItems * 100.0 / maxItems); + allSeenLibraryCatalogItems.Add(item); + + // This will make the method run until it imported the configured amount + // of items that were not present in the library yet, making it possible + // to gradually import the whole catalog. + //if (isNewLibraryItem) + //{ + // continue; + //} + + processedItems++; + progress?.Report(processedItems * 100.0 / catalogCfg.MaxItems); + } + catch (Exception ex) + { + logger.LogError(ex, "{CatalogId}: insert meta failed for {MetadataId}",catalogId, catalogItemMetadata.Id); + } } - skip += page.Count; + skip += catalogItems.Count; } if (catalogCfg.CreateCollection) { - await UpdateCollectionAsync(catalogCfg, importedIds.Take(100).ToList()) - .ConfigureAwait(false); - importedIds.Clear(); + await UpdateOrCreateCollectionAsync(cfg, catalogCfg, 100, allSeenLibraryCatalogItems.ToArray()).ConfigureAwait(false); // Hard cap of 100 items for a collection } - logger.LogInformation("{Id}: processed ({Count} items)", catalogCfg.Id, processedItems); + logger.LogInformation("{CatalogName}: processed ({Count} items)", catalogName, processedItems); } catch (OperationCanceledException ex) { - logger.LogWarning( - ex, - "Catalog {Id} aborted due to non-user cancellation, continuing with next catalog", - catalogId - ); + logger.LogWarning(ex, "Import for {CatalogName} aborted due to non-user cancellation!", catalogName); + throw; } catch (Exception ex) { - logger.LogError( - ex, - "Catalog sync failed for {Id}: {Message}", - catalogCfg.Id, - ex.Message - ); + logger.LogError(ex, "Catalog sync failed for {CatalogName}: {Message}", catalogName, ex.Message); } + finally + { + stopwatch.Stop(); + progress?.Report(100); - stopwatch.Stop(); - progress?.Report(100); - logger.LogInformation( - "Catalog {catalog} sync completed in {Minutes}m {Seconds}s ({TotalSeconds:F2}s total)", - catalogCfg.Name, - (int)stopwatch.Elapsed.TotalMinutes, - stopwatch.Elapsed.Seconds, - stopwatch.Elapsed.TotalSeconds - ); + logger.LogInformation( + "Catalog {catalog} sync completed in {Minutes}m {Seconds}s ({TotalSeconds:F2}s total)", + catalogName, + (int)stopwatch.Elapsed.TotalMinutes, + stopwatch.Elapsed.Seconds, + stopwatch.Elapsed.TotalSeconds); + } + } + + private (Folder?, BaseItemKind) GetLibraryFolder(StremioMeta catalogItemMetadata, Folder? seriesFolder, Folder? moviesFolder) + { + var baseItemKind = catalogItemMetadata.Type.ToBaseItem(); + var libraryFolder = baseItemKind switch + { + BaseItemKind.Series => seriesFolder, + BaseItemKind.Movie => moviesFolder, + _ => null, + }; + + return (libraryFolder, baseItemKind); } private async Task GetOrCreateBoxSetAsync(CatalogConfig config) { - var id = $"{config.Type}.{config.Id}"; + var providerValue = $"{config.Type}.{config.Id}"; + var collection = libraryManager .GetItemList( new InternalItemsQuery @@ -190,7 +234,7 @@ await UpdateCollectionAsync(catalogCfg, importedIds.Take(100).ToList()) IncludeItemTypes = [BaseItemKind.BoxSet], CollapseBoxSetItems = false, Recursive = true, - HasAnyProviderId = new Dictionary { { "Stremio", id } }, + HasAnyProviderId = new Dictionary { { ProviderKey, providerValue } }, } ) .OfType() @@ -202,9 +246,9 @@ await UpdateCollectionAsync(catalogCfg, importedIds.Take(100).ToList()) .CreateCollectionAsync( new CollectionCreationOptions { - Name = config.Name, + Name = $"{config.Name} ({config.Type.FirstCharToUpperInvariant()})", IsLocked = true, - ProviderIds = new Dictionary { { "Stremio", id } }, + ProviderIds = new Dictionary { { ProviderKey, providerValue } }, } ) .ConfigureAwait(false); @@ -217,38 +261,66 @@ await collection return collection; } - private async Task UpdateCollectionAsync(CatalogConfig config, List ids) + private async Task UpdateOrCreateCollectionAsync( + PluginConfiguration pluginConfig, + CatalogConfig catalogConfig, + int maxCollectionItemsAllowed, + BaseItem[] allRetrievedCatalogItems) { - logger.LogInformation( - "Updating collection {Name} with {Count} items", - config.Name, - ids.Count - ); + var catalogName = $"{catalogConfig.Name} {catalogConfig.Type}"; + try { - var collection = await GetOrCreateBoxSetAsync(config).ConfigureAwait(false); - if (collection != null) + var collection = await GetOrCreateBoxSetAsync(catalogConfig).ConfigureAwait(false); + if (collection is null) { - var currentChildren = libraryManager - .GetItemList(new InternalItemsQuery { Parent = collection, Recursive = false }) - .Select(i => i.Id) - .ToList(); + logger.LogError("Unable to retrieve or create Jellyfin collection for {CatalogName}, catalog will be skipped.", catalogName); + return; + } - if (currentChildren.Count != 0) - { - await collectionManager - .RemoveFromCollectionAsync(collection.Id, currentChildren) - .ConfigureAwait(false); - } + var currentChildren = collection.GetLinkedChildren().Select(i => i.Id).ToArray(); + var currentChildrenSet = new HashSet(currentChildren); - await collectionManager - .AddToCollectionAsync(collection.Id, ids) - .ConfigureAwait(false); + var newItems = allRetrievedCatalogItems + .Take(maxCollectionItemsAllowed) + .Select(i => i.Id) + .Where(id => !currentChildrenSet.Contains(id)).ToArray(); + + if (newItems.Length == 0) + { + logger.LogInformation("No new items detected for {Name}.", catalogName); + return; } + + var collectionCapReached = allRetrievedCatalogItems.Length > maxCollectionItemsAllowed; + if (collectionCapReached) + { + logger.LogWarning("Max Collection Size reached for {CatalogName}, collection will be updated with the newest {MaxItems} items!", catalogName, maxCollectionItemsAllowed); + } + + // We have to remove the current items, as we will be adding all the retrieved items in the order we got them from the catalog + if (currentChildren.Length != 0) + { + await collectionManager.RemoveFromCollectionAsync(collection.Id, currentChildren).ConfigureAwait(false); + } + + var amountToImport = collectionCapReached ? maxCollectionItemsAllowed : allRetrievedCatalogItems.Length; + + // Use this instead of above line if the check in the 'ImportCatalogAsync' method for 'isNewLibraryItem' is used: + //var amountToImport = collectionCapReached ? maxCollectionItemsAllowed : Math.Max(allRetrievedCatalogItems.Length, catalogConfig.MaxItems); + + await collectionManager + .AddToCollectionAsync( + collection.Id, + allRetrievedCatalogItems.Take(amountToImport).Select(i => i.Id)) + .ConfigureAwait(false); + + logger.LogInformation("Updated collection {CollectionName} with {Amount} new items. Total: {TotalItems}", + catalogName, newItems.Length, amountToImport); } catch (Exception ex) { - logger.LogError(ex, "Error updating collection for {Name}", config.Name); + logger.LogError(ex, "Error updating collection for {CatalogName}", catalogName); } } @@ -268,26 +340,29 @@ public async Task SyncAllEnabledAsync(CancellationToken ct, IProgress? p foreach (var cat in enabled) { + var catalogName = $"{cat.Name} {cat.Type}"; + ct.ThrowIfCancellationRequested(); - logger.LogInformation("Processing enabled catalog: {Name}", cat.Name); + logger.LogInformation("Processing enabled catalog: {Name}", catalogName); var catMax = cat.MaxItems; var localOffset = offset; var catProgress = progress is null ? null - : (IProgress) - new Progress(p => - progress.Report((localOffset + p / 100.0 * catMax) / total * 100.0) - ); + : (IProgress)new Progress(p => progress.Report((localOffset + p / 100.0 * catMax) / total * 100.0)); - await ImportCatalogAsync(cat.Id, cat.Type, ct, catProgress).ConfigureAwait(false); + try + { + await ImportCatalogAsync(cat.Id, cat.Type, ct, catProgress).ConfigureAwait(false); + } + catch (OperationCanceledException ex) + { + logger.LogWarning(ex, "Import for {CatalogName} aborted, continuing with next catalog.", catalogName); + } offset += catMax; } - // collections appear empty after inporting this fixes that.. sometimes... - libraryManager.QueueLibraryScan(); - progress?.Report(100); } } diff --git a/Services/CatalogService.cs b/Services/CatalogService.cs index 9d444f7..9b00bab 100644 --- a/Services/CatalogService.cs +++ b/Services/CatalogService.cs @@ -74,11 +74,4 @@ public void UpdateCatalogConfig(CatalogConfig updatedConfig) GelatoPlugin.Instance.SaveConfiguration(); } - - public CatalogConfig? GetCatalogConfig(string id, string type) - { - return GelatoPlugin.Instance!.Configuration.Catalogs.FirstOrDefault(c => - c.Id == id && c.Type == type - ); - } }