Implement retry for failed files in Slskd

Author: Meyn

README.md

@@ -61,7 +61,10 @@ Tubifarry supports **Slskd**, the Soulseek client, as both an **indexer** and **
 
 ---
 
-### YouTube Downloader Setup 🎥  
+### YouTube Downloader Setup 🎥 
+> #### YouTube Warning ⚠️
+> Please be aware that YouTube often blocks Tubifarry as a bot. We are currently waiting for external updates. Logging in and the YouTube-only indexer are disabled for now. If login is necessary, please revert to versions earlier than 1.6.0. We appreciate your patience and understanding during this time.
+
 Tubifarry allows you to download music directly from YouTube. Follow the steps below to configure the YouTube downloader.  
 
 #### **Configure the Indexer**:  

Tubifarry/Core/Model/AlbumData.cs

@@ -10,6 +10,8 @@ namespace Tubifarry.Core.Model
     /// </summary>
     public class AlbumData
     {
+        public string? Guid { get; set; }
+
         public string IndexerName { get; }
         // Mixed
         public string AlbumId { get; set; } = string.Empty;
@@ -34,6 +36,7 @@ public class AlbumData
         // Soulseek
         public long? Size { get; set; }
         public int Priotity { get; set; }
+        public string? ExtraInfo { get; set; }
 
         // Not used
         public AudioFormat Codec { get; set; } = AudioFormat.AAC;
@@ -45,7 +48,7 @@ public class AlbumData
         /// </summary>
         public ReleaseInfo ToReleaseInfo() => new()
         {
-            Guid = $"{IndexerName}-{AlbumId}-{Bitrate}",
+            Guid = Guid ?? $"{IndexerName}-{AlbumId}-{Bitrate}",
             Artist = ArtistName,
             Album = AlbumName,
             DownloadUrl = AlbumId,
@@ -92,12 +95,15 @@ private string ConstructTitle()
                 calculatedBitrate = (int)(Size.Value * 8 / (Duration * 1000));
 
             if (AudioFormatHelper.IsLossyFormat(Codec) && calculatedBitrate != 0)
-                title += $" [{Codec} {calculatedBitrate}kbps] [WEB]";
-            if (!AudioFormatHelper.IsLossyFormat(Codec) && BitDepth != 0)
-                title += $" [{Codec} {BitDepth}bit] [WEB]";
+                title += $" [{Codec} {calculatedBitrate}kbps]";
+            else if (!AudioFormatHelper.IsLossyFormat(Codec) && BitDepth != 0)
+                title += $" [{Codec} {BitDepth}bit]";
             else
-                title += $" [{Codec}] [WEB]";
+                title += $" [{Codec}]";
+            if (ExtraInfo != null)
+                title += $" [{ExtraInfo}]";
 
+            title += " [WEB]";
             return title;
         }
 

Tubifarry/Download/Clients/Soulseek/SlskdClient.cs

@@ -32,26 +32,71 @@ public SlskdClient(IHttpClient httpClient, IConfigService configService, IDiskPr
         public override async Task<string> Download(RemoteAlbum remoteAlbum, IIndexer indexer)
         {
             SlskdDownloadItem item = new(remoteAlbum);
-            HttpRequest request = BuildHttpRequest(remoteAlbum.Release.DownloadUrl, HttpMethod.Post, remoteAlbum.Release.Source);
-            HttpResponse response = await _httpClient.ExecuteAsync(request);
+            try
+            {
+                HttpRequest request = BuildHttpRequest(remoteAlbum.Release.DownloadUrl, HttpMethod.Post, remoteAlbum.Release.Source);
+                HttpResponse response = await _httpClient.ExecuteAsync(request);
 
-            if (response.StatusCode != HttpStatusCode.Created)
-                throw new DownloadClientException("Failed to create download.");
-            if (Settings.UseLRCLIB)
+                if (response.StatusCode != HttpStatusCode.Created)
+                    throw new DownloadClientException("Failed to create download.");
                 item.FileStateChanged += FileStateChanged;
-            AddDownloadItem(item);
+                AddDownloadItem(item);
+            }
+            catch (Exception)
+            {
+                try
+                {
+                    RemoveItemAsync(item).Wait();
+                }
+                catch (Exception) { }
+                return null!;
+            }
             return item.ID.ToString();
         }
 
-        private void FileStateChanged(object? sender, SlskdDownloadFile file)
+        private void FileStateChanged(object? sender, SlskdFileState fileState)
         {
-            string filename = file.Filename;
+            fileState.UpdateMaxRetryCount(Settings.RetryAttempts);
+            string filename = fileState.File.Filename;
             string extension = Path.GetExtension(filename);
             AudioFormat format = AudioFormatHelper.GetAudioCodecFromExtension(extension.TrimStart('.'));
+            if (fileState.GetStatus() == DownloadItemStatus.Warning)
+            {
+                _logger.Trace($"Retrying download for file: {filename}. Attempt {fileState.RetryCount} of {fileState.MaxRetryCount}");
+                _ = RetryDownloadAsync(fileState, (SlskdDownloadItem)sender!);
+                return;
+            }
 
-            if (file.GetStatus() != DownloadItemStatus.Completed || format == AudioFormat.Unknown)
+            if (fileState.GetStatus() != DownloadItemStatus.Completed || format == AudioFormat.Unknown)
                 return;
-            PostProcess((SlskdDownloadItem)sender!, file);
+            if (Settings.UseLRCLIB)
+                PostProcess((SlskdDownloadItem)sender!, fileState.File);
+        }
+
+        private async Task RetryDownloadAsync(SlskdFileState fileState, SlskdDownloadItem item)
+        {
+            try
+            {
+                using JsonDocument doc = JsonDocument.Parse(item.RemoteAlbum.Release.Source);
+                JsonElement root = doc.RootElement;
+                JsonElement matchingItem = root.EnumerateArray()
+                    .FirstOrDefault(x => x.GetProperty("Filename").GetString() == fileState.File.Filename);
+
+                if (matchingItem.ValueKind == JsonValueKind.Undefined)
+                    return;
+                string payload = JsonSerializer.Serialize(new[] { matchingItem });
+
+                HttpRequest request = BuildHttpRequest(item.RemoteAlbum.Release.DownloadUrl, HttpMethod.Post, payload);
+                HttpResponse response = await _httpClient.ExecuteAsync(request);
+
+                if (response.StatusCode == HttpStatusCode.Created)
+                    _logger.Trace($"Successfully retried download for file: {fileState.File.Filename}");
+            }
+            catch (Exception ex)
+            {
+                _logger.Error(ex, $"Failed to retry download for file: {fileState.File.Filename}");
+            }
+            fileState.IncrementAttempt();
         }
 
         private void PostProcess(SlskdDownloadItem item, SlskdDownloadFile file) => item.PostProcessTasks.Add(Task.Run(async () =>
@@ -111,8 +156,13 @@ private async Task UpdateDownloadItemsAsync()
                 foreach (SlskdDownloadDirectory dir in data)
                 {
                     HashCode hash = new();
-                    foreach (SlskdDownloadFile file in dir.Files ?? new List<SlskdDownloadFile>())
-                        hash.Add(file.Filename);
+                    List<string> sortedFilenames = dir.Files?
+                        .Select(file => file.Filename)
+                        .OrderBy(filename => filename)
+                        .ToList() ?? new List<string>();
+
+                    foreach (string? filename in sortedFilenames)
+                        hash.Add(filename);
                     SlskdDownloadItem? item = GetDownloadItem(hash.ToHashCode());
                     if (item == null)
                         continue;

Tubifarry/Download/Clients/Soulseek/SlskdModels.cs

@@ -19,12 +19,12 @@ public class SlskdDownloadItem
         public string? Username { get; set; }
         public RemoteAlbum RemoteAlbum { get; set; }
 
-        public event EventHandler<SlskdDownloadFile>? FileStateChanged;
+        public event EventHandler<SlskdFileState>? FileStateChanged;
 
         private Logger _logger;
 
         private SlskdDownloadDirectory? _slskdDownloadDirectory;
-        private Dictionary<string, string> _previousFileStates = new();
+        private Dictionary<string, SlskdFileState> _previousFileStates = new();
 
         public List<Task> PostProcessTasks { get; } = new();
 
@@ -35,7 +35,7 @@ public SlskdDownloadDirectory? SlskdDownloadDirectory
             {
                 if (_slskdDownloadDirectory == value)
                     return;
-                CompareFileStates(_slskdDownloadDirectory, value);
+                CompareFileStates(value);
                 _slskdDownloadDirectory = value;
             }
         }
@@ -48,21 +48,34 @@ public SlskdDownloadItem(RemoteAlbum remoteAlbum)
             _lastUpdateTime = DateTime.UtcNow;
             _lastDownloadedSize = 0;
             HashCode hash = new();
-            foreach (SlskdFileData file in FileData)
-                hash.Add(file.Filename);
+            List<string?> sortedFilenames = FileData
+                .Select(file => file.Filename)
+                .OrderBy(filename => filename)
+                .ToList();
+            foreach (string? filename in sortedFilenames)
+                hash.Add(filename);
             ID = hash.ToHashCode();
             _downloadClientItem = new() { DownloadId = ID.ToString(), CanBeRemoved = true, CanMoveFiles = true };
         }
 
-        private void CompareFileStates(SlskdDownloadDirectory? previousDirectory, SlskdDownloadDirectory? newDirectory)
+        private void CompareFileStates(SlskdDownloadDirectory? newDirectory)
         {
             if (newDirectory?.Files == null)
                 return;
 
             foreach (SlskdDownloadFile file in newDirectory.Files)
-                if (_previousFileStates.TryGetValue(file.Id, out string? previousState) && previousState != file.State)
-                    FileStateChanged?.Invoke(this, file);
-            _previousFileStates = newDirectory.Files.ToDictionary(file => file.Id, file => file.State);
+            {
+                if (_previousFileStates.TryGetValue(file.Filename, out SlskdFileState? fileState) && fileState != null)
+                {
+                    fileState.UpdateFile(file);
+                    if (fileState.State != fileState.PreviousState)
+                        FileStateChanged?.Invoke(this, fileState);
+                }
+                else
+                    _previousFileStates.Add(file.Filename, new(file));
+
+
+            }
         }
 
         public OsPath GetFullFolderPath(string downloadPath) => new(Path.Combine(downloadPath, SlskdDownloadDirectory?.Directory
@@ -92,17 +105,17 @@ public DownloadClientItem GetDownloadClientItem(string downloadPath, TimeSpan? t
             _lastUpdateTime = now;
             _lastDownloadedSize = downloadedSize;
 
-            List<DownloadItemStatus> fileStatuses = SlskdDownloadDirectory.Files.Select(file => file.GetStatus()).ToList();
-            List<string> failedFiles = SlskdDownloadDirectory.Files
+            List<DownloadItemStatus> fileStatuses = _previousFileStates.Values.Select(file => file.GetStatus()).ToList();
+            List<string> failedFiles = _previousFileStates.Values
                 .Where(file => file.GetStatus() == DownloadItemStatus.Failed)
-                .Select(file => Path.GetFileName(file.Filename)).ToList();
+                .Select(file => Path.GetFileName(file.File.Filename)).ToList();
 
             DownloadItemStatus status = DownloadItemStatus.Queued;
             DateTime lastTime = SlskdDownloadDirectory.Files.Max(x => x.EnqueuedAt > x.StartedAt ? x.EnqueuedAt : x.StartedAt + x.ElapsedTime);
 
             if (now - lastTime > timeout)
                 status = DownloadItemStatus.Failed;
-            else if ((double)failedFiles.Count / fileStatuses.Count * 100 > 10)
+            else if ((double)failedFiles.Count / fileStatuses.Count * 100 > 20)
             {
                 status = DownloadItemStatus.Failed;
                 _downloadClientItem.Message = $"Downloading {failedFiles.Count} files failed: {string.Join(", ", failedFiles)}";
@@ -121,10 +134,13 @@ public DownloadClientItem GetDownloadClientItem(string downloadPath, TimeSpan? t
             }
             else if (fileStatuses.Any(status => status == DownloadItemStatus.Paused))
                 status = DownloadItemStatus.Paused;
-            else if (fileStatuses.Any(status => status == DownloadItemStatus.Downloading))
-                status = DownloadItemStatus.Downloading;
             else if (fileStatuses.Any(status => status == DownloadItemStatus.Warning))
+            {
+                _downloadClientItem.Message = "Some files failed. Retrying download...";
                 status = DownloadItemStatus.Warning;
+            }
+            else if (fileStatuses.Any(status => status == DownloadItemStatus.Downloading))
+                status = DownloadItemStatus.Downloading;
 
             // Update DownloadClientItem
             _downloadClientItem.TotalSize = totalSize;
@@ -136,6 +152,58 @@ public DownloadClientItem GetDownloadClientItem(string downloadPath, TimeSpan? t
         }
     }
 
+    public class SlskdFileState
+    {
+        public SlskdDownloadFile File { get; private set; } = null!;
+        public int RetryCount { get; private set; }
+        private bool _retried = true;
+        public int MaxRetryCount { get; private set; } = 1;
+        public string State => File.State;
+        public string PreviousState { get; private set; } = "Requested";
+
+        public DownloadItemStatus GetStatus()
+        {
+            DownloadItemStatus status = GetStatus(State);
+            if ((status == DownloadItemStatus.Failed && RetryCount < MaxRetryCount) || _retried)
+                return DownloadItemStatus.Warning;
+            return status;
+        }
+
+        private static DownloadItemStatus GetStatus(string state) => state switch
+        {
+            "Requested" => DownloadItemStatus.Queued, // "Requested" is treated as "Queued"
+            "Queued, Remotely" or "Queued, Locally" => DownloadItemStatus.Queued, // Both are queued states
+            "Initializing" => DownloadItemStatus.Queued, // "Initializing" is treated as "Queued"
+            "InProgress" => DownloadItemStatus.Downloading, // "InProgress" maps to "Downloading"
+            "Completed, Succeeded" => DownloadItemStatus.Completed, // Successful completion
+            "Completed, Cancelled" => DownloadItemStatus.Failed, // Cancelled is treated as "Failed"
+            "Completed, TimedOut" => DownloadItemStatus.Failed, // Timed out is treated as "Failed"
+            "Completed, Errored" => DownloadItemStatus.Failed, // Errored is treated as "Failed"
+            "Completed, Rejected" => DownloadItemStatus.Failed, // Rejected is treated as "Failed"
+            _ => DownloadItemStatus.Queued // Default to "Queued" for unknown states
+        };
+
+        public SlskdFileState(SlskdDownloadFile file) => UpdateFile(file);
+
+        public void UpdateFile(SlskdDownloadFile file)
+        {
+            if (!_retried)
+                PreviousState = State;
+            else if (File != null && GetStatus(file.State) == DownloadItemStatus.Failed)
+                PreviousState = "Requested";
+            File = file;
+            _retried = false;
+        }
+
+        public void UpdateMaxRetryCount(int maxRetryCount) => MaxRetryCount = maxRetryCount;
+
+        public void IncrementAttempt()
+        {
+            _retried = true;
+            RetryCount++;
+        }
+    }
+
     public record SlskdDownloadDirectory(string Directory, int FileCount, List<SlskdDownloadFile>? Files)
     {
         public static IEnumerable<SlskdDownloadDirectory> GetDirectories(JsonElement directoriesElement)
@@ -172,22 +240,8 @@ public record SlskdDownloadFile(
        double PercentComplete,
        TimeSpan RemainingTime,
        TimeSpan? EndedAt
-   )
+    )
     {
-        public DownloadItemStatus GetStatus() => State switch
-        {
-            "Requested" => DownloadItemStatus.Queued, // "Requested" is treated as "Queued"
-            "Queued, Remotely" or "Queued, Locally" => DownloadItemStatus.Queued, // Both are queued states
-            "Initializing" => DownloadItemStatus.Queued, // "Initializing" is treated as "Queued"
-            "InProgress" => DownloadItemStatus.Downloading, // "InProgress" maps to "Downloading"
-            "Completed, Succeeded" => DownloadItemStatus.Completed, // Successful completion
-            "Completed, Cancelled" => DownloadItemStatus.Failed, // Cancelled is treated as "Failed"
-            "Completed, TimedOut" => DownloadItemStatus.Failed, // Timed out is treated as "Failed"
-            "Completed, Errored" => DownloadItemStatus.Failed, // Errored is treated as "Failed"
-            "Completed, Rejected" => DownloadItemStatus.Failed, // Rejected is treated as "Failed"
-            _ => DownloadItemStatus.Queued // Default to "Queued" for unknown states
-        };
-
         public static IEnumerable<SlskdDownloadFile> GetFiles(JsonElement filesElement)
         {
             if (filesElement.ValueKind != JsonValueKind.Array)

Tubifarry/Download/Clients/Soulseek/SlskdProviderSettings.cs

@@ -36,6 +36,11 @@ public SlskdProviderSettingsValidator()
                 .GreaterThanOrEqualTo(0.1)
                 .WithMessage("Timeout must be at least 0.1 hours.")
                 .When(c => c.Timeout.HasValue);
+
+            // RetryAttempts validation
+            RuleFor(c => c.RetryAttempts)
+                .InclusiveBetween(0, 10)
+                .WithMessage("Retry attempts must be between 0 and 10.");
         }
     }
 
@@ -61,13 +66,15 @@ public class SlskdProviderSettings : IProviderConfig
         [FieldDefinition(7, Label = "Timeout", Type = FieldType.Textbox, HelpText = "Specify the maximum time to wait for a response from the Slskd instance before timing out. Fractional values are allowed (e.g., 1.5 for 1 hour and 30 minutes). Set leave blank for no timeout.", Unit = "hours", Advanced = true, Placeholder = "Enter timeout in hours")]
         public double? Timeout { get; set; }
 
+        [FieldDefinition(8, Label = "Retry Attempts", Type = FieldType.Number, HelpText = "The number of times to retry downloading a file if it fails.", Advanced = true, Placeholder = "Enter retry attempts")]
+        public int RetryAttempts { get; set; } = 1;
+
         [FieldDefinition(98, Label = "Is Fetched remote", Type = FieldType.Checkbox, Hidden = HiddenType.Hidden)]
         public bool IsRemotePath { get; set; }
 
         [FieldDefinition(99, Label = "Is Localhost", Type = FieldType.Checkbox, Hidden = HiddenType.Hidden)]
         public bool IsLocalhost { get; set; }
 
-
         public TimeSpan? GetTimeout() => Timeout == null ? null : TimeSpan.FromHours(Timeout.Value);
 
         public NzbDroneValidationResult Validate() => new(Validator.Validate(this));

Tubifarry/Download/Clients/YouTube/YoutubeClient.cs

@@ -8,6 +8,7 @@
 using NzbDrone.Core.Organizer;
 using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.RemotePathMappings;
+using Requests;
 using Tubifarry.Core.Model;
 using Tubifarry.Core.Utilities;
 using Xabe.FFmpeg;
@@ -23,6 +24,7 @@ public YoutubeClient(IYoutubeDownloadManager dlManager, IConfigService configSer
         {
             _dlManager = dlManager;
             _naminService = namingConfigService;
+            RequestHandler.MainRequestHandlers[1].MaxParallelism = 1;
         }
 
         public override string Name => "Youtube";

Tubifarry/Download/Clients/YouTube/YoutubeProviderSettings.cs

@@ -58,7 +58,7 @@ public class YoutubeProviderSettings : IProviderConfig
         [FieldDefinition(0, Label = "Download Path", Type = FieldType.Path, HelpText = "Specify the directory where downloaded files will be saved.")]
         public string DownloadPath { get; set; } = "";
 
-        [FieldDefinition(1, Label = "Cookie Path", Type = FieldType.FilePath, Placeholder = "/downloads/Cookies/cookies.txt", HelpText = "Specify the path to the YouTube cookies file. This is optional but required for accessing restricted content.", Advanced = true)]
+        [FieldDefinition(1, Label = "Cookie Path", Type = FieldType.FilePath, Hidden = HiddenType.HiddenIfNotSet, Placeholder = "/downloads/Cookies/cookies.txt", HelpText = "Specify the path to the YouTube cookies file. This is optional but required for accessing restricted content.", Advanced = true)]
         public string CookiePath { get; set; } = string.Empty;
 
         [FieldDefinition(2, Label = "Use ID3v2.3 Tags", HelpText = "Enable this option to use ID3v2.3 tags for better compatibility with older media players like Windows Media Player.", Type = FieldType.Checkbox, Advanced = true)]