2025.8.1.0

YT
Update 'ReplaceModificationDate'

SCrawler
API.Instagram: fix 'LastCursor' issue
API.Reddit: add OAuth validation; add default credentials; hide unused controls; add 'SeparatedTasks'; bypass 429 error; fix crossposts downloading
API.Redgifs: force delete cookies if user added them
API.TikTok: yt-dlp modification (date change)
API.Twitter: simplify large profiles download
SettingsCLS: change default max value for channel downloads

Author: AAndyProgram

Changelog.md

@@ -1,19 +1,34 @@
+# 2025.8.1.0
+
+*2025-08-01*
+
+- Added
+  - Sites:
+    - Reddit: **bypass error `429`**
+    - Twitter: **[large profile option](https://github.com/AAndyProgram/SCrawler/wiki/Settings#twitter-user-settings) in user settings**
+  - Minor improvements
+- Updated
+  - yt-dlp up to version **2025.27.21**
+  - gallery-dl up to version **1.30.2**
+- Fixed
+  - Reddit: in some cases crossposts don't download
+  - Minor bugs
+
 # 2025.7.18.0
 
 *2025-07-18*
 
 - Added
   - Sites:
-    - OnlyFans:
-      - **bypass unpurchased videos**
-      - support for GIF files
+    - OnlyFans: support for GIF files
     - Reddit: extended `429` error handling
     - Xhamster: support for downloading 'moments'
   - Minor improvements
 - Updated
   - yt-dlp up to version **2025.06.30**
   - gallery-dl up to version **1.30.0**
 - Fixed
+  - OnlyFans: **hanging on purchased content**
   - Minor bugs
 
 # 2025.6.12.0

SCrawler.YouTube/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2025.6.12.0")>
-<Assembly: AssemblyFileVersion("2025.6.12.0")>
+<Assembly: AssemblyVersion("2025.8.1.0")>
+<Assembly: AssemblyFileVersion("2025.8.1.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb

@@ -823,7 +823,9 @@ Namespace API.YouTube.Objects
                         'cmd = $"yt-dlp -f ""{cmd}"""
                         'cmd = $"yt-dlp -f {cmd}"
                         cmd = $"{YTDLP_NAME} -f {cmd}"
-                        If Not MyYouTubeSettings.ReplaceModificationDate Then cmd &= " --no-mtime"
+                        'yt-dlp 2025.07.21
+                        'If Not MyYouTubeSettings.ReplaceModificationDate Then cmd &= " --no-mtime"
+                        cmd &= $" --{IIf(MyYouTubeSettings.ReplaceModificationDate.Value, String.Empty, "no-")}mtime"
                         cmd.StringAppend(formats, " ")
                         cmd.StringAppend(subs, " ")
                         cmd.StringAppend(YouTubeFunctions.GetCookiesCommand(WithCookies, YouTubeCookieNetscapeFile), " ")

SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2025.6.12.0")>
-<Assembly: AssemblyFileVersion("2025.6.12.0")>
+<Assembly: AssemblyVersion("2025.8.1.0")>
+<Assembly: AssemblyFileVersion("2025.8.1.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler/API/Instagram/UserData.vb

@@ -412,6 +412,7 @@ Namespace API.Instagram
                 ThrowAny(Token)
                 HasError = False
                 Dim dt As Func(Of Boolean) = Function() (CBool(MySiteSettings.DownloadTimeline.Value) And GetTimeline) Or IsSavedPosts
+                If FirstLoadingDone Then LastCursor = String.Empty
                 If dt.Invoke And Not LastCursor.IsEmptyString Then
                     s = IIf(IsSavedPosts, Sections.SavedPosts, Sections.Timeline)
                     upClaimRequest.Invoke

SCrawler/API/Reddit/SiteSettings.vb

@@ -15,21 +15,32 @@ Imports PersonalUtilities.Tools.Web.Clients.Base
 Imports PersonalUtilities.Tools.Web.Documents.JSON
 Imports PersonalUtilities.Functions.XML
 Imports PersonalUtilities.Functions.RegularExpressions
+Imports DN = SCrawler.API.Base.DeclaredNames
 Imports DownDetector = SCrawler.API.Base.DownDetector
 Imports Download = SCrawler.Plugin.ISiteSettings.Download
 Namespace API.Reddit
-    <Manifest(RedditSiteKey), SavedPosts, SpecialForm(False), UseDownDetector>
+    <Manifest(RedditSiteKey), SavedPosts, SeparatedTasks, SpecialForm(False), UseDownDetector>
     Friend Class SiteSettings : Inherits SiteSettingsBase : Implements DownDetector.IDownDetector
 #Region "Declarations"
 #Region "Authorization"
+        Private Const ApiClientID_Default As String = "dYctRA-SIJxyykHe27lGZg"
+        Private Const ApiClientSecret_Default As String = "_5D6KzplRPDga-es1YlpzDIe9hiFlg"
         <PropertyOption(ControlText:="Login", ControlToolTip:="Your authorization username", IsAuth:=True), PXML, PClonable(Clone:=False)>
         Friend ReadOnly Property AuthUserName As PropertyValue
         <PropertyOption(ControlText:="Password", ControlToolTip:="Your authorization password", IsAuth:=True), PXML, PClonable(Clone:=False)>
         Friend ReadOnly Property AuthPassword As PropertyValue
         <PropertyOption(ControlText:="Client ID", ControlToolTip:="Your registered app client ID", IsAuth:=True), PXML, PClonable(Clone:=False)>
         Friend ReadOnly Property ApiClientID As PropertyValue
+        <PropertyUpdater(NameOf(ApiClientID))> Private Function ApiClientID_SetDefault() As Boolean
+            ApiClientID.Value = ApiClientID_Default
+            Return True
+        End Function
         <PropertyOption(ControlText:="Client Secret", ControlToolTip:="Your registered app client secret", IsAuth:=True), PXML, PClonable(Clone:=False)>
         Friend ReadOnly Property ApiClientSecret As PropertyValue
+        <PropertyUpdater(NameOf(ApiClientSecret))> Private Function ApiClientSecret_SetDefault() As Boolean
+            ApiClientSecret.Value = ApiClientSecret_Default
+            Return True
+        End Function
         <PropertyOption(ControlText:="Bearer token",
                         ControlToolTip:="Bearer token (can be null)." & vbCr &
                                         "If you are using cookies to download the timeline, it is highly recommended that you add a token." & vbCr &
@@ -103,12 +114,15 @@ Namespace API.Reddit
         End Function
 #End Region
 #Region "Other"
-        <PropertyOption(ControlText:="Use M3U8", ControlToolTip:="Use M3U8 or mp4 for Reddit videos", IsAuth:=False), PXML, PClonable>
+        <PropertyOption(ControlText:="Use M3U8", ControlToolTip:="Use M3U8 or mp4 for Reddit videos"), PXML, PClonable, HiddenControl>
         Friend ReadOnly Property UseM3U8 As PropertyValue
-        <PropertyOption(ControlText:="Check image", ControlToolTip:="Check the image if it exists before downloading (it makes downloading very slow)", IsAuth:=False), PXML, PClonable>
+        <PropertyOption(ControlText:="Check image", ControlToolTip:="Check the image if it exists before downloading (it makes downloading very slow)"), PXML, PClonable, HiddenControl>
         Friend ReadOnly Property CheckImage As PropertyValue
-        <PropertyOption(ControlText:="Check image: get original", ControlToolTip:="Get the original image if it exists", IsAuth:=False), PXML, PClonable>
+        <PropertyOption(ControlText:="Check image: get original", ControlToolTip:="Get the original image if it exists"), PXML, PClonable, HiddenControl>
         Friend ReadOnly Property CheckImageReturnOrig As PropertyValue
+        <PropertyOption(ControlText:=DN.ConcurrentDownloadsCaption,
+                        ControlToolTip:=DN.ConcurrentDownloadsToolTip, AllowNull:=False), PXML, TaskCounter, PClonable>
+        Friend ReadOnly Property ConcurrentDownloads As PropertyValue
 #End Region
 #Region "IDownDetector Support"
         Private ReadOnly Property IDownDetector_Value As Integer Implements DownDetector.IDownDetector.Value
@@ -160,17 +174,21 @@ Namespace API.Reddit
             UseM3U8 = New PropertyValue(True)
             CheckImage = New PropertyValue(False)
             CheckImageReturnOrig = New PropertyValue(True)
+            ConcurrentDownloads = New PropertyValue(1)
 
             MDD = New MyDownDetector(Me)
 
             UrlPatternUser = "https://www.reddit.com/{0}/{1}/"
             ImageVideoContains = "reddit.com"
             UserRegex = RParams.DM("[htps:/]{7,8}.*?reddit.com/([user]{1,4})/([^/\?&]+)", 0, RegexReturn.ListByMatch, EDP.ReturnValue)
         End Sub
-        Private Const SettingsVersionCurrent As Integer = 2
+        Private Const SettingsVersionCurrent As Integer = 3
         Friend Overrides Sub EndInit()
             If CInt(SettingsVersion.Value) < SettingsVersionCurrent Then
                 SettingsVersion.Value = SettingsVersionCurrent
+                UseM3U8.Value = True
+                CheckImage.Value = False
+                CheckImageReturnOrig.Value = True
                 BearerTokenUseCurl.Value = False
             End If
             MyBase.EndInit()
@@ -208,6 +226,7 @@ Namespace API.Reddit
             End Sub
         End Class
         Friend Property SessionInterrupted As Boolean = False
+        Friend Property RequestCount As Integer = 0
         Friend Overrides Function ReadyToDownload(ByVal What As Download) As Boolean
             If What = Download.Main Then
                 Return Not SessionInterrupted
@@ -223,6 +242,7 @@ Namespace API.Reddit
         End Function
         Friend Overrides Sub DownloadDone(ByVal What As Download)
             SessionInterrupted = False
+            RequestCount = 0
             MDD.Reset()
             MyBase.DownloadDone(What)
         End Sub