2025.3.17.0

API.SiteSettingsBase: fix incorrect class initializer
API.UserDataBase: add all objects to xml (STD)
API.Facebook: fix downloading reels from noname profiles
API.Pinterest: remove 'UserOptions' overrides (SiteSettings); add 'PwsHeader' to 'GetBoards'
API.PornHub: fix 'UpdateUserOptions' function ('NameTrue')
API.Threads: fix 'pinned' posts
API.TikTok: add photos download

Author: AAndyProgram

Changelog.md

@@ -1,3 +1,17 @@
+# 2025.3.17.0
+
+*2025-03-17*
+
+- Added
+  - **TikTok: downloading photos**
+- Updated
+  - gallery-dl up to version **1.29.2**
+- Fixed
+  - Sites
+    - Facebook: reels aren't downloaded from noname profiles
+    - PornHub: newly added users aren't downloading
+    - Threads: users aren't updated if there is a pinned post
+
 # 2025.2.25.0
 
 *2025-02-25*

README.md

@@ -45,7 +45,7 @@ A program to download photo and video from [any site](#supported-sites) (e.g. Yo
   - Instagram images and videos, tagged posts, stories, saved posts;
   - Threads images and videos, saved posts;
   - Facebook images and videos, stories, saved posts;
-  - TikTok videos;
+  - TikTok images and videos;
   - Pinterest boards, users, saved posts;
   - Imgur images, galleries and videos;
   - Gfycat videos;

SCrawler/API/Base/SiteSettingsBase.vb

@@ -394,7 +394,7 @@ Namespace API.Base
                                             With c.GetParameters
                                                 If .ListExists Then
                                                     If .Count = 1 Then
-                                                        Return .Self()(0).ParameterType.GetInterfaces.ListIfNothing.Where(Function(i) i Is Me.GetType).Count = 1
+                                                        Return .Self()(0).ParameterType Is Me.GetType
                                                     Else
                                                         Return False
                                                     End If
@@ -412,7 +412,8 @@ Namespace API.Base
                         End If
                     End With
                     If Not constructor Is Nothing Then
-                        If args > 0 AndAlso Not constructor.GetParameters()(0).ParameterType Is GetType(ISiteSettings) Then Throw New Exception
+                        If args > 0 AndAlso constructor.GetParameters()(0).ParameterType.GetInterface(GetType(ISiteSettings).Name) Is Nothing Then _
+                           Throw New Exception("Class Interface type is incompatible")
                         If args = 0 Then Options = constructor.Invoke(Nothing) Else Options = constructor.Invoke({Me})
                     End If
                     If Options Is Nothing Then Options = Activator.CreateInstance(_UserOptionsType)

SCrawler/API/Base/UserDataBase.vb

@@ -1461,6 +1461,7 @@ BlockNullPicture:
                     Data.DownloadState = UserMediaStates.Missing
                 End If
                 YouTube.Objects.YouTubeMediaContainerBase.Update(_ContentNew(0), Data)
+                If _ContentNew.Count > 1 Then Data.Files.ListAddList(_ContentNew.Select(Function(cc) cc.File), LNC)
                 If ResetTitle And Not _ContentNew(0).File.Name.IsEmptyString Then Data.Title = _ContentNew(0).File.Name
             Else
                 Data.DownloadState = UserMediaStates.Missing

SCrawler/API/Facebook/UserData.vb

@@ -638,7 +638,7 @@ Namespace API.Facebook
             End If
         End Function
         Private Sub GetVideoPageID(ByVal GetReels As Boolean, ByVal Token As CancellationToken)
-            Dim URL$ = $"{GetProfileUrl()}\{IIf(GetReels, "reels", "videos")}"
+            Dim URL$ = $"{GetProfileUrl()}{IIf(IsNoNameProfile, "&sk=", "/")}{IIf(GetReels, IIf(IsNoNameProfile, "reels_tab", "reels"), "videos")}"
             Dim resp As Responser = HtmlResponserCreate()
             Try
                 WaitTimer()

SCrawler/API/Pinterest/SiteSettings.vb

@@ -31,6 +31,7 @@ Namespace API.Pinterest
             CheckNetscapeCookiesOnEndInit = True
             UseNetscapeCookies = True
             UserRegex = RParams.DMS("https?://w{0,3}.?[^/]*?.?pinterest.com/([^/]+)/?(?(_)|([^/]*))/?([^/\?]*)", 0, RegexReturn.ListByMatch, EDP.ReturnValue)
+            UserOptionsType = GetType(EditorExchangeOptions)
         End Sub
 #End Region
 #Region "GetInstance, Available"
@@ -72,12 +73,6 @@ Namespace API.Pinterest
                 Return String.Empty
             End If
         End Function
-        Friend Overrides Sub UserOptions(ByRef Options As Object, ByVal OpenForm As Boolean)
-            If Options Is Nothing Then Options = New EditorExchangeOptions
-            If OpenForm Then
-                Using f As New InternalSettingsForm(Options, Me, False) : f.ShowDialog() : End Using
-            End If
-        End Sub
 #End Region
     End Class
 End Namespace

SCrawler/API/Pinterest/UserData.vb

@@ -170,6 +170,7 @@ Namespace API.Pinterest
                 urls.ListAddList(GetDataFromGalleryDL(URL, True, Token), LNC)
                 If urls.ListExists Then urls.RemoveAll(Function(__url) Not __url.Contains("BoardsResource/get/"))
                 If urls.ListExists Then
+                    Responser.Headers.Add(PwsHeader)
                     ProgressPre.ChangeMax(urls.Count)
                     For Each URL In urls
                         ProgressPre.Perform()
@@ -193,6 +194,8 @@ Namespace API.Pinterest
             Catch ex As Exception
                 ProcessException(ex, Token, $"data (gallery-dl boards) downloading error [{URL}]")
                 Return Nothing
+            Finally
+                Responser.Headers.Remove(PwsHeader)
             End Try
         End Function
         Private Sub DownloadBoardImages(ByRef Board As BoardInfo, ByVal Token As CancellationToken)

SCrawler/API/PornHub/UserData.vb

@@ -195,7 +195,7 @@ Namespace API.PornHub
             If Not Force OrElse (Not IsUser AndAlso Not SiteMode = SiteModes.Playlists AndAlso Not NewUrl.IsEmptyString AndAlso MyFileSettings.Exists) Then
                 Dim eObj As Plugin.ExchangeOptions = Nothing
                 If Force Then eObj = MySettings.IsMyUser(NewUrl)
-                If (Force And Not eObj.UserName.IsEmptyString) Or (Not Force And Not Name.IsEmptyString And NameTrue.IsEmptyString) Then
+                If (Force And Not eObj.UserName.IsEmptyString) Or (Not Force And Not Name.IsEmptyString And NameTrue(True).IsEmptyString) Then
                     If Not If(Force, eObj.Options, Options).IsEmptyString Then
                         If (IsUser Or SiteMode = SiteModes.Playlists) And Force Then
                             Return False
@@ -241,7 +241,7 @@ Namespace API.PornHub
                     SiteMode = .Value(Name_SiteMode).FromXML(Of Integer)(SiteModes.User)
                     UpdateUserOptions()
                 Else
-                    If UpdateUserOptions() Then .Value(Name_LabelsName) = LabelsString
+                    If UpdateUserOptions() Then .Value(Name_LabelsName) = LabelsString : .Value(Name_TrueName) = NameTrue(True)
                     .Add(Name_PersonType, PersonType)
                     .Add(Name_DownloadUHD, DownloadUHD.BoolToInteger)
                     .Add(Name_DownloadUploaded, DownloadUploaded.BoolToInteger)

SCrawler/API/ThreadsNet/UserData.vb

@@ -128,10 +128,13 @@ Namespace API.ThreadsNet
                 If IsSavedPosts Then
                     Return False
                 Else
-                    If MaxLastDownDate.HasValue Then
-                        Dim d As Date? = AConvert(Of Date)(Items(Index).ItemF(DefaultParser_ElemNode_Default).Value("taken_at"), UnixDate32Provider, Nothing)
-                        If d.HasValue Then Return d.Value < MaxLastDownDate.Value
-                    End If
+                    With Items(Index).ItemF(DefaultParser_ElemNode)
+                        Return .Value({"text_post_app_info", "pinned_post_info"}, "is_pinned_to_profile").FromXML(Of Boolean)(False)
+                        If MaxLastDownDate.HasValue Then
+                            Dim d As Date? = AConvert(Of Date)(.Value("taken_at"), UnixDate32Provider, Nothing)
+                            If d.HasValue Then Return d.Value <= MaxLastDownDate.Value
+                        End If
+                    End With
                     Return Not FirstLoadingDone
                 End If
             Catch ex As Exception

SCrawler/API/TikTok/Declarations.vb

@@ -6,11 +6,14 @@
 '
 ' This program is distributed in the hope that it will be useful,
 ' but WITHOUT ANY WARRANTY
+Imports System.Text.RegularExpressions
 Imports PersonalUtilities.Functions.RegularExpressions
 Namespace API.TikTok
     Friend Module Declarations
         Friend ReadOnly SimpleDateConverter As New ADateTime("yyyyMMdd")
         Friend ReadOnly RegexTagsReplacer As RParams = RParams.DM("#\w+\s?", -1, RegexReturn.Replace,
                                                                   CType(Function(input$) String.Empty, Func(Of String, String)), EDP.ReturnValue)
+        Friend ReadOnly RegexPhotoJson As RParams = RParams.DMS("UNIVERSAL_DATA_FOR_REHYDRATION__"" type=""application/json""\>([^\<]+)\<", 1,
+                                                                RegexOptions.IgnoreCase, EDP.ReturnValue)
     End Module
 End Namespace