2025.6.12.0

YT
MainModShared: fix environment output
YouTubeMediaContainerBase: fix 'm3u8' audio formats

SCrawler
UserDataBase: text downloading with saved posts; update 'ID' property (handle '_ForceSaveUserInfo')
API.Bluesky: data is not downloaded
API.Reddit: update 'RedditViewExchange'; set base inheritance; inherit default settings for new users
API.ALL: update functions with property 'ID'

Author: AAndyProgram

Changelog.md

@@ -1,3 +1,17 @@
+# 2025.6.12.0
+
+*2025-06-12*
+
+- Updated
+  - yt-dlp up to version **2025.06.09**
+- Fixed
+  - Sites:
+    - YouTube: audio formats of protocol `m3u8` are not handled correctly
+    - BlueSky: data is not downloaded in some cases
+    - Reddit: new users do not inherit default text settings
+    - Saved posts: text downloading with saved posts
+  - Environment incorrect output
+
 # 2025.6.1.0
 
 *2025-06-01*

README.md

@@ -35,15 +35,15 @@ A program to download photo and video from [any site](#supported-sites) (e.g. Yo
 # What can program do:
 - Download pictures and videos from user profiles:
   - YouTube videos, shorts, community feeds, users, artists, playlists, music, tracks;
-  - Reddit images, galleries of images, videos, saved posts;
+  - Reddit images, galleries of images, videos, text, saved posts;
   - Redgifs images and videos (https://www.redgifs.com/);
-  - Twitter images and videos, saved (bookmarked) posts, likes, communities;
-  - Bluesky images and videos;
-  - OnlyFans images and videos, saved (bookmarked) posts, stories;
+  - Twitter images and videos, text, saved (bookmarked) posts, likes, communities;
+  - Bluesky images and videos, text;
+  - OnlyFans images and videos, text, saved (bookmarked) posts, stories;
   - JustForFans images and videos, saved (bookmarked) posts;
   - Mastodon images and videos, saved (bookmarked) posts;
-  - Instagram images and videos, tagged posts, stories, saved posts;
-  - Threads images and videos, saved posts;
+  - Instagram images and videos, text, tagged posts, stories, saved posts;
+  - Threads images and videos, text, saved posts;
   - Facebook images and videos, stories, saved posts;
   - TikTok images and videos;
   - Pinterest boards, users, saved posts;

SCrawler.YouTube/MainModShared.vb

@@ -10,6 +10,7 @@ Imports System.Threading
 Imports PersonalUtilities.Tools
 Imports PersonalUtilities.Tools.Web
 Imports PersonalUtilities.Functions.Messaging
+Imports PersonalUtilities.Functions.RegularExpressions
 Imports SCrawler.DownloadObjects.STDownloader
 Public Module MainModShared
     Public Property BATCH As BatchExecutor
@@ -135,9 +136,11 @@ Namespace Editors
         Public Shared Function GetProgramEnvirText(ByVal EnvirData As IDownloaderSettings, ByVal IsYouTube As Boolean) As String
             Try
                 Dim output$ = String.Empty
+                Dim verAfter As RParams = RParams.DM("\A\w\:\\.*", 0, EDP.ReturnValue)
                 Using b As New BatchExecutor(True)
                     Dim f As SFile
                     Dim cmd$, ff$, vText$
+                    Dim ii%
 
                     For i% = 0 To IIf(IsYouTube, 1, 3)
                         cmd = "--version"
@@ -154,7 +157,17 @@ Namespace Editors
                             Else
                                 b.Reset()
                                 b.Execute($"""{f}"" {cmd}", EDP.None)
-                                If b.OutputData.Count > 3 Then vText = b.OutputData(3) Else vText = "undefined"
+                                'If b.OutputData.Count > 3 Then vText = b.OutputData(3) Else vText = "undefined"
+
+                                vText = String.Empty
+                                With b.OutputData
+                                    If .Count > 0 Then
+                                        ii = .FindIndex(Function(bb) Not CStr(RegexReplace(bb, verAfter)).IsEmptyString)
+                                        If ii >= 0 And ii + 1 <= .Count - 1 Then vText = .Item(ii + 1)
+                                    End If
+                                End With
+                                If vText.IsEmptyString Then vText = "undefined"
+
                                 output.StringAppendLine($"{ff} version: {vText}")
                             End If
                         End If

SCrawler.YouTube/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2025.6.1.0")>
-<Assembly: AssemblyFileVersion("2025.6.1.0")>
+<Assembly: AssemblyVersion("2025.6.12.0")>
+<Assembly: AssemblyFileVersion("2025.6.12.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler.YouTube/Objects/YouTubeMediaContainerBase.vb

@@ -738,6 +738,7 @@ Namespace API.YouTube.Objects
 #Region "Command"
         <XMLEC> Public Property UseCookies As Boolean = MyYouTubeSettings.DefaultUseCookies Implements IYouTubeMediaContainer.UseCookies
         Protected Const mp3 As String = "mp3"
+        Private Const mp4 As String = "mp4"
         Private Const aac As String = "aac"
         Private Const ac3 As String = "ac3"
         Protected PostProcessing_AudioAC3 As Boolean = False
@@ -773,7 +774,12 @@ Namespace API.YouTube.Objects
                         '2023.3.4 -> 2023.7.6
                         'cmd.StringAppend($"ba*[format_id={SelectedAudio.ID}]", "+")
                         cmd.StringAppend(SelectedAudio.ID, "+")
-                        If OutputAudioCodec.StringToLower = ac3 Then
+                        If SelectedVideoIndex >= 0 And SelectedAudio.ProtocolType = Protocols.m3u8 And
+                           (SelectedAudio.Codec.StringToLower = mp4 Or OutputAudioCodec.StringToLower = mp4) Then
+                            PostProcessing_AudioAC3 = True
+                            formats.StringAppend($"--merge-output-format ""{mp4}{IIf(OutputVideoExtension.IsEmptyString, String.Empty, $"/{OutputVideoExtension.StringToLower}")}""", " ")
+                            atCodec = aac
+                        ElseIf OutputAudioCodec.StringToLower = ac3 Then
                             PostProcessing_AudioAC3 = True
                             formats.StringAppend($"--audio-format {aac}", " ")
                             atCodec = aac
@@ -1753,9 +1759,12 @@ Namespace API.YouTube.Objects
             If If(e({"formats"})?.Count, 0) > 0 Then
                 Dim obj As MediaObject
                 Dim nValue#
-                Dim sValue$
+                Dim sValue$ = String.Empty
                 Dim allowWebm As Boolean = MyYouTubeSettings.DefaultVideoAllowWebm
-                Dim validCodecValue As Func(Of String, Boolean) = Function(codec) Not codec.IsEmptyString AndAlso Not codec = "none"
+                Dim validCodecValue As Func(Of String, Boolean) = Function(ByVal codec As String) As Boolean
+                                                                      sValue = codec
+                                                                      Return Not codec.IsEmptyString AndAlso Not codec = "none"
+                                                                  End Function
 
                 For Each ee In e({"formats"})
                     obj = New MediaObject With {
@@ -1779,19 +1788,30 @@ Namespace API.YouTube.Objects
                     If obj.Size <= 0 And obj.Bitrate > 0 And Duration.TotalSeconds > 0 Then _
                        obj.Size = (obj.Bitrate / 8 * Duration.TotalSeconds).RoundVal(2)
 
-                    sValue = ee.Value("vcodec")
-                    If validCodecValue(sValue) Then
+                    'sValue = ee.Value("vcodec")
+                    If validCodecValue(ee.Value("vcodec")) Then
                         obj.Type = UMTypes.Video
                         obj.Codec = sValue.Split(".").First
                         If validCodecValue(ee.Value("acodec")) Then obj.Type = av
+                    ElseIf validCodecValue(ee.Value("acodec")) Then
+                        obj.Type = UMTypes.Audio
+                        obj.Codec = sValue.Split(".").First
                     Else
-                        sValue = ee.Value("acodec")
-                        If validCodecValue(sValue) Then
-                            obj.Type = UMTypes.Audio
-                            obj.Codec = sValue.Split(".").First
-                        Else
-                            Continue For
+                        Dim fd As Boolean = False
+                        sValue = ee.Value("format_note")
+                        If Not sValue.IsEmptyString Then
+                            With ListAddList(Nothing, sValue.Split(","), CType(Function(v) CStr(v).StringToLower.StringTrim, Func(Of Object, Object)), EDP.ReturnValue)
+                                If .ListContains({"high", "low"}) Then
+                                    obj.Type = UMTypes.Audio
+                                    obj.Codec = ee.Value("ext")
+                                    If obj.Protocol.StringToLower.StartsWith("m3u8") Then obj.Protocol = "m3u8"
+                                    If obj.Bitrate <= 0 Then obj.Bitrate = IIf(.Contains("high"), 129, 53)
+                                    If obj.Size <= 0 Then obj.Size = 1
+                                    fd = True
+                                End If
+                            End With
                         End If
+                        If Not fd Then Continue For
                     End If
                     MediaObjects.Add(obj)
                 Next
@@ -1803,15 +1823,18 @@ Namespace API.YouTube.Objects
                         Dim data As New List(Of MediaObject)(MediaObjects.Where(Function(mo) mo.Type = t And mo.Extension = webm))
                         If data.Count > 0 Then
                             Dim d As MediaObject = Nothing
-                            Dim expWebm As Predicate(Of MediaObject) = Function(mo) mo.Extension = webm
-                            Dim expAVC As Predicate(Of MediaObject) = Function(mo) mo.Codec.IfNullOrEmpty("/").ToLower.StartsWith(avc)
+                            Dim allWebm As Boolean = False, allAVC As Boolean = False
+                            Dim expWebm As Predicate(Of MediaObject) = Function(mo) Not allWebm And mo.Extension = webm
+                            Dim expAVC As Predicate(Of MediaObject) = Function(mo) Not allAVC And mo.Codec.IfNullOrEmpty("/").ToLower.StartsWith(avc)
                             Dim comp As Func(Of MediaObject, Predicate(Of MediaObject), Boolean, Boolean, Boolean) =
                                 Function(mo, exp, isTrue, checkHttp) mo.Type = t And exp.Invoke(mo) = isTrue And mo.Width = d.Width And
                                                                      (Not checkHttp OrElse mo.ProtocolType = Protocols.https)
                             Dim CountWebm As Func(Of MediaObject, Boolean) = Function(mo) comp.Invoke(mo, expWebm, False, allowWebm)
                             Dim RemoveWebm As Predicate(Of MediaObject) = Function(mo) comp.Invoke(mo, expWebm, True, allowWebm)
                             Dim CountAVC As Func(Of MediaObject, Boolean) = Function(mo) comp.Invoke(mo, expAVC, True, False)
                             Dim RemoveAVC As Predicate(Of MediaObject) = Function(mo) comp.Invoke(mo, expAVC, False, False)
+                            allWebm = data.All(FPredicate(Of MediaObject).ToFunc(expWebm))
+                            allAVC = data.All(FPredicate(Of MediaObject).ToFunc(expAVC))
                             For Each d In data
                                 If MediaObjects.Count = 0 Then Exit For
                                 If MediaObjects.LongCount(CountWebm) > 0 Then MediaObjects.RemoveAll(RemoveWebm)

SCrawler.YouTubeDownloader/My Project/AssemblyInfo.vb

@@ -32,6 +32,6 @@ Imports System.Runtime.InteropServices
 ' by using the '*' as shown below:
 ' <Assembly: AssemblyVersion("1.0.*")>
 
-<Assembly: AssemblyVersion("2025.6.1.0")>
-<Assembly: AssemblyFileVersion("2025.6.1.0")>
+<Assembly: AssemblyVersion("2025.6.12.0")>
+<Assembly: AssemblyFileVersion("2025.6.12.0")>
 <Assembly: NeutralResourcesLanguage("en")>

SCrawler/API/Base/UserDataBase.vb

@@ -252,7 +252,20 @@ Namespace API.Base
 #End Region
 #Region "User name, ID, exist, suspend, options"
         Friend User As UserInfo
+        Private _IsSavedPosts As Boolean = False
         Friend Property IsSavedPosts As Boolean Implements IPluginContentProvider.IsSavedPosts
+            Get
+                Return _IsSavedPosts
+            End Get
+            Set(ByVal __IsSavedPosts As Boolean)
+                _IsSavedPosts = __IsSavedPosts
+                If _IsSavedPosts Then
+                    DownloadText = True
+                    DownloadTextPosts = True
+                    DownloadTextSpecialFolder = True
+                End If
+            End Set
+        End Property
         Private _UserExists As Boolean = True
         Friend Overridable Property UserExists As Boolean Implements IUserData.Exists, IPluginContentProvider.UserExists
             Get
@@ -300,7 +313,16 @@ Namespace API.Base
                 Return If(Exact, _NameTrue, _NameTrue.IfNullOrEmpty(Name))
             End Get
         End Property
-        Friend Overridable Property ID As String = String.Empty Implements IUserData.ID, IPluginContentProvider.ID
+        Private _ID As String = String.Empty
+        Friend Property ID As String Implements IUserData.ID, IPluginContentProvider.ID
+            Get
+                Return _ID
+            End Get
+            Set(ByVal NewId As String)
+                If Not _ID = NewId Then EnvirChanged(NewId)
+                _ID = NewId
+            End Set
+        End Property
         Protected _FriendlyName As String = String.Empty
         Friend Overridable Property FriendlyName As String Implements IUserData.FriendlyName
             Get
@@ -985,8 +1007,8 @@ BlockNullPicture:
                         ReadyForDownload = x.Value(Name_ReadyForDownload).FromXML(Of Boolean)(True)
                         DownloadImages = x.Value(Name_DownloadImages).FromXML(Of Boolean)(True)
                         DownloadVideos = x.Value(Name_DownloadVideos).FromXML(Of Boolean)(True)
-                        DownloadText = x.Value(Name_DownloadText).FromXML(Of Boolean)(False)
-                        DownloadTextPosts = x.Value(Name_DownloadTextPosts).FromXML(Of Boolean)(False)
+                        DownloadText = x.Value(Name_DownloadText).FromXML(Of Boolean)(IsSavedPosts)
+                        DownloadTextPosts = x.Value(Name_DownloadTextPosts).FromXML(Of Boolean)(IsSavedPosts)
                         DownloadTextSpecialFolder = x.Value(Name_DownloadTextSpecialFolder).FromXML(Of Boolean)(True)
                         _IconBannerDownloaded = x.Value(Name_IconBannerDownloaded).FromXML(Of Boolean)(False)
                         DownloadedVideos(True) = x.Value(Name_VideoCount).FromXML(Of Integer)(0)
@@ -1222,7 +1244,8 @@ BlockNullPicture:
                 Select Case Caller
                     Case NameOf(UserExists) : If Not _EnvirUserExists = CBool(NewValue) Then _EnvirChanged = True : _EnvirInvokeUserUpdated = True
                     Case NameOf(UserSuspended) : If Not _EnvirUserSuspended = CBool(NewValue) Then _EnvirChanged = True : _EnvirInvokeUserUpdated = True
-                    Case NameOf(NameTrue) : _EnvirChanged = True : _EnvirInvokeUserUpdated = True : _ForceSaveUserInfo = True : _ForceSaveUserInfoOnException = True
+                    Case NameOf(NameTrue) : _EnvirChanged = True : _ForceSaveUserInfo = True : _ForceSaveUserInfoOnException = True
+                    Case NameOf(ID) : _EnvirChanged = True : _ForceSaveUserInfo = True : _ForceSaveUserInfoOnException = True
                     Case Else : _EnvirChanged = True
                 End Select
             End If

SCrawler/API/Bluesky/UserData.vb

@@ -27,6 +27,7 @@ Namespace API.Bluesky
                 Return If(ID.IsEmptyString, String.Empty, SymbolsConverter.ASCII.EncodeSymbolsOnly(ID))
             End Get
         End Property
+        Private ReadOnly _TmpPosts2 As List(Of String)
 #End Region
 #Region "Loader"
         Protected Overrides Sub LoadUserInformation_OptionalFields(ByRef Container As XmlFile, ByVal Loading As Boolean)
@@ -42,6 +43,7 @@ Namespace API.Bluesky
 #Region "Initializer"
         Friend Sub New()
             UseInternalM3U8Function = True
+            _TmpPosts2 = New List(Of String)
         End Sub
 #End Region
 #Region "Token"
@@ -62,11 +64,17 @@ Namespace API.Bluesky
 #Region "Download"
         Private _PostCount As Integer = 0
         Protected Overrides Sub DownloadDataF(ByVal Token As CancellationToken)
-            If Not CBool(MySettings.CookiesEnabled.Value) Then Responser.Cookies.Clear()
-            UpdateToken(, True)
-            _TokenUpdateCount = 0
-            _PostCount = 0
-            DownloadData(String.Empty, Token)
+            _TmpPosts2.Clear()
+            Try
+                If Not CBool(MySettings.CookiesEnabled.Value) Then Responser.Cookies.Clear()
+                UpdateToken(, True)
+                _TokenUpdateCount = 0
+                _PostCount = 0
+                DownloadData(String.Empty, Token)
+            Finally
+                _TempPostsList.ListAddList(_TmpPosts2, LNC)
+                _TmpPosts2.Clear()
+            End Try
         End Sub
         Private Overloads Sub DownloadData(ByVal Cursor As String, ByVal Token As CancellationToken)
             Dim URL$ = String.Empty
@@ -117,7 +125,8 @@ Namespace API.Bluesky
         Private Function DefaultParser(ByVal e As EContainer, Optional ByVal CheckDateLimits As Boolean = True, Optional ByRef NextCursor As String = Nothing,
                                        Optional ByVal CheckTempPosts As Boolean = True, Optional ByVal State As UStates = UStates.Unknown) As Integer
             Const exitReturn% = CInt(DateResult.Exit) * -1
-            Dim postID$, postDate$, __url$, __urlBase$, __txt$
+            Const skipReturn% = CInt(DateResult.Skip) * -1
+            Dim postID$, postDate$, __url$, __urlBase$, __txt$, __userId$
             Dim updateUrl As Boolean
             Dim c% = 0
             Dim m As UserMedia
@@ -128,21 +137,26 @@ Namespace API.Bluesky
                     postDate = String.Empty
                     __urlBase = String.Empty
                     __txt = String.Empty
+                    __userId = .Value({"author"}, "did")
                     With .Item({"record"})
                         If .ListExists Then
                             '2025-01-28T02:42:12.415Z
                             postDate = .Value("createdAt")
                             NextCursor = postDate
                             If CheckDateLimits Then
                                 Select Case CheckDatesLimit(postDate, DateProvider)
-                                    Case DateResult.Skip : Return CInt(DateResult.Skip) * -1 'Continue For
+                                    Case DateResult.Skip : Return skipReturn 'Continue For
                                     Case DateResult.Exit : Return exitReturn 'Exit Sub
                                 End Select
                             End If
 
                             If CheckTempPosts Then
-                                If _TempPostsList.Contains(postID) Then Return exitReturn Else _TempPostsList.Add(postID)
+                                'If _TempPostsList.Contains(postID) Then Return exitReturn Else _TempPostsList.Add(postID)
+                                If _TempPostsList.Contains(postID) Then Return exitReturn Else _TmpPosts2.Add(postID)
                             End If
+
+                            If ParseUserMediaOnly And Not ID.IsEmptyString And Not __userId.IsEmptyString And Not ID = __userId Then Return skipReturn
+
                             __urlBase = $"https://bsky.app/profile/{NameTrue}/post/{postID}"
                         End If
                     End With
@@ -338,6 +352,12 @@ Namespace API.Bluesky
                 Return 0
             End If
         End Function
+#End Region
+#Region "IDisposable Support"
+        Protected Overrides Sub Dispose(ByVal disposing As Boolean)
+            If disposing Then _TmpPosts2.Clear()
+            MyBase.Dispose(disposing)
+        End Sub
 #End Region
     End Class
 End Namespace

SCrawler/API/Facebook/UserData.vb

@@ -676,10 +676,7 @@ Namespace API.Facebook
                     End If
                     Token_Photosby = RegexReplace(r, Regex_Photos_by)
                     If StoryBucket.IsEmptyString Then StoryBucket = RegexReplace(r, Regex_StoryBucket)
-                    If ID.IsEmptyString Then
-                        ID = RegexReplace(r, Regex_UserID)
-                        If Not ID.IsEmptyString Then _ForceSaveUserInfo = True
-                    End If
+                    If ID.IsEmptyString Then ID = RegexReplace(r, Regex_UserID)
                 End If
             Catch ex As Exception
                 ProcessException(ex, Token, "get user token",, resp)