^q^

Author: KurtBestor

src/extractor/afreeca_downloader.py

@@ -1,10 +1,7 @@
 import downloader
-from utils import Soup, Downloader, get_outdir, Session, LazyUrl, try_n, format_filename, cut_pair, File
+from utils import Soup, Downloader, Session, try_n, format_filename, cut_pair, File, get_print
 import ree as re
-from timee import sleep, time
-import os
 from io import BytesIO
-import shutil
 from m3u8_tools import playlist2stream, M3u8_stream
 import errors
 import json
@@ -57,6 +54,7 @@ class Video(File):
     type = 'afreeca'
 
     def get(self):
+        print_ = get_print(self.cw)
         url, session = self['referer'], self.session
 
         html = downloader.read_html(url, session=session)
@@ -71,7 +69,7 @@ def get(self):
         url_thumb = soup.find('meta', {'property': 'og:image'}).attrs['content']
         print_('url_thumb: {}'.format(url_thumb))
 
-        vid = re.find(f'/player/([0-9]+)', url, err='no vid')
+        vid = re.find('/player/([0-9]+)', url, err='no vid')
         if f'{vid}/catch' in url: #6215
             url_api = 'https://api.m.afreecatv.com/station/video/a/catchview'
             r = session.post(url_api, data={'nPageNo': '1', 'nLimit': '10', 'nTitleNo': vid}, headers={'Referer': url})
@@ -105,7 +103,7 @@ def get(self):
             else:
                 file = file['file']
             urls_m3u8.append(file)
-        self.cw.print_(f'urls_m3u8: {len(urls_m3u8)}')
+        print_(f'urls_m3u8: {len(urls_m3u8)}')
 
         streams = []
         for url_m3u8 in urls_m3u8:
@@ -119,4 +117,4 @@ def get(self):
             streams[0] += stream
         stream = streams[0]
 
-        return {'url': stream, 'title': title, 'name': format_filename(title, id, '.mp4'), 'url_thumb': url_thumb}
+        return {'url': stream, 'title': title, 'name': format_filename(title, vid, '.mp4'), 'url_thumb': url_thumb}

src/extractor/artstation_downloader.py

@@ -2,7 +2,7 @@
 import downloader
 from error_printer import print_error
 from translator import tr_
-from utils import Downloader, Soup, get_print, lazy, Session, try_n, File, clean_title, check_alive, File, get_ext, get_max_range
+from utils import Downloader, Soup, get_print, lazy, Session, try_n, File, clean_title, check_alive, get_ext, get_max_range
 import dateutil.parser
 import utils
 
@@ -46,6 +46,14 @@ def init(self):
             self.url = self.url_main
         self.print_(self.url)
 
+    @classmethod
+    def fix_url(cls, url): #6516
+        if '.artstation.com' in url:
+            sub = url.split('.artstation.com')[0].split('/')[-1]
+            if sub != 'www':
+                url = f'https://www.artstation.com/{sub}'
+        return url
+
     @lazy
     def _id(self):
         _id = get_id(self.url, self.cw)
@@ -86,41 +94,25 @@ def get_imgs(id_, title, session, cw=None):
     url = f'https://www.artstation.com/users/{id_}/quick.json'
     j = downloader.read_json(url, referer, session=session)
     uid = j['id']
-    aids = [a['id'] for a in j['albums_with_community_projects']]
-    print_(f'albums: {aids}')
 
     datas = []
     ids = set()
-    for aid in aids:
-        p = 1
-        while p < 1000:
-            check_alive(cw)
-            url = f'https://www.artstation.com/users/{id_}/projects.json?album_id={aid}&page={p}&user_id={uid}'
-            print(url)
-            _e = None
-            for i in range(4):
-                try:
-                    j = downloader.read_json(url, referer, session=session)
-                    break
-                except Exception as e:
-                    _e = e
-                    print(e)
-            else:
-                if _e is not None:
-                    raise _e
-
-            data = j['data']
-            if not data:
-                break
-            for d in data:
-                if d['id'] not in ids:
-                    ids.add(d['id'])
-                    datas.append(d)
-            if cw:
-                cw.setTitle(f'{tr_("페이지 읽는 중...")}  {title} - {len(datas)}')
-            else:
-                print(len(datas))
-            p += 1
+    for p in range(1, 1000):
+        check_alive(cw)
+        url = f'https://www.artstation.com/users/{id_}/projects.json??user_id={uid}&page={p}' #6516
+        j = try_n(4)(downloader.read_json)(url, referer, session=session)
+
+        data = j['data']
+        if not data:
+            break
+        for d in data:
+            if d['id'] not in ids:
+                ids.add(d['id'])
+                datas.append(d)
+        if cw:
+            cw.setTitle(f'{tr_("페이지 읽는 중...")}  {title} - {len(datas)}')
+        else:
+            print(len(datas))
 
     datas = sorted(datas, key=lambda data: int(data['id']), reverse=True)
 
@@ -234,7 +226,7 @@ def get_imgs_page(id_art, session, date=None, cw=None, names=None):
 
         d = {
             'date': date,
-            'name': name,
+            'name': clean_title(name),
             'page': page,
             }
         filename = utils.format('artstation', d, get_ext(url))

src/extractor/asmhentai_downloader.py

@@ -1,7 +1,7 @@
 #coding: utf8
 import downloader
 import ree as re
-from utils import Soup, urljoin, Downloader, join, Session, get_print, File
+from utils import Soup, urljoin, Downloader, join, Session, File, clean_title
 import os
 from ratelimit import limits, sleep_and_retry
 import utils
@@ -27,7 +27,7 @@ def get(self):
         url = img['data-src']
         name, ext = os.path.splitext(os.path.basename(url).split('?')[0])
         d = {
-            'name': name,
+            'name': clean_title(name),
             }
         return {'url': url, 'name': utils.format('asmhentai', d, ext)}
 
@@ -68,7 +68,6 @@ def read(self):
 
 
 def get_info(url, session, cw):
-    print_ = get_print(cw)
     html = downloader.read_html(url, session=session)
     soup = Soup(html)
 

src/extractor/avgle_downloader.py

@@ -1,11 +1,8 @@
 #coding: utf8
 import downloader
-import os
 from m3u8_tools import M3u8_stream
-from utils import Soup, Downloader, LazyUrl, get_print, try_n, clean_title, check_alive, format_filename
+from utils import Soup, Downloader, LazyUrl, get_print, try_n, check_alive, format_filename
 from io import BytesIO
-import constants
-from error_printer import print_error
 import base64
 import json
 import webbrowser

src/extractor/bcy_downloader.py

@@ -2,7 +2,6 @@
 import downloader
 from utils import Soup, cut_pair, LazyUrl, Downloader, get_print, get_max_range, try_n, clean_title, check_alive
 import json
-import ree as re
 import os
 from translator import tr_
 

src/extractor/bdsmlr_downloader.py

@@ -5,7 +5,6 @@
 import ree as re
 import os
 from translator import tr_
-from timee import sleep
 from error_printer import print_error
 import clf2
 import errors

src/extractor/bili_downloader.py

@@ -1,12 +1,11 @@
 import downloader
 import downloader_v3
-from utils import Downloader, get_print, format_filename, clean_title, get_resolution, try_n, Session, uuid, File, get_max_range
+from utils import Downloader, get_print, format_filename, clean_title, get_resolution, try_n, Session, uuid, File, get_max_range, query_url
 import os
 from io import BytesIO
 import ffmpeg
 import math
 import ree as re
-import utils
 import ytdl
 import constants
 from putils import DIR
@@ -154,13 +153,15 @@ def init(self):
     def key_id(cls, url):
         mobj = re.match(_VALID_URL, url)
         video_id = mobj.group('id')
-        return video_id or url
+        qs = query_url(url)
+        p = qs.get('p', ['1'])[0] #6580
+        return f'{video_id or url} {p}'
 
     @property
     def id_(self):
         mobj = re.match(_VALID_URL, self.url)
         video_id = mobj.group('id')
-        anime_id = mobj.group('anime_id')
+        #anime_id = mobj.group('anime_id')
         return video_id
 
     def read(self):

src/extractor/comicwalker_downloader.py

@@ -4,7 +4,6 @@
 import ree as re
 from itertools import cycle
 from io import BytesIO
-from timee import sleep
 from translator import tr_
 import page_selector
 import utils
@@ -77,7 +76,7 @@ def get_imgs_page(page, cw):
         if hash is None:
             continue
         d = {
-            'title': page.title,
+            'title': clean_title(page.title),
             'page': len(imgs),
             }
         img = File_comicwalker({'src': src, 'hash': hash, 'referer': page.url, 'name': utils.format('comicwalker', d, '.jpg')})
@@ -113,7 +112,6 @@ def get_pages(url, soup=None):
 
 
 def get_title(soup, cw=None):
-    print_ = get_print(cw)
     div = soup.find('div', class_='comicIndex-box') #6231
     if div is None:
         raise Exception('no title')

src/extractor/coub_downloader.py

@@ -3,7 +3,6 @@
 from io import BytesIO as IO
 import downloader
 import ree as re
-import ffmpeg
 PATTEN_IMAGIZER = r'coub-com-.+\.imagizer\.com'
 
 
@@ -67,6 +66,7 @@ def get(self,  url):
         return self._url
 
     def pp(self, filename):
+##        import ffmpeg
 ##        f = IO()
 ##        downloader.download(self.f_audio['url'], buffer=f)
 ##        ffmpeg.merge(filename, f)

src/extractor/etc_downloader.py

@@ -86,6 +86,7 @@ def _get_video(url, session, cw, ie_key=None, allow_m3u8=True):
         'noplaylist': True,
         #'extract_flat': True,
         'playlistend': 1,
+        'writesubtitles': True,
         }
     if ytdl.get_extractor_name(url) == 'spankbang':
         options['legacyserverconnect'] = True #6545