Skip to content

Commit a0518a8

Browse files
committed
^q^
1 parent 47956c6 commit a0518a8

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+944
-484
lines changed

src/extractor/afreeca_downloader.py

Lines changed: 26 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,20 +6,21 @@
66
import errors
77
import utils
88
import os
9+
import dateutil.parser
910

1011

1112
class LoginRequired(errors.LoginRequired):
1213
def __init__(self, *args):
13-
super().__init__(*args, method='browser', url='https://login.afreecatv.com/afreeca/login.php')
14+
super().__init__(*args, method='browser', url='https://login.sooplive.co.kr/afreeca/login.php')
1415

1516

1617

1718
class Downloader_afreeca(Downloader):
1819
type = 'afreeca'
19-
URLS = ['afreecatv.com']
20+
URLS = ['afreecatv.com', 'sooplive.co.kr']
2021
single = True
21-
display_name = 'AfreecaTV'
22-
ACCEPT_COOKIES = [r'(.*\.)?afreecatv\.com']
22+
display_name = 'SOOP'
23+
ACCEPT_COOKIES = [r'(.*\.)?afreecatv\.com', r'(.*\.)?sooplive\.co\.kr']
2324

2425
def init(self):
2526
self.session = Session()
@@ -82,23 +83,24 @@ def get(self):
8283
if alert:
8384
raise LoginRequired(alert)
8485
soup = Soup(html)
86+
date = None
8587

8688
url_thumb = soup.find('meta', {'property': 'og:image'}).attrs['content']
8789
print_('url_thumb: {}'.format(url_thumb))
8890

8991
vid = re.find('/player/([0-9]+)', url)
9092
if vid is None: # live
91-
bid = re.find('afreecatv.com/([^/]+)', url, err='no bid')
93+
bid = re.find('sooplive.co.kr/([^/]+)', url, err='no bid')
9294

93-
url_api = f'https://st.afreecatv.com/api/get_station_status.php?szBjId={bid}'
95+
url_api = f'https://st.sooplive.co.kr/api/get_station_status.php?szBjId={bid}'
9496
r = session.post(url_api, headers={'Referer': url})
9597
d = json.loads(r.text)
9698
artist = d['DATA']['user_nick']
9799
if self._live_info is not None:
98100
self._live_info['title'] = artist
99101

100-
url_api = f'https://live.afreecatv.com/afreeca/player_live_api.php?bjid={bid}'
101-
#bno = re.find('afreecatv.com/[^/]+/([0-9]+)', url, err='no bno')
102+
url_api = f'https://live.sooplive.co.kr/afreeca/player_live_api.php?bjid={bid}'
103+
#bno = re.find('sooplive.co.kr/[^/]+/([0-9]+)', url, err='no bno')
102104
bno = re.find(r'nBroadNo\s=\s([0-9]+)', html, err='no bno') #6915
103105
r = session.post(url_api, data={'bid': bid, 'bno': bno, 'type': 'aid', 'pwd': '', 'player_type': 'html5', 'stream_type': 'common', 'quality': 'master', 'mode': 'landing', 'from_api': '0'}, headers={'Referer': url})
104106
d = json.loads(r.text)
@@ -110,11 +112,11 @@ def get(self):
110112

111113
data = {}
112114
data['title'] = soup.find('meta', {'property': 'og:title'})['content'].strip()
113-
data['files'] = [{'file': f'https://pc-web.stream.afreecatv.com/live-stm-16/auth_master_playlist.m3u8?aid={aid}'}]
115+
data['files'] = [{'file': f'https://pc-web.stream.sooplive.co.kr/live-stm-16/auth_master_playlist.m3u8?aid={aid}'}]
114116
data['writer_nick'] = artist
115117
data['live'] = True
116118
elif f'{vid}/catch' in url: #6215
117-
url_api = 'https://api.m.afreecatv.com/station/video/a/catchview'
119+
url_api = 'https://api.m.sooplive.co.kr/station/video/a/catchview'
118120
r = session.post(url_api, data={'nPageNo': '1', 'nLimit': '10', 'nTitleNo': vid}, headers={'Referer': url})
119121
try:
120122
s = cut_pair(r.text)
@@ -123,8 +125,9 @@ def get(self):
123125
print_(r.text)
124126
raise e
125127
data = d['data'][0]
128+
date = dateutil.parser.parse(data['reg_date']) #7054
126129
else:
127-
url_api = 'https://api.m.afreecatv.com/station/video/a/view'
130+
url_api = 'https://api.m.sooplive.co.kr/station/video/a/view'
128131
r = session.post(url_api, data={'nTitleNo': vid, 'nApiLevel': '10', 'nPlaylistIdx': '0'}, headers={'Referer': url})
129132
try:
130133
s = cut_pair(r.text)
@@ -133,6 +136,7 @@ def get(self):
133136
print_(r.text)
134137
raise e
135138
data = d['data']
139+
date = dateutil.parser.parse(data.get('broad_start') or data['write_tm']) #7054, #7093
136140

137141
title = data.get('full_title') or data['title']
138142
artist = data.get('copyright_nickname') or data.get('original_user_nick') or data['writer_nick']
@@ -150,9 +154,12 @@ def get(self):
150154
print_(f'urls_m3u8: {len(urls_m3u8)}')
151155

152156
if data.get('live'):
153-
hdr = session.headers.copy()
154-
hdr['Referer'] = url
155-
stream = utils.LiveStream(urls_m3u8[0], headers=hdr, cw=self.cw)
157+
stream = playlist2stream(urls_m3u8[0], url, session=session) #6934
158+
if stream.ms:
159+
stream = stream.live
160+
stream._cw = self.cw
161+
if not stream.live:
162+
stream.live = True#
156163
else:
157164
streams = []
158165
for url_m3u8 in urls_m3u8:
@@ -165,9 +172,10 @@ def get(self):
165172
for stream in streams[1:]:
166173
streams[0] += stream
167174
stream = streams[0]
175+
stream.live = None#
168176

169177
live = data.get('live') or False
170-
return {'url': stream, 'title': title, 'name': format_filename(title, vid, '.mp4', artist=artist, live=live), 'url_thumb': url_thumb, 'artist': artist, 'live': live}
178+
return {'url': stream, 'title': title, 'name': format_filename(title, vid, '.mp4', artist=artist, live=live, date=date), 'url_thumb': url_thumb, 'artist': artist, 'live': live}
171179

172180

173181

@@ -176,12 +184,12 @@ class Live_afreeca(utils.Live):
176184

177185
@classmethod
178186
def is_live(cls, url):
179-
return bool(re.match(r'https?://(play|bj).afreecatv.com/([^/?#]+)', url)) and url.strip('/').count('/') <= 4
187+
return bool(re.match(r'https?://(play|bj|ch).(afreecatv.com|sooplive.co.kr)/([^/?#]+)', url)) and url.strip('/').count('/') <= 4
180188

181189
@classmethod
182190
def fix_url(cls, url):
183-
bj = re.find(r'https?://(play|bj).afreecatv.com/([^/?#]+)', url)[1]
184-
return f'https://play.afreecatv.com/{bj}'
191+
bj = re.find(r'https?://(play|bj|ch).(afreecatv.com|sooplive.co.kr)/([^/?#]+)', url)[2]
192+
return f'https://play.sooplive.co.kr/{bj}'
185193

186194
@classmethod
187195
def check_live(cls, url, info=None):

src/extractor/artstation_downloader.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,6 @@ def read(self):
8686

8787
@try_n(2)
8888
def get_imgs(id_, title, session, cw=None):
89-
print_ = get_print(cw)
9089
referer = f'https://www.artstation.com/{id_}'
9190
downloader.read_html(referer, session=session)
9291
#print(session.cookies.keys())

src/extractor/bdsmlr_downloader.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ def id_(self):
3131
if 'bdsmlr.com' in url:
3232
if 'www.bdsmlr.com' in url:
3333
raise Exception('www.bdsmlr.com')
34-
gal_num = url.split('.bdsmlr.com')[0].split('/')[(-1)]
34+
gal_num = url.split('.bdsmlr.com')[0].split('/')[-1]
3535
else:
3636
gal_num = url
3737
return gal_num

src/extractor/bili_downloader.py

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import ree as re
99
import ytdl
1010
import constants
11-
from putils import DIR
11+
import putils
1212
import threading
1313
import errors
1414
_VALID_URL = r'''(?x)
@@ -76,17 +76,19 @@ def get(self):
7676

7777
mobj = re.match(_VALID_URL, self['referer'])
7878
video_id = mobj.group('id')
79+
artist = info.get('uploader') or None
7980

8081
info = {
8182
'url': f_video['url'],
8283
'url_thumb': url_thumb,
83-
'name': format_filename(title, video_id, ext),
84+
'artist': artist,
85+
'name': format_filename(title, video_id, ext, artist=artist), #7127
8486
}
8587

8688
if f_audio:
8789
def f():
8890
audio = f_audio['url']
89-
path = os.path.join(DIR, f'{uuid()}_a.tmp')
91+
path = os.path.join(putils.DIRf, f'{uuid()}_a.tmp')
9092
if cw is not None:
9193
cw.trash_can.append(path)
9294
if constants.FAST:
@@ -211,3 +213,4 @@ def read(self):
211213
n = int(math.ceil(8.0 / len(self.urls)))
212214
self.print_(f'n_threads: {n}')
213215
self.enableSegment(n_threads=n, overwrite=True)
216+
self.artist = file['artist']

src/extractor/danbooru_downloader.py

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,23 @@
11
#coding: utf-8
22
import downloader
33
import ree as re
4-
from utils import Downloader, get_max_range, clean_title, get_print, try_n, urljoin, check_alive, LazyUrl, get_ext, limits
4+
from utils import Downloader, Session, get_max_range, clean_title, get_print, try_n, urljoin, check_alive, LazyUrl, get_ext, limits
55
from translator import tr_
66
from urllib.parse import urlparse, parse_qs, quote
77
import clf2
88

99

10-
1110
class Downloader_danbooru(Downloader):
1211
type = 'danbooru'
1312
URLS = ['danbooru.donmai.us']
14-
MAX_CORE = 6
13+
MAX_CORE = 4
1514
_name = None
1615
ACCEPT_COOKIES = [r'(.*\.)?donmai\.us']
1716

1817
def init(self):
19-
self.session = clf2.solve(self.url, cw=self.cw)['session'] #5336
18+
self.session = Session('chrome')
19+
clf2.solve(self.url, session=self.session, cw=self.cw) #5336
20+
self.session.headers['User-Agent'] = 'Mozilla/5.' #7034
2021

2122
@classmethod
2223
def fix_url(cls, url):
@@ -95,7 +96,7 @@ def get(self, url):
9596
return img, None
9697

9798

98-
@limits(.5)
99+
@limits(1)
99100
def wait(cw):
100101
check_alive(cw)
101102

src/extractor/etc_downloader.py

Lines changed: 63 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import downloader
22
import ytdl
3-
from utils import Downloader, Session, try_n, LazyUrl, get_ext, format_filename, get_print, get_resolution
3+
from utils import Downloader, Session, try_n, LazyUrl, get_ext, format_filename, get_print, get_resolution, print_error, cut_pair, json
44
from io import BytesIO
55
import ree as re
66
from m3u8_tools import playlist2stream, M3u8_stream
@@ -19,7 +19,8 @@ class Downloader_etc(Downloader):
1919
display_name = 'Etc'
2020
PRIORITY = 10
2121

22-
def init(self):
22+
@try_n(2)
23+
def read(self):
2324
self.session = Session()
2425
name = ytdl.get_extractor_name(self.url)
2526
self.print_('extractor: {}'.format(name))
@@ -28,21 +29,20 @@ def init(self):
2829
#if name == 'generic':
2930
# raise NotImplementedError()
3031

31-
def read(self):
3232
video = get_video(self.url, self.session, self.cw)
3333

3434
if video.artist:
3535
self.artist = video.artist
3636

37-
self.urls.append(video.url)
38-
3937
self.print_('url_thumb: {}'.format(video.url_thumb))
4038
self.setIcon(video.thumb)
41-
if video.header.lower() not in ['yourporn', 'spankbang']:
39+
if video.header.lower() not in ['yourporn']:
4240
self.enableSegment()#
4341
if isinstance(video.url(), M3u8_stream):
4442
self.disableSegment()
4543

44+
self.urls.append(video.url)
45+
4646
self.title = os.path.splitext(video.filename)[0].replace(':', ':')
4747

4848

@@ -69,17 +69,52 @@ def get_video(url, session, cw, ie_key=None):
6969
if isinstance(video, Exception):
7070
raise video
7171
if isinstance(video.url(), M3u8_stream):
72-
c = video.url().segs[0].download(2, cw)
72+
c = video.url().urls[0][1].download(cw)
7373
if not c:
7474
raise Exception('invalid m3u8')
7575
return video
7676
except Exception as e:
7777
if isinstance(e, UnSupportedError):
7878
raise e
79-
print_(e)
79+
print_(print_error(e))
8080
return _get_video(url, session, cw, ie_key, allow_m3u8=False)
8181

82-
@try_n(4)
82+
83+
def extract_info_spankbang(url, session, cw): # temp fix
84+
print_ = get_print(cw)
85+
soup = downloader.read_soup(url, session=session)
86+
87+
for script in soup.findAll('script'):
88+
script = script.string
89+
if script and 'var stream_data'in script:
90+
s = cut_pair(script.split('var stream_data')[-1].strip(' \t=').replace("'", '"'))
91+
break
92+
else:
93+
raise Exception('no stream_data')
94+
95+
info = {}
96+
info['url'] = url
97+
info['title'] = soup.find('h1').text.strip()
98+
info['id'] = re.find(r'spankbang\.com/([^/]+)', soup.find('meta', {'property': 'og:url'})['content'], err='no id')
99+
info['thumbnail'] = soup.find('meta', {'property': 'og:image'})['content']
100+
info['formats'] = []
101+
data = json.loads(s)
102+
for res, item in data.items():
103+
if res == 'main':
104+
continue
105+
if item and isinstance(item, list):
106+
item = item[0]
107+
else:
108+
continue
109+
ext = get_ext_(item, session, url)
110+
res = {'4k': '2160p', '8k': '4320p', '16k': '8640p'}.get(res, res)
111+
f = {'url': item, 'format': res}
112+
info['formats'].append(f)
113+
114+
return info
115+
116+
117+
@try_n(2)
83118
def _get_video(url, session, cw, ie_key=None, allow_m3u8=True):
84119
print_ = get_print(cw)
85120
print_('get_video: {}, {}'.format(allow_m3u8, url))
@@ -90,14 +125,15 @@ def _get_video(url, session, cw, ie_key=None, allow_m3u8=True):
90125
'writesubtitles': True,
91126
}
92127
if ytdl.get_extractor_name(url) == 'spankbang':
93-
options['legacyserverconnect'] = True #6545
94-
ydl = ytdl.YoutubeDL(options, cw=cw)
95-
try:
96-
info = ydl.extract_info(url)
97-
except Exception as e:
98-
if 'ERROR: Unsupported URL' in str(e):
99-
return UnSupportedError(str(e))
100-
raise e
128+
info = extract_info_spankbang(url, session, cw)
129+
else:
130+
ydl = ytdl.YoutubeDL(options, cw=cw)
131+
try:
132+
info = ydl.extract_info(url)
133+
except Exception as e:
134+
if 'ERROR: Unsupported URL' in str(e):
135+
return UnSupportedError(str(e))
136+
raise e
101137
if not ie_key:
102138
ie_key = ytdl.get_extractor_name(url)
103139
info['ie_key'] = ie_key
@@ -191,6 +227,7 @@ class Video:
191227
def __init__(self, f, f_audio, info, session, referer, cw=None):
192228
self.f_audio = f_audio
193229
self.cw = cw
230+
print_ = get_print(cw)
194231
self.title = title = info['title']
195232
self.id = info['id']
196233
self.url = f['url']
@@ -222,15 +259,19 @@ def foo():
222259

223260
if ext.lower() == '.m3u8':
224261
res = get_resolution() #4773
225-
if info.get('live_status') == 'is_live':
262+
ls = info.get('live_status')
263+
print_(f'live_status: {ls}')
264+
if ls == 'is_live':
226265
url = foo()
227266
else:
228267
try:
229-
url = playlist2stream(self.url, referer, session=session, n_thread=4)
268+
url = playlist2stream(self.url, referer, session=session)
230269
except:
231-
url = M3u8_stream(self.url, referer=referer, session=session, n_thread=4)
232-
if url.live is not None: #5110
233-
url = foo()
270+
url = M3u8_stream(self.url, referer=referer, session=session)
271+
print_(f'mpegts: {url.mpegts}')
272+
if url.ms or url.mpegts == False: #5110
273+
url = url.live
274+
url._cw = cw
234275
ext = '.mp4'
235276
elif ext.lower() == '.mpd': # TVer
236277
url = foo()

0 commit comments

Comments
 (0)