diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py index 7726b1943..8fdcce8d4 100644 --- a/test/test_youtube_signature.py +++ b/test/test_youtube_signature.py @@ -84,6 +84,21 @@ _SIG_TESTS = [ '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', '0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xxAj7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJ2OySqa0q', ), + ( + 'https://www.youtube.com/s/player/643afba4/tv-player-ias.vflset/tv-player-ias.js', + '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + 'AAOAOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7vgpDL0QwbdV06sCIEzpWqMGkFR20CFOS21Tp-7vj_EMu-m37KtXJoOy1', + ), + ( + 'https://www.youtube.com/s/player/363db69b/player_ias.vflset/en_US/base.js', + '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + '0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpz2ICs6EVdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + ), + ( + 'https://www.youtube.com/s/player/4fcd6e4a/player_ias.vflset/en_US/base.js', + '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', + 'wAOAOq0QJ8ARAIgXmPlOPSBkkUs1bYFYlJCfe29xx8q7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0', + ), ] _NSIG_TESTS = [ @@ -153,7 +168,7 @@ _NSIG_TESTS = [ ), ( 'https://www.youtube.com/s/player/c57c113c/player_ias.vflset/en_US/base.js', - '-Txvy6bT5R6LqgnQNx', 'dcklJCnRUHbgSg', + 'M92UUMHa8PdvPd3wyM', '3hPqLJsiNZx7yA', ), ( 'https://www.youtube.com/s/player/5a3b6271/player_ias.vflset/en_US/base.js', @@ -173,7 +188,7 @@ _NSIG_TESTS = [ ), ( 'https://www.youtube.com/s/player/cfa9e7cb/player_ias.vflset/en_US/base.js', - 'qO0NiMtYQ7TeJnfFG2', 'k9cuJDHNS5O7kQ', + 'aCi3iElgd2kq0bxVbQ', 'QX1y8jGb2IbZ0w', ), ( 'https://www.youtube.com/s/player/8c7583ff/player_ias.vflset/en_US/base.js', @@ -231,10 +246,6 @@ _NSIG_TESTS = [ 'https://www.youtube.com/s/player/f6e09c70/player_ias_tce.vflset/en_US/base.js', 'W9HJZKktxuYoDTqW', 'jHbbkcaxm54', ), - ( - 'https://www.youtube.com/s/player/643afba4/player_ias.vflset/en_US/base.js', - 'W9HJZKktxuYoDTqW', 'larxUlagTRAcSw', - ), ( 'https://www.youtube.com/s/player/e7567ecf/player_ias_tce.vflset/en_US/base.js', 'Sy4aDGc0VpYRR9ew_', '5UPOT1VhoZxNLQ', @@ -259,6 +270,22 @@ _NSIG_TESTS = [ 'https://www.youtube.com/s/player/643afba4/tv-player-ias.vflset/tv-player-ias.js', 'ir9-V6cdbCiyKxhr', '2PL7ZDYAALMfmA', ), + ( + 'https://www.youtube.com/s/player/643afba4/player_ias.vflset/en_US/base.js', + 'W9HJZKktxuYoDTqW', 'larxUlagTRAcSw', + ), + ( + 'https://www.youtube.com/s/player/363db69b/player_ias.vflset/en_US/base.js', + 'eWYu5d5YeY_4LyEDc', 'XJQqf-N7Xra3gg', + ), + ( + 'https://www.youtube.com/s/player/4fcd6e4a/player_ias.vflset/en_US/base.js', + 'o_L251jm8yhZkWtBW', 'lXoxI3XvToqn6A', + ), + ( + 'https://www.youtube.com/s/player/4fcd6e4a/tv-player-ias.vflset/tv-player-ias.js', + 'o_L251jm8yhZkWtBW', 'lXoxI3XvToqn6A', + ), ] @@ -271,6 +298,8 @@ class TestPlayerInfo(unittest.TestCase): ('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-phone-en_US.vflset/base.js', '64dddad9'), ('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-phone-de_DE.vflset/base.js', '64dddad9'), ('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-tablet-en_US.vflset/base.js', '64dddad9'), + ('https://www.youtube.com/s/player/e7567ecf/player_ias_tce.vflset/en_US/base.js', 'e7567ecf'), + ('https://www.youtube.com/s/player/643afba4/tv-player-ias.vflset/tv-player-ias.js', '643afba4'), # obsolete ('https://www.youtube.com/yts/jsbin/player_ias-vfle4-e03/en_US/base.js', 'vfle4-e03'), ('https://www.youtube.com/yts/jsbin/player_ias-vfl49f_g4/en_US/base.js', 'vfl49f_g4'), @@ -280,8 +309,9 @@ class TestPlayerInfo(unittest.TestCase): ('https://s.ytimg.com/yts/jsbin/html5player-en_US-vflXGBaUN.js', 'vflXGBaUN'), ('https://s.ytimg.com/yts/jsbin/html5player-en_US-vflKjOTVq/html5player.js', 'vflKjOTVq'), ) + ie = YoutubeIE(FakeYDL({'cachedir': False})) for player_url, expected_player_id in PLAYER_URLS: - player_id = YoutubeIE._extract_player_info(player_url) + player_id = ie._extract_player_info(player_url) self.assertEqual(player_id, expected_player_id) @@ -301,8 +331,8 @@ class TestSignature(unittest.TestCase): def t_factory(name, sig_func, url_pattern): def make_tfunc(url, sig_input, expected_sig): m = url_pattern.match(url) - assert m, '%r should follow URL format' % url - test_id = m.group('id') + assert m, '{0!r} should follow URL format'.format(url) + test_id = re.sub(r'[/.-]', '_', m.group('id') or m.group('compat_id')) def test_func(self): basename = 'player-{0}-{1}.js'.format(name, test_id) @@ -335,12 +365,16 @@ def n_sig(jscode, sig_input): make_sig_test = t_factory( - 'signature', signature, re.compile(r'.*(?:-|/player/)(?P[a-zA-Z0-9_-]+)(?:/.+\.js|(?:/watch_as3|/html5player)?\.[a-z]+)$')) + 'signature', signature, + re.compile(r'''(?x) + .+/(?P
html5)?player(?(h5)(?:-en_US)?-|/)(?P[a-zA-Z0-9/._-]+) + (?(h5)/(?:watch_as3|html5player))?\.js$ + ''')) for test_spec in _SIG_TESTS: make_sig_test(*test_spec) make_nsig_test = t_factory( - 'nsig', n_sig, re.compile(r'.+/player/(?P[a-zA-Z0-9_-]+)/.+.js$')) + 'nsig', n_sig, re.compile(r'.+/player/(?P[a-zA-Z0-9_/.-]+)\.js$')) for test_spec in _NSIG_TESTS: make_nsig_test(*test_spec) diff --git a/youtube_dl/extractor/bokecc.py b/youtube_dl/extractor/bokecc.py index 6017e8344..4b8bef391 100644 --- a/youtube_dl/extractor/bokecc.py +++ b/youtube_dl/extractor/bokecc.py @@ -32,7 +32,7 @@ class BokeCCBaseIE(InfoExtractor): class BokeCCIE(BokeCCBaseIE): - _IE_DESC = 'CC视频' + IE_DESC = 'CC视频' _VALID_URL = r'https?://union\.bokecc\.com/playvideo\.bo\?(?P.*)' _TESTS = [{ diff --git a/youtube_dl/extractor/cloudy.py b/youtube_dl/extractor/cloudy.py index 85ca20ecc..d39a9a5c2 100644 --- a/youtube_dl/extractor/cloudy.py +++ b/youtube_dl/extractor/cloudy.py @@ -9,7 +9,7 @@ from ..utils import ( class CloudyIE(InfoExtractor): - _IE_DESC = 'cloudy.ec' + IE_DESC = 'cloudy.ec' _VALID_URL = r'https?://(?:www\.)?cloudy\.ec/(?:v/|embed\.php\?.*?\bid=)(?P[A-Za-z0-9]+)' _TESTS = [{ 'url': 'https://www.cloudy.ec/v/af511e2527aac', diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 78704b557..cb67b976d 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -422,6 +422,8 @@ class InfoExtractor(object): _GEO_COUNTRIES = None _GEO_IP_BLOCKS = None _WORKING = True + # supply this in public subclasses: used in supported sites list, etc + # IE_DESC = 'short description of IE' def __init__(self, downloader=None): """Constructor. Receives an optional downloader.""" diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py index c64af3be6..2510ad887 100644 --- a/youtube_dl/extractor/itv.py +++ b/youtube_dl/extractor/itv.py @@ -35,15 +35,6 @@ from ..utils import ( class ITVBaseIE(InfoExtractor): - def _search_nextjs_data(self, webpage, video_id, **kw): - transform_source = kw.pop('transform_source', None) - fatal = kw.pop('fatal', True) - return self._parse_json( - self._search_regex( - r''']+\bid=('|")__NEXT_DATA__\1[^>]*>(?P[^<]+)''', - webpage, 'next.js data', group='js', fatal=fatal, **kw), - video_id, transform_source=transform_source, fatal=fatal) - def __handle_request_webpage_error(self, err, video_id=None, errnote=None, fatal=True): if errnote is False: return False @@ -109,7 +100,9 @@ class ITVBaseIE(InfoExtractor): class ITVIE(ITVBaseIE): _VALID_URL = r'https?://(?:www\.)?itv\.com/(?:(?Pwatch)|hub)/[^/]+/(?(w)[\w-]+/)(?P\w+)' - _IE_DESC = 'ITVX' + IE_DESC = 'ITVX' + _WORKING = False + _TESTS = [{ 'note': 'Hub URLs redirect to ITVX', 'url': 'https://www.itv.com/hub/liar/2a4547a0012', @@ -270,7 +263,7 @@ class ITVIE(ITVBaseIE): 'ext': determine_ext(href, 'vtt'), }) - next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default='{}') + next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default={}) video_data.update(traverse_obj(next_data, ('props', 'pageProps', ('title', 'episode')), expected_type=dict)[0] or {}) title = traverse_obj(video_data, 'headerTitle', 'episodeTitle') info = self._og_extract(webpage, require_title=not title) @@ -323,7 +316,7 @@ class ITVIE(ITVBaseIE): class ITVBTCCIE(ITVBaseIE): _VALID_URL = r'https?://(?:www\.)?itv\.com/(?!(?:watch|hub)/)(?:[^/]+/)+(?P[^/?#&]+)' - _IE_DESC = 'ITV articles: News, British Touring Car Championship' + IE_DESC = 'ITV articles: News, British Touring Car Championship' _TESTS = [{ 'note': 'British Touring Car Championship', 'url': 'https://www.itv.com/btcc/articles/btcc-2018-all-the-action-from-brands-hatch', diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py index db5ef8b57..b8ac58713 100644 --- a/youtube_dl/extractor/senateisvp.py +++ b/youtube_dl/extractor/senateisvp.py @@ -47,7 +47,7 @@ class SenateISVPIE(InfoExtractor): ['vetaff', '76462', 'http://vetaff-f.akamaihd.net'], ['arch', '', 'http://ussenate-f.akamaihd.net/'] ] - _IE_NAME = 'senate.gov' + IE_NAME = 'senate.gov' _VALID_URL = r'https?://(?:www\.)?senate\.gov/isvp/?\?(?P.+)' _TESTS = [{ 'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png', diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 415354811..ce97fd75b 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -692,9 +692,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'invidious': '|'.join(_INVIDIOUS_SITES), } _PLAYER_INFO_RE = ( - r'/s/player/(?P[a-zA-Z0-9_-]{8,})//(?:tv-)?player', - r'/(?P[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$', - r'\b(?Pvfl[a-zA-Z0-9_-]+)\b.*?\.js$', + r'/s/player/(?P[a-zA-Z0-9_-]{8,})/(?:tv-)?player', + r'/(?P[a-zA-Z0-9_-]{8,})/player(?:_ias(?:_tce)?\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$', + r'\b(?Pvfl[a-zA-Z0-9_-]{6,})\b.*?\.js$', ) _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt') @@ -1626,15 +1626,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor): """ Return a string representation of a signature """ return '.'.join(compat_str(len(part)) for part in example_sig.split('.')) - @classmethod - def _extract_player_info(cls, player_url): - for player_re in cls._PLAYER_INFO_RE: - id_m = re.search(player_re, player_url) - if id_m: - break - else: - raise ExtractorError('Cannot identify player %r' % player_url) - return id_m.group('id') + def _extract_player_info(self, player_url): + try: + return self._search_regex( + self._PLAYER_INFO_RE, player_url, 'player info', group='id') + except ExtractorError as e: + raise ExtractorError( + 'Cannot identify player %r' % (player_url,), cause=e) def _load_player(self, video_id, player_url, fatal=True, player_id=None): if not player_id: @@ -1711,6 +1709,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor): ' return %s\n') % (signature_id_tuple, expr_code) self.to_screen('Extracted signature function:\n' + code) + def _extract_sig_fn(self, jsi, funcname): + var_ay = self._search_regex( + r'''(?x) + (?:\*/|\{|\n|^)\s*(?:'[^']+'\s*;\s*) + (var\s*[\w$]+\s*=\s*(?: + ('|")(?:\\\2|(?!\2).)+\2\s*\.\s*split\(\s*('|")\W+\3\s*\)| + \[\s*(?:('|")(?:\\\4|(?!\4).)*\4\s*(?:(?=\])|,\s*))+\] + ))(?=\s*[,;]) + ''', jsi.code, 'useful values', default='') + + sig_fn = jsi.extract_function_code(funcname) + + if var_ay: + sig_fn = (sig_fn[0], ';\n'.join((var_ay, sig_fn[1]))) + + return sig_fn + def _parse_sig_js(self, jscode): # Examples where `sig` is funcname: # sig=function(a){a=a.split(""); ... ;return a.join("")}; @@ -1736,8 +1751,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor): jscode, 'Initial JS player signature function name', group='sig') jsi = JSInterpreter(jscode) - initial_function = jsi.extract_function(funcname) - return lambda s: initial_function([s]) + + initial_function = self._extract_sig_fn(jsi, funcname) + + func = jsi.extract_function_from_code(*initial_function) + + return lambda s: func([s]) def _cached(self, func, *cache_id): def inner(*args, **kwargs): @@ -1856,15 +1875,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor): def _extract_n_function_code_jsi(self, video_id, jsi, player_id=None): - var_ay = self._search_regex( - r'(?:[;\s]|^)\s*(var\s*[\w$]+\s*=\s*"(?:\\"|[^"])+"\s*\.\s*split\("\W+"\))(?=\s*[,;])', - jsi.code, 'useful values', default='') - func_name = self._extract_n_function_name(jsi.code) - func_code = jsi.extract_function_code(func_name) - if var_ay: - func_code = (func_code[0], ';\n'.join((var_ay, func_code[1]))) + func_code = self._extract_sig_fn(jsi, func_name) if player_id: self.cache.store('youtube-nsig', player_id, func_code) @@ -2136,7 +2149,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor): video_details = merge_dicts(*traverse_obj( (player_response, api_player_response), (Ellipsis, 'videoDetails', T(dict)))) - player_response.update(api_player_response or {}) + player_response.update(filter_dict( + api_player_response or {}, cndn=lambda k, _: k != 'captions')) player_response['videoDetails'] = video_details def is_agegated(playability): @@ -2566,8 +2580,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor): } pctr = traverse_obj( - player_response, - ('captions', 'playerCaptionsTracklistRenderer', T(dict))) + (player_response, api_player_response), + (Ellipsis, 'captions', 'playerCaptionsTracklistRenderer', T(dict))) if pctr: def process_language(container, base_url, lang_code, query): lang_subs = [] @@ -2584,20 +2598,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor): def process_subtitles(): subtitles = {} for caption_track in traverse_obj(pctr, ( - 'captionTracks', lambda _, v: v.get('baseUrl'))): + Ellipsis, 'captionTracks', lambda _, v: ( + v.get('baseUrl') and v.get('languageCode')))): base_url = self._yt_urljoin(caption_track['baseUrl']) if not base_url: continue + lang_code = caption_track['languageCode'] if caption_track.get('kind') != 'asr': - lang_code = caption_track.get('languageCode') - if not lang_code: - continue process_language( subtitles, base_url, lang_code, {}) continue automatic_captions = {} + process_language( + automatic_captions, base_url, lang_code, {}) for translation_language in traverse_obj(pctr, ( - 'translationLanguages', lambda _, v: v.get('languageCode'))): + Ellipsis, 'translationLanguages', lambda _, v: v.get('languageCode'))): translation_language_code = translation_language['languageCode'] process_language( automatic_captions, base_url, translation_language_code, diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py index d9b33fa44..69c8f77ca 100644 --- a/youtube_dl/jsinterp.py +++ b/youtube_dl/jsinterp.py @@ -678,7 +678,7 @@ class JSInterpreter(object): return len(obj) try: return obj[int(idx)] if isinstance(obj, list) else obj[compat_str(idx)] - except (TypeError, KeyError, IndexError) as e: + except (TypeError, KeyError, IndexError, ValueError) as e: # allow_undefined is None gives correct behaviour if allow_undefined or ( allow_undefined is None and not isinstance(e, TypeError)): @@ -686,6 +686,8 @@ class JSInterpreter(object): raise self.Exception('Cannot get index {idx!r:.100}'.format(**locals()), expr=repr(obj), cause=e) def _dump(self, obj, namespace): + if obj is JS_Undefined: + return 'undefined' try: return json.dumps(obj) except TypeError: @@ -1038,6 +1040,10 @@ class JSInterpreter(object): left_val = self._index(left_val, idx) if isinstance(idx, float): idx = int(idx) + if isinstance(left_val, list) and len(left_val) <= int_or_none(idx, default=-1): + # JS Array is a sparsely assignable list + # TODO: handle extreme sparsity without memory bloat, eg using auxiliary dict + left_val.extend((idx - len(left_val) + 1) * [JS_Undefined]) left_val[idx] = self._operator( m.group('op'), self._index(left_val, idx) if m.group('op') else None, m.group('expr'), expr, local_vars, allow_recursion) @@ -1204,9 +1210,10 @@ class JSInterpreter(object): elif member == 'join': assertion(isinstance(obj, list), 'must be applied on a list') assertion(len(argvals) <= 1, 'takes at most one argument') - return (',' if len(argvals) == 0 else argvals[0]).join( - ('' if x in (None, JS_Undefined) else _js_toString(x)) - for x in obj) + return (',' if len(argvals) == 0 or argvals[0] in (None, JS_Undefined) + else argvals[0]).join( + ('' if x in (None, JS_Undefined) else _js_toString(x)) + for x in obj) elif member == 'reverse': assertion(not argvals, 'does not take any arguments') obj.reverse() @@ -1364,19 +1371,21 @@ class JSInterpreter(object): code, _ = self._separate_at_paren(func_m.group('code')) # refine the match return self.build_arglist(func_m.group('args')), code - def extract_function(self, funcname): + def extract_function(self, funcname, *global_stack): return function_with_repr( - self.extract_function_from_code(*self.extract_function_code(funcname)), + self.extract_function_from_code(*itertools.chain( + self.extract_function_code(funcname), global_stack)), 'F<%s>' % (funcname,)) def extract_function_from_code(self, argnames, code, *global_stack): local_vars = {} + start = None while True: - mobj = re.search(r'function\((?P[^)]*)\)\s*{', code) + mobj = re.search(r'function\((?P[^)]*)\)\s*{', code[start:]) if mobj is None: break - start, body_start = mobj.span() + start, body_start = ((start or 0) + x for x in mobj.span()) body, remaining = self._separate_at_paren(code[body_start - 1:]) name = self._named_object(local_vars, self.extract_function_from_code( [x.strip() for x in mobj.group('args').split(',')],