From 91b1569f68471d685382b738806b2e07d8f52707 Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 28 Feb 2025 00:02:10 +0000 Subject: [PATCH 01/12] [YouTube] Fix channel playlist extraction (#33074) * [YouTube] Extract playlist items from LOCKUP_VIEW_MODEL_... * resolves #33073 * thx seproDev (yt-dlp/yt-dlp#11615) Co-authored-by: sepro --- youtube_dl/extractor/youtube.py | 49 +++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index c93a2a1f9..cc84a193a 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -27,6 +27,7 @@ from ..compat import ( ) from ..jsinterp import JSInterpreter from ..utils import ( + bug_reports_message, clean_html, dict_get, error_to_compat_str, @@ -65,6 +66,7 @@ from ..utils import ( url_or_none, urlencode_postdata, urljoin, + variadic, ) @@ -460,6 +462,26 @@ class YoutubeBaseInfoExtractor(InfoExtractor): 'uploader': uploader, } + @staticmethod + def _extract_thumbnails(data, *path_list, **kw_final_key): + """ + Extract thumbnails from thumbnails dict + @param path_list: path list to level that contains 'thumbnails' key + """ + final_key = kw_final_key.get('final_key', 'thumbnails') + + return traverse_obj(data, (( + tuple(variadic(path) + (final_key, Ellipsis) + for path in path_list or [()])), { + 'url': ('url', T(url_or_none), + # Sometimes youtube gives a wrong thumbnail URL. See: + # https://github.com/yt-dlp/yt-dlp/issues/233 + # https://github.com/ytdl-org/youtube-dl/issues/28023 + T(lambda u: update_url(u, query=None) if u and 'maxresdefault' in u else u)), + 'height': ('height', T(int_or_none)), + 'width': ('width', T(int_or_none)), + }, T(lambda t: t if t.get('url') else None))) + def _search_results(self, query, params): data = { 'context': { @@ -3183,8 +3205,12 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): expected_type=txt_or_none) def _grid_entries(self, grid_renderer): - for item in grid_renderer['items']: - if not isinstance(item, dict): + for item in traverse_obj(grid_renderer, ('items', Ellipsis, T(dict))): + lockup_view_model = traverse_obj(item, ('lockupViewModel', T(dict))) + if lockup_view_model: + entry = self._extract_lockup_view_model(lockup_view_model) + if entry: + yield entry continue renderer = self._extract_grid_item_renderer(item) if not isinstance(renderer, dict): @@ -3268,6 +3294,25 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): continue yield self._extract_video(renderer) + def _extract_lockup_view_model(self, view_model): + content_id = view_model.get('contentId') + if not content_id: + return + content_type = view_model.get('contentType') + if content_type not in ('LOCKUP_CONTENT_TYPE_PLAYLIST', 'LOCKUP_CONTENT_TYPE_PODCAST'): + self.report_warning( + 'Unsupported lockup view model content type "{0}"{1}'.format(content_type, bug_reports_message()), only_once=True) + return + return merge_dicts(self.url_result( + update_url_query('https://www.youtube.com/playlist', {'list': content_id}), + ie=YoutubeTabIE, video_id=content_id), { + 'title': traverse_obj(view_model, ( + 'metadata', 'lockupMetadataViewModel', 'title', 'content', T(compat_str))), + 'thumbnails': self._extract_thumbnails(view_model, ( + 'contentImage', 'collectionThumbnailViewModel', 'primaryThumbnail', + 'thumbnailViewModel', 'image'), final_key='sources'), + }) + def _video_entry(self, video_renderer): video_id = video_renderer.get('videoId') if video_id: From 673277e510ebd996b62a2fcc76169bf3cce29910 Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 28 Feb 2025 01:02:20 +0000 Subject: [PATCH 02/12] [YouTube] Fix 91b1569 --- youtube_dl/extractor/youtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index cc84a193a..5f8c08201 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -3305,7 +3305,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): return return merge_dicts(self.url_result( update_url_query('https://www.youtube.com/playlist', {'list': content_id}), - ie=YoutubeTabIE, video_id=content_id), { + ie=YoutubeTabIE.ie_key(), video_id=content_id), { 'title': traverse_obj(view_model, ( 'metadata', 'lockupMetadataViewModel', 'title', 'content', T(compat_str))), 'thumbnails': self._extract_thumbnails(view_model, ( From cecaa18b80e33323193915ef9fbd2f68d94d7bce Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 7 Mar 2025 23:03:17 +0000 Subject: [PATCH 03/12] [compat] Clean-up * make workaround_optparse_bug9161 private * add comments * avoid leaving test objects behind --- youtube_dl/__init__.py | 4 +- youtube_dl/compat.py | 172 ++++++++++++++++++++++++++++++----------- 2 files changed, 129 insertions(+), 47 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 06bdfb689..3c1272e7b 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -18,7 +18,7 @@ from .compat import ( compat_getpass, compat_register_utf8, compat_shlex_split, - workaround_optparse_bug9161, + _workaround_optparse_bug9161, ) from .utils import ( _UnsafeExtensionError, @@ -50,7 +50,7 @@ def _real_main(argv=None): # Compatibility fix for Windows compat_register_utf8() - workaround_optparse_bug9161() + _workaround_optparse_bug9161() setproctitle('youtube-dl') diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index 26b655fb6..e617dd511 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -16,7 +16,6 @@ import os import platform import re import shlex -import shutil import socket import struct import subprocess @@ -24,11 +23,15 @@ import sys import types import xml.etree.ElementTree +_IDENTITY = lambda x: x + # naming convention # 'compat_' + Python3_name.replace('.', '_') # other aliases exist for convenience and/or legacy +# wrap disposable test values in type() to reclaim storage -# deal with critical unicode/str things first +# deal with critical unicode/str things first: +# compat_str, compat_basestring, compat_chr try: # Python 2 compat_str, compat_basestring, compat_chr = ( @@ -39,18 +42,23 @@ except NameError: str, (str, bytes), chr ) -# casefold + +# compat_casefold try: compat_str.casefold compat_casefold = lambda s: s.casefold() except AttributeError: from .casefold import _casefold as compat_casefold + +# compat_collections_abc try: import collections.abc as compat_collections_abc except ImportError: import collections as compat_collections_abc + +# compat_urllib_request try: import urllib.request as compat_urllib_request except ImportError: # Python 2 @@ -79,11 +87,15 @@ except TypeError: _add_init_method_arg(compat_urllib_request.Request) del _add_init_method_arg + +# compat_urllib_error try: import urllib.error as compat_urllib_error except ImportError: # Python 2 import urllib2 as compat_urllib_error + +# compat_urllib_parse try: import urllib.parse as compat_urllib_parse except ImportError: # Python 2 @@ -98,17 +110,23 @@ except ImportError: # Python 2 compat_urlparse = compat_urllib_parse compat_urllib_parse_urlparse = compat_urllib_parse.urlparse + +# compat_urllib_response try: import urllib.response as compat_urllib_response except ImportError: # Python 2 import urllib as compat_urllib_response + +# compat_urllib_response.addinfourl try: compat_urllib_response.addinfourl.status except AttributeError: # .getcode() is deprecated in Py 3. compat_urllib_response.addinfourl.status = property(lambda self: self.getcode()) + +# compat_http_cookiejar try: import http.cookiejar as compat_cookiejar except ImportError: # Python 2 @@ -127,12 +145,16 @@ else: compat_cookiejar_Cookie = compat_cookiejar.Cookie compat_http_cookiejar_Cookie = compat_cookiejar_Cookie + +# compat_http_cookies try: import http.cookies as compat_cookies except ImportError: # Python 2 import Cookie as compat_cookies compat_http_cookies = compat_cookies + +# compat_http_cookies_SimpleCookie if sys.version_info[0] == 2 or sys.version_info < (3, 3): class compat_cookies_SimpleCookie(compat_cookies.SimpleCookie): def load(self, rawdata): @@ -155,11 +177,15 @@ else: compat_cookies_SimpleCookie = compat_cookies.SimpleCookie compat_http_cookies_SimpleCookie = compat_cookies_SimpleCookie + +# compat_html_entities, probably useless now try: import html.entities as compat_html_entities except ImportError: # Python 2 import htmlentitydefs as compat_html_entities + +# compat_html_entities_html5 try: # Python >= 3.3 compat_html_entities_html5 = compat_html_entities.html5 except AttributeError: @@ -2408,18 +2434,24 @@ except AttributeError: # Py < 3.1 compat_http_client.HTTPResponse.getcode = lambda self: self.status + +# compat_urllib_HTTPError try: from urllib.error import HTTPError as compat_HTTPError except ImportError: # Python 2 from urllib2 import HTTPError as compat_HTTPError compat_urllib_HTTPError = compat_HTTPError + +# compat_urllib_request_urlretrieve try: from urllib.request import urlretrieve as compat_urlretrieve except ImportError: # Python 2 from urllib import urlretrieve as compat_urlretrieve compat_urllib_request_urlretrieve = compat_urlretrieve + +# compat_html_parser_HTMLParser, compat_html_parser_HTMLParseError try: from HTMLParser import ( HTMLParser as compat_HTMLParser, @@ -2432,22 +2464,33 @@ except ImportError: # Python 3 # HTMLParseError was deprecated in Python 3.3 and removed in # Python 3.5. Introducing dummy exception for Python >3.5 for compatible # and uniform cross-version exception handling + class compat_HTMLParseError(Exception): pass + compat_html_parser_HTMLParser = compat_HTMLParser compat_html_parser_HTMLParseError = compat_HTMLParseError + +# compat_subprocess_get_DEVNULL try: _DEVNULL = subprocess.DEVNULL compat_subprocess_get_DEVNULL = lambda: _DEVNULL except AttributeError: compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w') + +# compat_http_server try: import http.server as compat_http_server except ImportError: import BaseHTTPServer as compat_http_server + +# compat_urllib_parse_unquote_to_bytes, +# compat_urllib_parse_unquote, compat_urllib_parse_unquote_plus, +# compat_urllib_parse_urlencode, +# compat_urllib_parse_parse_qs try: from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes from urllib.parse import unquote as compat_urllib_parse_unquote @@ -2598,6 +2641,8 @@ except ImportError: # Python 2 compat_urllib_parse_parse_qs = compat_parse_qs + +# compat_urllib_request_DataHandler try: from urllib.request import DataHandler as compat_urllib_request_DataHandler except ImportError: # Python < 3.4 @@ -2632,16 +2677,20 @@ except ImportError: # Python < 3.4 return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url) + +# compat_xml_etree_ElementTree_ParseError try: from xml.etree.ElementTree import ParseError as compat_xml_parse_error except ImportError: # Python 2.6 from xml.parsers.expat import ExpatError as compat_xml_parse_error compat_xml_etree_ElementTree_ParseError = compat_xml_parse_error -etree = xml.etree.ElementTree + +# compat_xml_etree_ElementTree_Element +_etree = xml.etree.ElementTree -class _TreeBuilder(etree.TreeBuilder): +class _TreeBuilder(_etree.TreeBuilder): def doctype(self, name, pubid, system): pass @@ -2650,7 +2699,7 @@ try: # xml.etree.ElementTree.Element is a method in Python <=2.6 and # the following will crash with: # TypeError: isinstance() arg 2 must be a class, type, or tuple of classes and types - isinstance(None, etree.Element) + isinstance(None, _etree.Element) from xml.etree.ElementTree import Element as compat_etree_Element except TypeError: # Python <=2.6 from xml.etree.ElementTree import _ElementInterface as compat_etree_Element @@ -2658,12 +2707,12 @@ compat_xml_etree_ElementTree_Element = compat_etree_Element if sys.version_info[0] >= 3: def compat_etree_fromstring(text): - return etree.XML(text, parser=etree.XMLParser(target=_TreeBuilder())) + return _etree.XML(text, parser=_etree.XMLParser(target=_TreeBuilder())) else: # python 2.x tries to encode unicode strings with ascii (see the # XMLParser._fixtext method) try: - _etree_iter = etree.Element.iter + _etree_iter = _etree.Element.iter except AttributeError: # Python <=2.6 def _etree_iter(root): for el in root.findall('*'): @@ -2675,27 +2724,29 @@ else: # 2.7 source def _XML(text, parser=None): if not parser: - parser = etree.XMLParser(target=_TreeBuilder()) + parser = _etree.XMLParser(target=_TreeBuilder()) parser.feed(text) return parser.close() def _element_factory(*args, **kwargs): - el = etree.Element(*args, **kwargs) + el = _etree.Element(*args, **kwargs) for k, v in el.items(): if isinstance(v, bytes): el.set(k, v.decode('utf-8')) return el def compat_etree_fromstring(text): - doc = _XML(text, parser=etree.XMLParser(target=_TreeBuilder(element_factory=_element_factory))) + doc = _XML(text, parser=_etree.XMLParser(target=_TreeBuilder(element_factory=_element_factory))) for el in _etree_iter(doc): if el.text is not None and isinstance(el.text, bytes): el.text = el.text.decode('utf-8') return doc -if hasattr(etree, 'register_namespace'): - compat_etree_register_namespace = etree.register_namespace -else: + +# compat_xml_etree_register_namespace +try: + compat_etree_register_namespace = _etree.register_namespace +except AttributeError: def compat_etree_register_namespace(prefix, uri): """Register a namespace prefix. The registry is global, and any existing mapping for either the @@ -2704,14 +2755,16 @@ else: attributes in this namespace will be serialized with prefix if possible. ValueError is raised if prefix is reserved or is invalid. """ - if re.match(r"ns\d+$", prefix): - raise ValueError("Prefix format reserved for internal use") - for k, v in list(etree._namespace_map.items()): + if re.match(r'ns\d+$', prefix): + raise ValueError('Prefix format reserved for internal use') + for k, v in list(_etree._namespace_map.items()): if k == uri or v == prefix: - del etree._namespace_map[k] - etree._namespace_map[uri] = prefix + del _etree._namespace_map[k] + _etree._namespace_map[uri] = prefix compat_xml_etree_register_namespace = compat_etree_register_namespace + +# compat_xpath, compat_etree_iterfind if sys.version_info < (2, 7): # Here comes the crazy part: In 2.6, if the xpath is a unicode, # .//node does not match if a node is a direct child of . ! @@ -2898,7 +2951,6 @@ if sys.version_info < (2, 7): def __init__(self, root): self.root = root - ## # Generate all matching objects. def compat_etree_iterfind(elem, path, namespaces=None): @@ -2933,13 +2985,15 @@ if sys.version_info < (2, 7): else: - compat_xpath = lambda xpath: xpath compat_etree_iterfind = lambda element, match: element.iterfind(match) + compat_xpath = _IDENTITY +# compat_os_name compat_os_name = os._name if os.name == 'java' else os.name +# compat_shlex_quote if compat_os_name == 'nt': def compat_shlex_quote(s): return s if re.match(r'^[-_\w./]+$', s) else '"%s"' % s.replace('"', '\\"') @@ -2954,6 +3008,7 @@ else: return "'" + s.replace("'", "'\"'\"'") + "'" +# compat_shlex.split try: args = shlex.split('中文') assert (isinstance(args, list) @@ -2969,6 +3024,7 @@ except (AssertionError, UnicodeEncodeError): return list(map(lambda s: s.decode('utf-8'), shlex.split(s, comments, posix))) +# compat_ord def compat_ord(c): if isinstance(c, int): return c @@ -2976,6 +3032,7 @@ def compat_ord(c): return ord(c) +# compat_getenv, compat_os_path_expanduser, compat_setenv if sys.version_info >= (3, 0): compat_getenv = os.getenv compat_expanduser = os.path.expanduser @@ -3063,6 +3120,7 @@ else: compat_os_path_expanduser = compat_expanduser +# compat_os_path_realpath if compat_os_name == 'nt' and sys.version_info < (3, 8): # os.path.realpath on Windows does not follow symbolic links # prior to Python 3.8 (see https://bugs.python.org/issue9949) @@ -3076,6 +3134,7 @@ else: compat_os_path_realpath = compat_realpath +# compat_print if sys.version_info < (3, 0): def compat_print(s): from .utils import preferredencoding @@ -3086,6 +3145,7 @@ else: print(s) +# compat_getpass_getpass if sys.version_info < (3, 0) and sys.platform == 'win32': def compat_getpass(prompt, *args, **kwargs): if isinstance(prompt, compat_str): @@ -3098,22 +3158,22 @@ else: compat_getpass_getpass = compat_getpass +# compat_input try: compat_input = raw_input except NameError: # Python 3 compat_input = input +# compat_kwargs # Python < 2.6.5 require kwargs to be bytes try: - def _testfunc(x): - pass - _testfunc(**{'x': 0}) + (lambda x: x)(**{'x': 0}) except TypeError: def compat_kwargs(kwargs): return dict((bytes(k), v) for k, v in kwargs.items()) else: - compat_kwargs = lambda kwargs: kwargs + compat_kwargs = _IDENTITY # compat_numeric_types @@ -3132,6 +3192,8 @@ except NameError: # Python 3 # compat_int compat_int = compat_integer_types[-1] + +# compat_socket_create_connection if sys.version_info < (2, 7): def compat_socket_create_connection(address, timeout, source_address=None): host, port = address @@ -3158,6 +3220,7 @@ else: compat_socket_create_connection = socket.create_connection +# compat_contextlib_suppress try: from contextlib import suppress as compat_contextlib_suppress except ImportError: @@ -3200,12 +3263,12 @@ except AttributeError: # repeated .close() is OK, but just in case with compat_contextlib_suppress(EnvironmentError): f.close() - popen.wait() + popen.wait() # Fix https://github.com/ytdl-org/youtube-dl/issues/4223 # See http://bugs.python.org/issue9161 for what is broken -def workaround_optparse_bug9161(): +def _workaround_optparse_bug9161(): op = optparse.OptionParser() og = optparse.OptionGroup(op, 'foo') try: @@ -3224,9 +3287,10 @@ def workaround_optparse_bug9161(): optparse.OptionGroup.add_option = _compat_add_option -if hasattr(shutil, 'get_terminal_size'): # Python >= 3.3 - compat_get_terminal_size = shutil.get_terminal_size -else: +# compat_shutil_get_terminal_size +try: + from shutil import get_terminal_size as compat_get_terminal_size # Python >= 3.3 +except ImportError: _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines']) def compat_get_terminal_size(fallback=(80, 24)): @@ -3256,27 +3320,33 @@ else: columns = _columns if lines is None or lines <= 0: lines = _lines + return _terminal_size(columns, lines) +compat_shutil_get_terminal_size = compat_get_terminal_size + +# compat_itertools_count try: - itertools.count(start=0, step=1) + type(itertools.count(start=0, step=1)) compat_itertools_count = itertools.count -except TypeError: # Python 2.6 +except TypeError: # Python 2.6 lacks step def compat_itertools_count(start=0, step=1): while True: yield start start += step +# compat_tokenize_tokenize if sys.version_info >= (3, 0): from tokenize import tokenize as compat_tokenize_tokenize else: from tokenize import generate_tokens as compat_tokenize_tokenize +# compat_struct_pack, compat_struct_unpack, compat_Struct try: - struct.pack('!I', 0) + type(struct.pack('!I', 0)) except TypeError: # In Python 2.6 and 2.7.x < 2.7.7, struct requires a bytes argument # See https://bugs.python.org/issue19099 @@ -3308,8 +3378,10 @@ else: compat_Struct = struct.Struct -# compat_map/filter() returning an iterator, supposedly the -# same versioning as for zip below +# builtins returning an iterator + +# compat_map, compat_filter +# supposedly the same versioning as for zip below try: from future_builtins import map as compat_map except ImportError: @@ -3326,6 +3398,7 @@ except ImportError: except ImportError: compat_filter = filter +# compat_zip try: from future_builtins import zip as compat_zip except ImportError: # not 2.6+ or is 3.x @@ -3335,6 +3408,7 @@ except ImportError: # not 2.6+ or is 3.x compat_zip = zip +# compat_itertools_zip_longest # method renamed between Py2/3 try: from itertools import zip_longest as compat_itertools_zip_longest @@ -3342,7 +3416,8 @@ except ImportError: from itertools import izip_longest as compat_itertools_zip_longest -# new class in collections +# compat_collections_chain_map +# collections.ChainMap: new class try: from collections import ChainMap as compat_collections_chain_map # Py3.3's ChainMap is deficient @@ -3405,12 +3480,14 @@ except ImportError: return compat_collections_chain_map(*(self.maps[1:])) +# compat_re_Pattern, compat_re_Match # Pythons disagree on the type of a pattern (RegexObject, _sre.SRE_Pattern, Pattern, ...?) compat_re_Pattern = type(re.compile('')) # and on the type of a match compat_re_Match = type(re.match('a', 'a')) +# compat_base64_b64decode if sys.version_info < (3, 3): def compat_b64decode(s, *args, **kwargs): if isinstance(s, compat_str): @@ -3422,6 +3499,7 @@ else: compat_base64_b64decode = compat_b64decode +# compat_ctypes_WINFUNCTYPE if platform.python_implementation() == 'PyPy' and sys.pypy_version_info < (5, 4, 0): # PyPy2 prior to version 5.4.0 expects byte strings as Windows function # names, see the original PyPy issue [1] and the youtube-dl one [2]. @@ -3440,6 +3518,7 @@ else: return ctypes.WINFUNCTYPE(*args, **kwargs) +# compat_open if sys.version_info < (3, 0): # open(file, mode='r', buffering=- 1, encoding=None, errors=None, newline=None, closefd=True) not: opener=None def compat_open(file_, *args, **kwargs): @@ -3467,12 +3546,15 @@ except AttributeError: def compat_datetime_timedelta_total_seconds(td): return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6 + # optional decompression packages +# compat_brotli # PyPi brotli package implements 'br' Content-Encoding try: import brotli as compat_brotli except ImportError: compat_brotli = None +# compat_ncompress # PyPi ncompress package implements 'compress' Content-Encoding try: import ncompress as compat_ncompress @@ -3495,6 +3577,7 @@ legacy = [ 'compat_getpass', 'compat_parse_qs', 'compat_realpath', + 'compat_shlex_split', 'compat_urllib_parse_parse_qs', 'compat_urllib_parse_unquote', 'compat_urllib_parse_unquote_plus', @@ -3508,8 +3591,6 @@ legacy = [ __all__ = [ - 'compat_html_parser_HTMLParseError', - 'compat_html_parser_HTMLParser', 'compat_Struct', 'compat_base64_b64decode', 'compat_basestring', @@ -3518,13 +3599,9 @@ __all__ = [ 'compat_chr', 'compat_collections_abc', 'compat_collections_chain_map', - 'compat_datetime_timedelta_total_seconds', - 'compat_http_cookiejar', - 'compat_http_cookiejar_Cookie', - 'compat_http_cookies', - 'compat_http_cookies_SimpleCookie', 'compat_contextlib_suppress', 'compat_ctypes_WINFUNCTYPE', + 'compat_datetime_timedelta_total_seconds', 'compat_etree_fromstring', 'compat_etree_iterfind', 'compat_filter', @@ -3533,6 +3610,12 @@ __all__ = [ 'compat_getpass_getpass', 'compat_html_entities', 'compat_html_entities_html5', + 'compat_html_parser_HTMLParseError', + 'compat_html_parser_HTMLParser', + 'compat_http_cookiejar', + 'compat_http_cookiejar_Cookie', + 'compat_http_cookies', + 'compat_http_cookies_SimpleCookie', 'compat_http_client', 'compat_http_server', 'compat_input', @@ -3555,7 +3638,7 @@ __all__ = [ 'compat_register_utf8', 'compat_setenv', 'compat_shlex_quote', - 'compat_shlex_split', + 'compat_shutil_get_terminal_size', 'compat_socket_create_connection', 'compat_str', 'compat_struct_pack', @@ -3575,5 +3658,4 @@ __all__ = [ 'compat_xml_etree_register_namespace', 'compat_xpath', 'compat_zip', - 'workaround_optparse_bug9161', ] From 8738407d77f6da843f8f5ded1ccad73172b4abac Mon Sep 17 00:00:00 2001 From: dirkf Date: Sun, 2 Mar 2025 13:36:05 +0000 Subject: [PATCH 04/12] [compat] Support zstd Content-Encoding * see RFC 8878 7.2 --- youtube_dl/compat.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index e617dd511..6cd7abd24 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -3561,6 +3561,13 @@ try: except ImportError: compat_ncompress = None +# compat_zstandard +# PyPi zstandard package implements 'zstd' Content-Encoding (RFC 8878 7.2) +try: + import zstandard as compat_zstandard +except ImportError: + compat_zstandard = None + legacy = [ 'compat_HTMLParseError', @@ -3658,4 +3665,5 @@ __all__ = [ 'compat_xml_etree_register_namespace', 'compat_xpath', 'compat_zip', + 'compat_zstandard', ] From 974c7d7f349831cf32026ec57e75bc821843a07b Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 7 Mar 2025 19:17:18 +0000 Subject: [PATCH 05/12] [compat] Fix inheriting from compat_collections_chain_map * see ytdl-org/youtube-dl#33079#issuecomment-2704038049 --- youtube_dl/compat.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index 6cd7abd24..8910a4dac 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -3473,11 +3473,12 @@ except ImportError: def new_child(self, m=None, **kwargs): m = m or {} m.update(kwargs) - return compat_collections_chain_map(m, *self.maps) + # support inheritance ! + return type(self)(m, *self.maps) @property def parents(self): - return compat_collections_chain_map(*(self.maps[1:])) + return type(self)(*(self.maps[1:])) # compat_re_Pattern, compat_re_Match From 94849bc997d232b344b0f3666198feec7b004b43 Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 7 Mar 2025 19:32:56 +0000 Subject: [PATCH 06/12] [JSInterp] Improve Date processing * add JS_Date class implementing JS Date * support constructor args other than date string * support static methods of Date * Date objects are still automatically coerced to timestamp before using in JS. --- test/test_jsinterp.py | 22 ++++++++++++ youtube_dl/jsinterp.py | 76 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py index 6c34bc896..4c5256c4b 100644 --- a/test/test_jsinterp.py +++ b/test/test_jsinterp.py @@ -11,6 +11,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import math import re +import time from youtube_dl.compat import compat_str as str from youtube_dl.jsinterp import JS_Undefined, JSInterpreter @@ -208,6 +209,27 @@ class TestJSInterpreter(unittest.TestCase): self._test(jsi, 86000, args=['12/31/1969 18:01:26 MDT']) # epoch 0 self._test(jsi, 0, args=['1 January 1970 00:00:00 UTC']) + # undefined + self._test(jsi, NaN, args=[JS_Undefined]) + # y,m,d, ... - may fail with older dates lacking DST data + jsi = JSInterpreter('function f() { return new Date(%s); }' + % ('2024, 5, 29, 2, 52, 12, 42',)) + self._test(jsi, 1719625932042) + # no arg + self.assertAlmostEqual(JSInterpreter( + 'function f() { return new Date() - 0; }').call_function('f'), + time.time() * 1000, delta=100) + # Date.now() + self.assertAlmostEqual(JSInterpreter( + 'function f() { return Date.now(); }').call_function('f'), + time.time() * 1000, delta=100) + # Date.parse() + jsi = JSInterpreter('function f(dt) { return Date.parse(dt); }') + self._test(jsi, 0, args=['1 January 1970 00:00:00 UTC']) + # Date.UTC() + jsi = JSInterpreter('function f() { return Date.UTC(%s); }' + % ('1970, 0, 1, 0, 0, 0, 0',)) + self._test(jsi, 0) def test_call(self): jsi = JSInterpreter(''' diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py index 2859bc734..c3ee3bb03 100644 --- a/youtube_dl/jsinterp.py +++ b/youtube_dl/jsinterp.py @@ -1,10 +1,12 @@ # coding: utf-8 from __future__ import unicode_literals +import calendar import itertools import json import operator import re +import time from functools import update_wrapper, wraps @@ -12,8 +14,10 @@ from .utils import ( error_to_compat_str, ExtractorError, float_or_none, + int_or_none, js_to_json, remove_quotes, + str_or_none, unified_timestamp, variadic, write_string, @@ -475,6 +479,73 @@ class JSInterpreter(object): flags |= cls.RE_FLAGS[ch] return flags, expr[idx + 1:] + class JS_Date(object): + _t = None + + @staticmethod + def __ymd_etc(*args, **kw_is_utc): + # args: year, monthIndex, day, hours, minutes, seconds, milliseconds + is_utc = kw_is_utc.get('is_utc', False) + + args = list(args[:7]) + args += [0] * (9 - len(args)) + args[1] += 1 # month 0..11 -> 1..12 + ms = args[6] + for i in range(6, 9): + args[i] = -1 # don't know + if is_utc: + args[-1] = 1 + # TODO: [MDN] When a segment overflows or underflows its expected + # range, it usually "carries over to" or "borrows from" the higher segment. + try: + mktime = calendar.timegm if is_utc else time.mktime + return mktime(time.struct_time(args)) * 1000 + ms + except (OverflowError, ValueError): + return None + + @classmethod + def UTC(cls, *args): + t = cls.__ymd_etc(*args, is_utc=True) + return _NaN if t is None else t + + @staticmethod + def parse(date_str, **kw_is_raw): + is_raw = kw_is_raw.get('is_raw', False) + + t = unified_timestamp(str_or_none(date_str), False) + return int(t * 1000) if t is not None else t if is_raw else _NaN + + @staticmethod + def now(**kw_is_raw): + is_raw = kw_is_raw.get('is_raw', False) + + t = time.time() + return int(t * 1000) if t is not None else t if is_raw else _NaN + + def __init__(self, *args): + if not args: + args = [self.now(is_raw=True)] + if len(args) == 1: + if isinstance(args[0], JSInterpreter.JS_Date): + self._t = int_or_none(args[0].valueOf(), default=None) + else: + arg_type = _js_typeof(args[0]) + if arg_type == 'string': + self._t = self.parse(args[0], is_raw=True) + elif arg_type == 'number': + self._t = int(args[0]) + else: + self._t = self.__ymd_etc(*args) + + def toString(self): + try: + return time.strftime('%a %b %0d %Y %H:%M:%S %Z%z', self._t).rstrip() + except TypeError: + return "Invalid Date" + + def valueOf(self): + return _NaN if self._t is None else self._t + @classmethod def __op_chars(cls): op_chars = set(';,[') @@ -715,7 +786,7 @@ class JSInterpreter(object): new_kw, _, obj = expr.partition('new ') if not new_kw: - for klass, konstr in (('Date', lambda x: int(unified_timestamp(x, False) * 1000)), + for klass, konstr in (('Date', lambda *x: self.JS_Date(*x).valueOf()), ('RegExp', self.JS_RegExp), ('Error', self.Exception)): if not obj.startswith(klass + '('): @@ -1034,6 +1105,7 @@ class JSInterpreter(object): 'String': compat_str, 'Math': float, 'Array': list, + 'Date': self.JS_Date, } obj = local_vars.get(variable) if obj in (JS_Undefined, None): @@ -1086,6 +1158,8 @@ class JSInterpreter(object): assertion(len(argvals) == 2, 'takes two arguments') return argvals[0] ** argvals[1] raise self.Exception('Unsupported Math method ' + member, expr=expr) + elif obj is self.JS_Date: + return getattr(obj, member)(*argvals) if member == 'split': assertion(len(argvals) <= 2, 'takes at most two arguments') From af049e309bfa47141a9788cd1730dd50dad6176d Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 7 Mar 2025 19:37:56 +0000 Subject: [PATCH 07/12] [JSInterp] Handle undefined, etc, passed to JS_RegExp and Exception --- youtube_dl/jsinterp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py index c3ee3bb03..9b4157a43 100644 --- a/youtube_dl/jsinterp.py +++ b/youtube_dl/jsinterp.py @@ -408,6 +408,7 @@ class JSInterpreter(object): class Exception(ExtractorError): def __init__(self, msg, *args, **kwargs): expr = kwargs.pop('expr', None) + msg = str_or_none(msg, default='"None"') if expr is not None: msg = '{0} in: {1!r:.100}'.format(msg.rstrip(), expr) super(JSInterpreter.Exception, self).__init__(msg, *args, **kwargs) @@ -435,6 +436,7 @@ class JSInterpreter(object): flags, _ = self.regex_flags(flags) # First, avoid https://github.com/python/cpython/issues/74534 self.__self = None + pattern_txt = str_or_none(pattern_txt) or '(?:)' self.__pattern_txt = pattern_txt.replace('[[', r'[\[') self.__flags = flags From 1dc27e1c3bda9cb8f44b805c89918aa7d11ffcdc Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 7 Mar 2025 19:40:53 +0000 Subject: [PATCH 08/12] [JSInterp] Make indexing error handling more conformant * by default TypeError -> undefined, else raise * set allow_undefined=True/False to override --- youtube_dl/jsinterp.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py index 9b4157a43..5a45fbb03 100644 --- a/youtube_dl/jsinterp.py +++ b/youtube_dl/jsinterp.py @@ -672,14 +672,15 @@ class JSInterpreter(object): except Exception as e: raise self.Exception('Failed to evaluate {left_val!r:.50} {op} {right_val!r:.50}'.format(**locals()), expr, cause=e) - def _index(self, obj, idx, allow_undefined=True): + def _index(self, obj, idx, allow_undefined=None): if idx == 'length' and isinstance(obj, list): return len(obj) try: return obj[int(idx)] if isinstance(obj, list) else obj[compat_str(idx)] except (TypeError, KeyError, IndexError) as e: - if allow_undefined: - # when is not allowed? + # allow_undefined is None gives correct behaviour + if allow_undefined or ( + allow_undefined is None and not isinstance(e, TypeError)): return JS_Undefined raise self.Exception('Cannot get index {idx!r:.100}'.format(**locals()), expr=repr(obj), cause=e) From 422b1b31cf398d60b4606fa57be8e39c1181932f Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 7 Mar 2025 20:00:58 +0000 Subject: [PATCH 09/12] [YouTube] Temporarily redirect from tce-style player JS --- youtube_dl/extractor/youtube.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 5f8c08201..9e200105e 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1607,9 +1607,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor): webpage or '', 'player URL', fatal=False) if player_url: ytcfgs = ytcfgs + ({'PLAYER_JS_URL': player_url},) - return traverse_obj( + player_url = traverse_obj( ytcfgs, (Ellipsis, 'PLAYER_JS_URL'), (Ellipsis, 'WEB_PLAYER_CONTEXT_CONFIGS', Ellipsis, 'jsUrl'), get_all=False, expected_type=lambda u: urljoin('https://www.youtube.com', u)) + nplayer_url, is_tce = re.subn(r'(?<=/player_ias)_tce(?=\.vflset/)', '', player_url or '') + if is_tce: + # TODO: Add proper support for the 'tce' variant players + # See https://github.com/yt-dlp/yt-dlp/issues/12398 + self.write_debug('Modifying tce player URL: {0}'.format(player_url)) + return nplayer_url + return player_url def _download_player_url(self, video_id, fatal=False): res = self._download_webpage( From 283dca56feb9f23978733810ab155472d6473c38 Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 7 Mar 2025 21:02:28 +0000 Subject: [PATCH 10/12] [YouTube] Initially support tce-style player JS * resolves #33079 --- test/test_youtube_signature.py | 21 +++++++++++++++++---- youtube_dl/extractor/youtube.py | 23 +++++++++++++---------- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py index 67ef75fde..166614e62 100644 --- a/test/test_youtube_signature.py +++ b/test/test_youtube_signature.py @@ -223,6 +223,18 @@ _NSIG_TESTS = [ 'https://www.youtube.com/s/player/9c6dfc4a/player_ias.vflset/en_US/base.js', 'jbu7ylIosQHyJyJV', 'uwI0ESiynAmhNg', ), + ( + 'https://www.youtube.com/s/player/f6e09c70/player_ias.vflset/en_US/base.js', + 'W9HJZKktxuYoDTqW', 'jHbbkcaxm54', + ), + ( + 'https://www.youtube.com/s/player/f6e09c70/player_ias_tce.vflset/en_US/base.js', + 'W9HJZKktxuYoDTqW', 'jHbbkcaxm54', + ), + ( + 'https://www.youtube.com/s/player/91201489/player_ias_tce.vflset/en_US/base.js', + 'W9HJZKktxuYoDTqW', 'U48vOZHaeYS6vO', + ), ] @@ -284,7 +296,7 @@ def t_factory(name, sig_func, url_pattern): def signature(jscode, sig_input): - func = YoutubeIE(FakeYDL())._parse_sig_js(jscode) + func = YoutubeIE(FakeYDL({'cachedir': False}))._parse_sig_js(jscode) src_sig = ( compat_str(string.printable[:sig_input]) if isinstance(sig_input, int) else sig_input) @@ -292,9 +304,10 @@ def signature(jscode, sig_input): def n_sig(jscode, sig_input): - funcname = YoutubeIE(FakeYDL())._extract_n_function_name(jscode) - return JSInterpreter(jscode).call_function( - funcname, sig_input, _ytdl_do_not_return=sig_input) + ie = YoutubeIE(FakeYDL({'cachedir': False})) + jsi = JSInterpreter(jscode) + jsi, _, func_code = ie._extract_n_function_code_jsi(sig_input, jsi) + return ie._extract_n_function_from_code(jsi, func_code)(sig_input) make_sig_test = t_factory( diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 9e200105e..11bed6cae 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1607,16 +1607,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor): webpage or '', 'player URL', fatal=False) if player_url: ytcfgs = ytcfgs + ({'PLAYER_JS_URL': player_url},) - player_url = traverse_obj( + return traverse_obj( ytcfgs, (Ellipsis, 'PLAYER_JS_URL'), (Ellipsis, 'WEB_PLAYER_CONTEXT_CONFIGS', Ellipsis, 'jsUrl'), get_all=False, expected_type=lambda u: urljoin('https://www.youtube.com', u)) - nplayer_url, is_tce = re.subn(r'(?<=/player_ias)_tce(?=\.vflset/)', '', player_url or '') - if is_tce: - # TODO: Add proper support for the 'tce' variant players - # See https://github.com/yt-dlp/yt-dlp/issues/12398 - self.write_debug('Modifying tce player URL: {0}'.format(player_url)) - return nplayer_url - return player_url def _download_player_url(self, video_id, fatal=False): res = self._download_webpage( @@ -1858,12 +1851,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor): if func_code: return jsi, player_id, func_code + return self._extract_n_function_code_jsi(video_id, jsi, player_id) - func_name = self._extract_n_function_name(jscode) + def _extract_n_function_code_jsi(self, video_id, jsi, player_id=None): + + var_ay = self._search_regex( + r'(?:[;\s]|^)\s*(var\s*[\w$]+\s*=\s*"[^"]+"\s*\.\s*split\("\{"\))(?=\s*[,;])', + jsi.code, 'useful values', default='') + + func_name = self._extract_n_function_name(jsi.code) func_code = jsi.extract_function_code(func_name) + if var_ay: + func_code = (func_code[0], ';\n'.join((var_ay, func_code[1]))) - self.cache.store('youtube-nsig', player_id, func_code) + if player_id: + self.cache.store('youtube-nsig', player_id, func_code) return jsi, player_id, func_code def _extract_n_function_from_code(self, jsi, func_code): From 32f89de92b652bf246aa458a552c9bb397abef77 Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 7 Mar 2025 21:03:54 +0000 Subject: [PATCH 11/12] [YouTube] Update TVHTML5 client parameters * resolves #33078 --- youtube_dl/extractor/youtube.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 11bed6cae..6364e3aee 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -122,7 +122,8 @@ class YoutubeBaseInfoExtractor(InfoExtractor): 'INNERTUBE_CONTEXT': { 'client': { 'clientName': 'TVHTML5', - 'clientVersion': '7.20241201.18.00', + 'clientVersion': '7.20250120.19.00', + 'userAgent': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/Version', }, }, 'INNERTUBE_CONTEXT_CLIENT_NAME': 7, From 420d53387cff54ea1fccca061438d59bdb50a39c Mon Sep 17 00:00:00 2001 From: dirkf Date: Mon, 10 Mar 2025 11:44:06 +0000 Subject: [PATCH 12/12] [JSInterp] Improve tests * from yt-dlp/yt-dlp#12313 * also fix d7c2708 --- test/test_jsinterp.py | 21 ++++++++++++++++++--- youtube_dl/jsinterp.py | 3 ++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py index 4c5256c4b..3c9650ab6 100644 --- a/test/test_jsinterp.py +++ b/test/test_jsinterp.py @@ -212,9 +212,16 @@ class TestJSInterpreter(unittest.TestCase): # undefined self._test(jsi, NaN, args=[JS_Undefined]) # y,m,d, ... - may fail with older dates lacking DST data - jsi = JSInterpreter('function f() { return new Date(%s); }' - % ('2024, 5, 29, 2, 52, 12, 42',)) - self._test(jsi, 1719625932042) + jsi = JSInterpreter( + 'function f() { return new Date(%s); }' + % ('2024, 5, 29, 2, 52, 12, 42',)) + self._test(jsi, ( + 1719625932042 # UK value + + ( + + 3600 # back to GMT + + (time.altzone if time.daylight # host's DST + else time.timezone) + ) * 1000)) # no arg self.assertAlmostEqual(JSInterpreter( 'function f() { return new Date() - 0; }').call_function('f'), @@ -485,6 +492,14 @@ class TestJSInterpreter(unittest.TestCase): self._test('function f(){return NaN << 42}', 0) self._test('function f(){return "21.9" << 1}', 42) self._test('function f(){return 21 << 4294967297}', 42) + self._test('function f(){return true << "5";}', 32) + self._test('function f(){return true << true;}', 2) + self._test('function f(){return "19" & "21.9";}', 17) + self._test('function f(){return "19" & false;}', 0) + self._test('function f(){return "11.0" >> "2.1";}', 2) + self._test('function f(){return 5 ^ 9;}', 12) + self._test('function f(){return 0.0 << NaN}', 0) + self._test('function f(){return null << undefined}', 0) def test_negative(self): self._test('function f(){return 2 * -2.0 ;}', -4) diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py index 5a45fbb03..d9b33fa44 100644 --- a/youtube_dl/jsinterp.py +++ b/youtube_dl/jsinterp.py @@ -154,6 +154,7 @@ def _js_to_primitive(v): ) +# more exact: yt-dlp/yt-dlp#12110 def _js_toString(v): return ( 'undefined' if v is JS_Undefined @@ -162,7 +163,7 @@ def _js_toString(v): else 'null' if v is None # bool <= int: do this first else ('false', 'true')[v] if isinstance(v, bool) - else '{0:.7f}'.format(v).rstrip('.0') if isinstance(v, compat_numeric_types) + else re.sub(r'(?<=\d)\.?0*$', '', '{0:.7f}'.format(v)) if isinstance(v, compat_numeric_types) else _js_to_primitive(v))