mirror of
https://github.com/ytdl-org/youtube-dl
synced 2025-05-10 15:54:49 +09:00
Merge remote-tracking branch 'ytdl-org/master' into fix-npo-support
This commit is contained in:
Bart Broere
commit
4f8b323d1b
@ -11,6 +11,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import math
|
||||
import re
|
||||
import time
|
||||
|
||||
from youtube_dl.compat import compat_str as str
|
||||
from youtube_dl.jsinterp import JS_Undefined, JSInterpreter
|
||||
@ -208,6 +209,34 @@ class TestJSInterpreter(unittest.TestCase):
|
||||
self._test(jsi, 86000, args=['12/31/1969 18:01:26 MDT'])
|
||||
# epoch 0
|
||||
self._test(jsi, 0, args=['1 January 1970 00:00:00 UTC'])
|
||||
# undefined
|
||||
self._test(jsi, NaN, args=[JS_Undefined])
|
||||
# y,m,d, ... - may fail with older dates lacking DST data
|
||||
jsi = JSInterpreter(
|
||||
'function f() { return new Date(%s); }'
|
||||
% ('2024, 5, 29, 2, 52, 12, 42',))
|
||||
self._test(jsi, (
|
||||
1719625932042 # UK value
|
||||
+ (
|
||||
+ 3600 # back to GMT
|
||||
+ (time.altzone if time.daylight # host's DST
|
||||
else time.timezone)
|
||||
) * 1000))
|
||||
# no arg
|
||||
self.assertAlmostEqual(JSInterpreter(
|
||||
'function f() { return new Date() - 0; }').call_function('f'),
|
||||
time.time() * 1000, delta=100)
|
||||
# Date.now()
|
||||
self.assertAlmostEqual(JSInterpreter(
|
||||
'function f() { return Date.now(); }').call_function('f'),
|
||||
time.time() * 1000, delta=100)
|
||||
# Date.parse()
|
||||
jsi = JSInterpreter('function f(dt) { return Date.parse(dt); }')
|
||||
self._test(jsi, 0, args=['1 January 1970 00:00:00 UTC'])
|
||||
# Date.UTC()
|
||||
jsi = JSInterpreter('function f() { return Date.UTC(%s); }'
|
||||
% ('1970, 0, 1, 0, 0, 0, 0',))
|
||||
self._test(jsi, 0)
|
||||
|
||||
def test_call(self):
|
||||
jsi = JSInterpreter('''
|
||||
@ -463,6 +492,14 @@ class TestJSInterpreter(unittest.TestCase):
|
||||
self._test('function f(){return NaN << 42}', 0)
|
||||
self._test('function f(){return "21.9" << 1}', 42)
|
||||
self._test('function f(){return 21 << 4294967297}', 42)
|
||||
self._test('function f(){return true << "5";}', 32)
|
||||
self._test('function f(){return true << true;}', 2)
|
||||
self._test('function f(){return "19" & "21.9";}', 17)
|
||||
self._test('function f(){return "19" & false;}', 0)
|
||||
self._test('function f(){return "11.0" >> "2.1";}', 2)
|
||||
self._test('function f(){return 5 ^ 9;}', 12)
|
||||
self._test('function f(){return 0.0 << NaN}', 0)
|
||||
self._test('function f(){return null << undefined}', 0)
|
||||
|
||||
def test_negative(self):
|
||||
self._test('function f(){return 2 * -2.0 ;}', -4)
|
||||
|
||||
@ -223,6 +223,18 @@ _NSIG_TESTS = [
|
||||
'https://www.youtube.com/s/player/9c6dfc4a/player_ias.vflset/en_US/base.js',
|
||||
'jbu7ylIosQHyJyJV', 'uwI0ESiynAmhNg',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/f6e09c70/player_ias.vflset/en_US/base.js',
|
||||
'W9HJZKktxuYoDTqW', 'jHbbkcaxm54',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/f6e09c70/player_ias_tce.vflset/en_US/base.js',
|
||||
'W9HJZKktxuYoDTqW', 'jHbbkcaxm54',
|
||||
),
|
||||
(
|
||||
'https://www.youtube.com/s/player/91201489/player_ias_tce.vflset/en_US/base.js',
|
||||
'W9HJZKktxuYoDTqW', 'U48vOZHaeYS6vO',
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@ -284,7 +296,7 @@ def t_factory(name, sig_func, url_pattern):
|
||||
|
||||
|
||||
def signature(jscode, sig_input):
|
||||
func = YoutubeIE(FakeYDL())._parse_sig_js(jscode)
|
||||
func = YoutubeIE(FakeYDL({'cachedir': False}))._parse_sig_js(jscode)
|
||||
src_sig = (
|
||||
compat_str(string.printable[:sig_input])
|
||||
if isinstance(sig_input, int) else sig_input)
|
||||
@ -292,9 +304,10 @@ def signature(jscode, sig_input):
|
||||
|
||||
|
||||
def n_sig(jscode, sig_input):
|
||||
funcname = YoutubeIE(FakeYDL())._extract_n_function_name(jscode)
|
||||
return JSInterpreter(jscode).call_function(
|
||||
funcname, sig_input, _ytdl_do_not_return=sig_input)
|
||||
ie = YoutubeIE(FakeYDL({'cachedir': False}))
|
||||
jsi = JSInterpreter(jscode)
|
||||
jsi, _, func_code = ie._extract_n_function_code_jsi(sig_input, jsi)
|
||||
return ie._extract_n_function_from_code(jsi, func_code)(sig_input)
|
||||
|
||||
|
||||
make_sig_test = t_factory(
|
||||
|
||||
@ -18,7 +18,7 @@ from .compat import (
|
||||
compat_getpass,
|
||||
compat_register_utf8,
|
||||
compat_shlex_split,
|
||||
workaround_optparse_bug9161,
|
||||
_workaround_optparse_bug9161,
|
||||
)
|
||||
from .utils import (
|
||||
_UnsafeExtensionError,
|
||||
@ -50,7 +50,7 @@ def _real_main(argv=None):
|
||||
# Compatibility fix for Windows
|
||||
compat_register_utf8()
|
||||
|
||||
workaround_optparse_bug9161()
|
||||
_workaround_optparse_bug9161()
|
||||
|
||||
setproctitle('youtube-dl')
|
||||
|
||||
|
||||
@ -16,7 +16,6 @@ import os
|
||||
import platform
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
import socket
|
||||
import struct
|
||||
import subprocess
|
||||
@ -24,11 +23,15 @@ import sys
|
||||
import types
|
||||
import xml.etree.ElementTree
|
||||
|
||||
_IDENTITY = lambda x: x
|
||||
|
||||
# naming convention
|
||||
# 'compat_' + Python3_name.replace('.', '_')
|
||||
# other aliases exist for convenience and/or legacy
|
||||
# wrap disposable test values in type() to reclaim storage
|
||||
|
||||
# deal with critical unicode/str things first
|
||||
# deal with critical unicode/str things first:
|
||||
# compat_str, compat_basestring, compat_chr
|
||||
try:
|
||||
# Python 2
|
||||
compat_str, compat_basestring, compat_chr = (
|
||||
@ -39,18 +42,23 @@ except NameError:
|
||||
str, (str, bytes), chr
|
||||
)
|
||||
|
||||
# casefold
|
||||
|
||||
# compat_casefold
|
||||
try:
|
||||
compat_str.casefold
|
||||
compat_casefold = lambda s: s.casefold()
|
||||
except AttributeError:
|
||||
from .casefold import _casefold as compat_casefold
|
||||
|
||||
|
||||
# compat_collections_abc
|
||||
try:
|
||||
import collections.abc as compat_collections_abc
|
||||
except ImportError:
|
||||
import collections as compat_collections_abc
|
||||
|
||||
|
||||
# compat_urllib_request
|
||||
try:
|
||||
import urllib.request as compat_urllib_request
|
||||
except ImportError: # Python 2
|
||||
@ -79,11 +87,15 @@ except TypeError:
|
||||
_add_init_method_arg(compat_urllib_request.Request)
|
||||
del _add_init_method_arg
|
||||
|
||||
|
||||
# compat_urllib_error
|
||||
try:
|
||||
import urllib.error as compat_urllib_error
|
||||
except ImportError: # Python 2
|
||||
import urllib2 as compat_urllib_error
|
||||
|
||||
|
||||
# compat_urllib_parse
|
||||
try:
|
||||
import urllib.parse as compat_urllib_parse
|
||||
except ImportError: # Python 2
|
||||
@ -98,17 +110,23 @@ except ImportError: # Python 2
|
||||
compat_urlparse = compat_urllib_parse
|
||||
compat_urllib_parse_urlparse = compat_urllib_parse.urlparse
|
||||
|
||||
|
||||
# compat_urllib_response
|
||||
try:
|
||||
import urllib.response as compat_urllib_response
|
||||
except ImportError: # Python 2
|
||||
import urllib as compat_urllib_response
|
||||
|
||||
|
||||
# compat_urllib_response.addinfourl
|
||||
try:
|
||||
compat_urllib_response.addinfourl.status
|
||||
except AttributeError:
|
||||
# .getcode() is deprecated in Py 3.
|
||||
compat_urllib_response.addinfourl.status = property(lambda self: self.getcode())
|
||||
|
||||
|
||||
# compat_http_cookiejar
|
||||
try:
|
||||
import http.cookiejar as compat_cookiejar
|
||||
except ImportError: # Python 2
|
||||
@ -127,12 +145,16 @@ else:
|
||||
compat_cookiejar_Cookie = compat_cookiejar.Cookie
|
||||
compat_http_cookiejar_Cookie = compat_cookiejar_Cookie
|
||||
|
||||
|
||||
# compat_http_cookies
|
||||
try:
|
||||
import http.cookies as compat_cookies
|
||||
except ImportError: # Python 2
|
||||
import Cookie as compat_cookies
|
||||
compat_http_cookies = compat_cookies
|
||||
|
||||
|
||||
# compat_http_cookies_SimpleCookie
|
||||
if sys.version_info[0] == 2 or sys.version_info < (3, 3):
|
||||
class compat_cookies_SimpleCookie(compat_cookies.SimpleCookie):
|
||||
def load(self, rawdata):
|
||||
@ -155,11 +177,15 @@ else:
|
||||
compat_cookies_SimpleCookie = compat_cookies.SimpleCookie
|
||||
compat_http_cookies_SimpleCookie = compat_cookies_SimpleCookie
|
||||
|
||||
|
||||
# compat_html_entities, probably useless now
|
||||
try:
|
||||
import html.entities as compat_html_entities
|
||||
except ImportError: # Python 2
|
||||
import htmlentitydefs as compat_html_entities
|
||||
|
||||
|
||||
# compat_html_entities_html5
|
||||
try: # Python >= 3.3
|
||||
compat_html_entities_html5 = compat_html_entities.html5
|
||||
except AttributeError:
|
||||
@ -2408,18 +2434,24 @@ except AttributeError:
|
||||
# Py < 3.1
|
||||
compat_http_client.HTTPResponse.getcode = lambda self: self.status
|
||||
|
||||
|
||||
# compat_urllib_HTTPError
|
||||
try:
|
||||
from urllib.error import HTTPError as compat_HTTPError
|
||||
except ImportError: # Python 2
|
||||
from urllib2 import HTTPError as compat_HTTPError
|
||||
compat_urllib_HTTPError = compat_HTTPError
|
||||
|
||||
|
||||
# compat_urllib_request_urlretrieve
|
||||
try:
|
||||
from urllib.request import urlretrieve as compat_urlretrieve
|
||||
except ImportError: # Python 2
|
||||
from urllib import urlretrieve as compat_urlretrieve
|
||||
compat_urllib_request_urlretrieve = compat_urlretrieve
|
||||
|
||||
|
||||
# compat_html_parser_HTMLParser, compat_html_parser_HTMLParseError
|
||||
try:
|
||||
from HTMLParser import (
|
||||
HTMLParser as compat_HTMLParser,
|
||||
@ -2432,22 +2464,33 @@ except ImportError: # Python 3
|
||||
# HTMLParseError was deprecated in Python 3.3 and removed in
|
||||
# Python 3.5. Introducing dummy exception for Python >3.5 for compatible
|
||||
# and uniform cross-version exception handling
|
||||
|
||||
class compat_HTMLParseError(Exception):
|
||||
pass
|
||||
|
||||
compat_html_parser_HTMLParser = compat_HTMLParser
|
||||
compat_html_parser_HTMLParseError = compat_HTMLParseError
|
||||
|
||||
|
||||
# compat_subprocess_get_DEVNULL
|
||||
try:
|
||||
_DEVNULL = subprocess.DEVNULL
|
||||
compat_subprocess_get_DEVNULL = lambda: _DEVNULL
|
||||
except AttributeError:
|
||||
compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
|
||||
|
||||
|
||||
# compat_http_server
|
||||
try:
|
||||
import http.server as compat_http_server
|
||||
except ImportError:
|
||||
import BaseHTTPServer as compat_http_server
|
||||
|
||||
|
||||
# compat_urllib_parse_unquote_to_bytes,
|
||||
# compat_urllib_parse_unquote, compat_urllib_parse_unquote_plus,
|
||||
# compat_urllib_parse_urlencode,
|
||||
# compat_urllib_parse_parse_qs
|
||||
try:
|
||||
from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
|
||||
from urllib.parse import unquote as compat_urllib_parse_unquote
|
||||
@ -2598,6 +2641,8 @@ except ImportError: # Python 2
|
||||
|
||||
compat_urllib_parse_parse_qs = compat_parse_qs
|
||||
|
||||
|
||||
# compat_urllib_request_DataHandler
|
||||
try:
|
||||
from urllib.request import DataHandler as compat_urllib_request_DataHandler
|
||||
except ImportError: # Python < 3.4
|
||||
@ -2632,16 +2677,20 @@ except ImportError: # Python < 3.4
|
||||
|
||||
return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)
|
||||
|
||||
|
||||
# compat_xml_etree_ElementTree_ParseError
|
||||
try:
|
||||
from xml.etree.ElementTree import ParseError as compat_xml_parse_error
|
||||
except ImportError: # Python 2.6
|
||||
from xml.parsers.expat import ExpatError as compat_xml_parse_error
|
||||
compat_xml_etree_ElementTree_ParseError = compat_xml_parse_error
|
||||
|
||||
etree = xml.etree.ElementTree
|
||||
|
||||
# compat_xml_etree_ElementTree_Element
|
||||
_etree = xml.etree.ElementTree
|
||||
|
||||
|
||||
class _TreeBuilder(etree.TreeBuilder):
|
||||
class _TreeBuilder(_etree.TreeBuilder):
|
||||
def doctype(self, name, pubid, system):
|
||||
pass
|
||||
|
||||
@ -2650,7 +2699,7 @@ try:
|
||||
# xml.etree.ElementTree.Element is a method in Python <=2.6 and
|
||||
# the following will crash with:
|
||||
# TypeError: isinstance() arg 2 must be a class, type, or tuple of classes and types
|
||||
isinstance(None, etree.Element)
|
||||
isinstance(None, _etree.Element)
|
||||
from xml.etree.ElementTree import Element as compat_etree_Element
|
||||
except TypeError: # Python <=2.6
|
||||
from xml.etree.ElementTree import _ElementInterface as compat_etree_Element
|
||||
@ -2658,12 +2707,12 @@ compat_xml_etree_ElementTree_Element = compat_etree_Element
|
||||
|
||||
if sys.version_info[0] >= 3:
|
||||
def compat_etree_fromstring(text):
|
||||
return etree.XML(text, parser=etree.XMLParser(target=_TreeBuilder()))
|
||||
return _etree.XML(text, parser=_etree.XMLParser(target=_TreeBuilder()))
|
||||
else:
|
||||
# python 2.x tries to encode unicode strings with ascii (see the
|
||||
# XMLParser._fixtext method)
|
||||
try:
|
||||
_etree_iter = etree.Element.iter
|
||||
_etree_iter = _etree.Element.iter
|
||||
except AttributeError: # Python <=2.6
|
||||
def _etree_iter(root):
|
||||
for el in root.findall('*'):
|
||||
@ -2675,27 +2724,29 @@ else:
|
||||
# 2.7 source
|
||||
def _XML(text, parser=None):
|
||||
if not parser:
|
||||
parser = etree.XMLParser(target=_TreeBuilder())
|
||||
parser = _etree.XMLParser(target=_TreeBuilder())
|
||||
parser.feed(text)
|
||||
return parser.close()
|
||||
|
||||
def _element_factory(*args, **kwargs):
|
||||
el = etree.Element(*args, **kwargs)
|
||||
el = _etree.Element(*args, **kwargs)
|
||||
for k, v in el.items():
|
||||
if isinstance(v, bytes):
|
||||
el.set(k, v.decode('utf-8'))
|
||||
return el
|
||||
|
||||
def compat_etree_fromstring(text):
|
||||
doc = _XML(text, parser=etree.XMLParser(target=_TreeBuilder(element_factory=_element_factory)))
|
||||
doc = _XML(text, parser=_etree.XMLParser(target=_TreeBuilder(element_factory=_element_factory)))
|
||||
for el in _etree_iter(doc):
|
||||
if el.text is not None and isinstance(el.text, bytes):
|
||||
el.text = el.text.decode('utf-8')
|
||||
return doc
|
||||
|
||||
if hasattr(etree, 'register_namespace'):
|
||||
compat_etree_register_namespace = etree.register_namespace
|
||||
else:
|
||||
|
||||
# compat_xml_etree_register_namespace
|
||||
try:
|
||||
compat_etree_register_namespace = _etree.register_namespace
|
||||
except AttributeError:
|
||||
def compat_etree_register_namespace(prefix, uri):
|
||||
"""Register a namespace prefix.
|
||||
The registry is global, and any existing mapping for either the
|
||||
@ -2704,14 +2755,16 @@ else:
|
||||
attributes in this namespace will be serialized with prefix if possible.
|
||||
ValueError is raised if prefix is reserved or is invalid.
|
||||
"""
|
||||
if re.match(r"ns\d+$", prefix):
|
||||
raise ValueError("Prefix format reserved for internal use")
|
||||
for k, v in list(etree._namespace_map.items()):
|
||||
if re.match(r'ns\d+$', prefix):
|
||||
raise ValueError('Prefix format reserved for internal use')
|
||||
for k, v in list(_etree._namespace_map.items()):
|
||||
if k == uri or v == prefix:
|
||||
del etree._namespace_map[k]
|
||||
etree._namespace_map[uri] = prefix
|
||||
del _etree._namespace_map[k]
|
||||
_etree._namespace_map[uri] = prefix
|
||||
compat_xml_etree_register_namespace = compat_etree_register_namespace
|
||||
|
||||
|
||||
# compat_xpath, compat_etree_iterfind
|
||||
if sys.version_info < (2, 7):
|
||||
# Here comes the crazy part: In 2.6, if the xpath is a unicode,
|
||||
# .//node does not match if a node is a direct child of . !
|
||||
@ -2898,7 +2951,6 @@ if sys.version_info < (2, 7):
|
||||
def __init__(self, root):
|
||||
self.root = root
|
||||
|
||||
##
|
||||
# Generate all matching objects.
|
||||
|
||||
def compat_etree_iterfind(elem, path, namespaces=None):
|
||||
@ -2933,13 +2985,15 @@ if sys.version_info < (2, 7):
|
||||
|
||||
|
||||
else:
|
||||
compat_xpath = lambda xpath: xpath
|
||||
compat_etree_iterfind = lambda element, match: element.iterfind(match)
|
||||
compat_xpath = _IDENTITY
|
||||
|
||||
|
||||
# compat_os_name
|
||||
compat_os_name = os._name if os.name == 'java' else os.name
|
||||
|
||||
|
||||
# compat_shlex_quote
|
||||
if compat_os_name == 'nt':
|
||||
def compat_shlex_quote(s):
|
||||
return s if re.match(r'^[-_\w./]+$', s) else '"%s"' % s.replace('"', '\\"')
|
||||
@ -2954,6 +3008,7 @@ else:
|
||||
return "'" + s.replace("'", "'\"'\"'") + "'"
|
||||
|
||||
|
||||
# compat_shlex.split
|
||||
try:
|
||||
args = shlex.split('中文')
|
||||
assert (isinstance(args, list)
|
||||
@ -2969,6 +3024,7 @@ except (AssertionError, UnicodeEncodeError):
|
||||
return list(map(lambda s: s.decode('utf-8'), shlex.split(s, comments, posix)))
|
||||
|
||||
|
||||
# compat_ord
|
||||
def compat_ord(c):
|
||||
if isinstance(c, int):
|
||||
return c
|
||||
@ -2976,6 +3032,7 @@ def compat_ord(c):
|
||||
return ord(c)
|
||||
|
||||
|
||||
# compat_getenv, compat_os_path_expanduser, compat_setenv
|
||||
if sys.version_info >= (3, 0):
|
||||
compat_getenv = os.getenv
|
||||
compat_expanduser = os.path.expanduser
|
||||
@ -3063,6 +3120,7 @@ else:
|
||||
compat_os_path_expanduser = compat_expanduser
|
||||
|
||||
|
||||
# compat_os_path_realpath
|
||||
if compat_os_name == 'nt' and sys.version_info < (3, 8):
|
||||
# os.path.realpath on Windows does not follow symbolic links
|
||||
# prior to Python 3.8 (see https://bugs.python.org/issue9949)
|
||||
@ -3076,6 +3134,7 @@ else:
|
||||
compat_os_path_realpath = compat_realpath
|
||||
|
||||
|
||||
# compat_print
|
||||
if sys.version_info < (3, 0):
|
||||
def compat_print(s):
|
||||
from .utils import preferredencoding
|
||||
@ -3086,6 +3145,7 @@ else:
|
||||
print(s)
|
||||
|
||||
|
||||
# compat_getpass_getpass
|
||||
if sys.version_info < (3, 0) and sys.platform == 'win32':
|
||||
def compat_getpass(prompt, *args, **kwargs):
|
||||
if isinstance(prompt, compat_str):
|
||||
@ -3098,22 +3158,22 @@ else:
|
||||
compat_getpass_getpass = compat_getpass
|
||||
|
||||
|
||||
# compat_input
|
||||
try:
|
||||
compat_input = raw_input
|
||||
except NameError: # Python 3
|
||||
compat_input = input
|
||||
|
||||
|
||||
# compat_kwargs
|
||||
# Python < 2.6.5 require kwargs to be bytes
|
||||
try:
|
||||
def _testfunc(x):
|
||||
pass
|
||||
_testfunc(**{'x': 0})
|
||||
(lambda x: x)(**{'x': 0})
|
||||
except TypeError:
|
||||
def compat_kwargs(kwargs):
|
||||
return dict((bytes(k), v) for k, v in kwargs.items())
|
||||
else:
|
||||
compat_kwargs = lambda kwargs: kwargs
|
||||
compat_kwargs = _IDENTITY
|
||||
|
||||
|
||||
# compat_numeric_types
|
||||
@ -3132,6 +3192,8 @@ except NameError: # Python 3
|
||||
# compat_int
|
||||
compat_int = compat_integer_types[-1]
|
||||
|
||||
|
||||
# compat_socket_create_connection
|
||||
if sys.version_info < (2, 7):
|
||||
def compat_socket_create_connection(address, timeout, source_address=None):
|
||||
host, port = address
|
||||
@ -3158,6 +3220,7 @@ else:
|
||||
compat_socket_create_connection = socket.create_connection
|
||||
|
||||
|
||||
# compat_contextlib_suppress
|
||||
try:
|
||||
from contextlib import suppress as compat_contextlib_suppress
|
||||
except ImportError:
|
||||
@ -3200,12 +3263,12 @@ except AttributeError:
|
||||
# repeated .close() is OK, but just in case
|
||||
with compat_contextlib_suppress(EnvironmentError):
|
||||
f.close()
|
||||
popen.wait()
|
||||
popen.wait()
|
||||
|
||||
|
||||
# Fix https://github.com/ytdl-org/youtube-dl/issues/4223
|
||||
# See http://bugs.python.org/issue9161 for what is broken
|
||||
def workaround_optparse_bug9161():
|
||||
def _workaround_optparse_bug9161():
|
||||
op = optparse.OptionParser()
|
||||
og = optparse.OptionGroup(op, 'foo')
|
||||
try:
|
||||
@ -3224,9 +3287,10 @@ def workaround_optparse_bug9161():
|
||||
optparse.OptionGroup.add_option = _compat_add_option
|
||||
|
||||
|
||||
if hasattr(shutil, 'get_terminal_size'): # Python >= 3.3
|
||||
compat_get_terminal_size = shutil.get_terminal_size
|
||||
else:
|
||||
# compat_shutil_get_terminal_size
|
||||
try:
|
||||
from shutil import get_terminal_size as compat_get_terminal_size # Python >= 3.3
|
||||
except ImportError:
|
||||
_terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])
|
||||
|
||||
def compat_get_terminal_size(fallback=(80, 24)):
|
||||
@ -3256,27 +3320,33 @@ else:
|
||||
columns = _columns
|
||||
if lines is None or lines <= 0:
|
||||
lines = _lines
|
||||
|
||||
return _terminal_size(columns, lines)
|
||||
|
||||
compat_shutil_get_terminal_size = compat_get_terminal_size
|
||||
|
||||
|
||||
# compat_itertools_count
|
||||
try:
|
||||
itertools.count(start=0, step=1)
|
||||
type(itertools.count(start=0, step=1))
|
||||
compat_itertools_count = itertools.count
|
||||
except TypeError: # Python 2.6
|
||||
except TypeError: # Python 2.6 lacks step
|
||||
def compat_itertools_count(start=0, step=1):
|
||||
while True:
|
||||
yield start
|
||||
start += step
|
||||
|
||||
|
||||
# compat_tokenize_tokenize
|
||||
if sys.version_info >= (3, 0):
|
||||
from tokenize import tokenize as compat_tokenize_tokenize
|
||||
else:
|
||||
from tokenize import generate_tokens as compat_tokenize_tokenize
|
||||
|
||||
|
||||
# compat_struct_pack, compat_struct_unpack, compat_Struct
|
||||
try:
|
||||
struct.pack('!I', 0)
|
||||
type(struct.pack('!I', 0))
|
||||
except TypeError:
|
||||
# In Python 2.6 and 2.7.x < 2.7.7, struct requires a bytes argument
|
||||
# See https://bugs.python.org/issue19099
|
||||
@ -3308,8 +3378,10 @@ else:
|
||||
compat_Struct = struct.Struct
|
||||
|
||||
|
||||
# compat_map/filter() returning an iterator, supposedly the
|
||||
# same versioning as for zip below
|
||||
# builtins returning an iterator
|
||||
|
||||
# compat_map, compat_filter
|
||||
# supposedly the same versioning as for zip below
|
||||
try:
|
||||
from future_builtins import map as compat_map
|
||||
except ImportError:
|
||||
@ -3326,6 +3398,7 @@ except ImportError:
|
||||
except ImportError:
|
||||
compat_filter = filter
|
||||
|
||||
# compat_zip
|
||||
try:
|
||||
from future_builtins import zip as compat_zip
|
||||
except ImportError: # not 2.6+ or is 3.x
|
||||
@ -3335,6 +3408,7 @@ except ImportError: # not 2.6+ or is 3.x
|
||||
compat_zip = zip
|
||||
|
||||
|
||||
# compat_itertools_zip_longest
|
||||
# method renamed between Py2/3
|
||||
try:
|
||||
from itertools import zip_longest as compat_itertools_zip_longest
|
||||
@ -3342,7 +3416,8 @@ except ImportError:
|
||||
from itertools import izip_longest as compat_itertools_zip_longest
|
||||
|
||||
|
||||
# new class in collections
|
||||
# compat_collections_chain_map
|
||||
# collections.ChainMap: new class
|
||||
try:
|
||||
from collections import ChainMap as compat_collections_chain_map
|
||||
# Py3.3's ChainMap is deficient
|
||||
@ -3398,19 +3473,22 @@ except ImportError:
|
||||
def new_child(self, m=None, **kwargs):
|
||||
m = m or {}
|
||||
m.update(kwargs)
|
||||
return compat_collections_chain_map(m, *self.maps)
|
||||
# support inheritance !
|
||||
return type(self)(m, *self.maps)
|
||||
|
||||
@property
|
||||
def parents(self):
|
||||
return compat_collections_chain_map(*(self.maps[1:]))
|
||||
return type(self)(*(self.maps[1:]))
|
||||
|
||||
|
||||
# compat_re_Pattern, compat_re_Match
|
||||
# Pythons disagree on the type of a pattern (RegexObject, _sre.SRE_Pattern, Pattern, ...?)
|
||||
compat_re_Pattern = type(re.compile(''))
|
||||
# and on the type of a match
|
||||
compat_re_Match = type(re.match('a', 'a'))
|
||||
|
||||
|
||||
# compat_base64_b64decode
|
||||
if sys.version_info < (3, 3):
|
||||
def compat_b64decode(s, *args, **kwargs):
|
||||
if isinstance(s, compat_str):
|
||||
@ -3422,6 +3500,7 @@ else:
|
||||
compat_base64_b64decode = compat_b64decode
|
||||
|
||||
|
||||
# compat_ctypes_WINFUNCTYPE
|
||||
if platform.python_implementation() == 'PyPy' and sys.pypy_version_info < (5, 4, 0):
|
||||
# PyPy2 prior to version 5.4.0 expects byte strings as Windows function
|
||||
# names, see the original PyPy issue [1] and the youtube-dl one [2].
|
||||
@ -3440,6 +3519,7 @@ else:
|
||||
return ctypes.WINFUNCTYPE(*args, **kwargs)
|
||||
|
||||
|
||||
# compat_open
|
||||
if sys.version_info < (3, 0):
|
||||
# open(file, mode='r', buffering=- 1, encoding=None, errors=None, newline=None, closefd=True) not: opener=None
|
||||
def compat_open(file_, *args, **kwargs):
|
||||
@ -3467,18 +3547,28 @@ except AttributeError:
|
||||
def compat_datetime_timedelta_total_seconds(td):
|
||||
return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6
|
||||
|
||||
|
||||
# optional decompression packages
|
||||
# compat_brotli
|
||||
# PyPi brotli package implements 'br' Content-Encoding
|
||||
try:
|
||||
import brotli as compat_brotli
|
||||
except ImportError:
|
||||
compat_brotli = None
|
||||
# compat_ncompress
|
||||
# PyPi ncompress package implements 'compress' Content-Encoding
|
||||
try:
|
||||
import ncompress as compat_ncompress
|
||||
except ImportError:
|
||||
compat_ncompress = None
|
||||
|
||||
# compat_zstandard
|
||||
# PyPi zstandard package implements 'zstd' Content-Encoding (RFC 8878 7.2)
|
||||
try:
|
||||
import zstandard as compat_zstandard
|
||||
except ImportError:
|
||||
compat_zstandard = None
|
||||
|
||||
|
||||
legacy = [
|
||||
'compat_HTMLParseError',
|
||||
@ -3495,6 +3585,7 @@ legacy = [
|
||||
'compat_getpass',
|
||||
'compat_parse_qs',
|
||||
'compat_realpath',
|
||||
'compat_shlex_split',
|
||||
'compat_urllib_parse_parse_qs',
|
||||
'compat_urllib_parse_unquote',
|
||||
'compat_urllib_parse_unquote_plus',
|
||||
@ -3508,8 +3599,6 @@ legacy = [
|
||||
|
||||
|
||||
__all__ = [
|
||||
'compat_html_parser_HTMLParseError',
|
||||
'compat_html_parser_HTMLParser',
|
||||
'compat_Struct',
|
||||
'compat_base64_b64decode',
|
||||
'compat_basestring',
|
||||
@ -3518,13 +3607,9 @@ __all__ = [
|
||||
'compat_chr',
|
||||
'compat_collections_abc',
|
||||
'compat_collections_chain_map',
|
||||
'compat_datetime_timedelta_total_seconds',
|
||||
'compat_http_cookiejar',
|
||||
'compat_http_cookiejar_Cookie',
|
||||
'compat_http_cookies',
|
||||
'compat_http_cookies_SimpleCookie',
|
||||
'compat_contextlib_suppress',
|
||||
'compat_ctypes_WINFUNCTYPE',
|
||||
'compat_datetime_timedelta_total_seconds',
|
||||
'compat_etree_fromstring',
|
||||
'compat_etree_iterfind',
|
||||
'compat_filter',
|
||||
@ -3533,6 +3618,12 @@ __all__ = [
|
||||
'compat_getpass_getpass',
|
||||
'compat_html_entities',
|
||||
'compat_html_entities_html5',
|
||||
'compat_html_parser_HTMLParseError',
|
||||
'compat_html_parser_HTMLParser',
|
||||
'compat_http_cookiejar',
|
||||
'compat_http_cookiejar_Cookie',
|
||||
'compat_http_cookies',
|
||||
'compat_http_cookies_SimpleCookie',
|
||||
'compat_http_client',
|
||||
'compat_http_server',
|
||||
'compat_input',
|
||||
@ -3555,7 +3646,7 @@ __all__ = [
|
||||
'compat_register_utf8',
|
||||
'compat_setenv',
|
||||
'compat_shlex_quote',
|
||||
'compat_shlex_split',
|
||||
'compat_shutil_get_terminal_size',
|
||||
'compat_socket_create_connection',
|
||||
'compat_str',
|
||||
'compat_struct_pack',
|
||||
@ -3575,5 +3666,5 @@ __all__ = [
|
||||
'compat_xml_etree_register_namespace',
|
||||
'compat_xpath',
|
||||
'compat_zip',
|
||||
'workaround_optparse_bug9161',
|
||||
'compat_zstandard',
|
||||
]
|
||||
|
||||
@ -27,6 +27,7 @@ from ..compat import (
|
||||
)
|
||||
from ..jsinterp import JSInterpreter
|
||||
from ..utils import (
|
||||
bug_reports_message,
|
||||
clean_html,
|
||||
dict_get,
|
||||
error_to_compat_str,
|
||||
@ -65,6 +66,7 @@ from ..utils import (
|
||||
url_or_none,
|
||||
urlencode_postdata,
|
||||
urljoin,
|
||||
variadic,
|
||||
)
|
||||
|
||||
|
||||
@ -120,7 +122,8 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
|
||||
'INNERTUBE_CONTEXT': {
|
||||
'client': {
|
||||
'clientName': 'TVHTML5',
|
||||
'clientVersion': '7.20241201.18.00',
|
||||
'clientVersion': '7.20250120.19.00',
|
||||
'userAgent': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/Version',
|
||||
},
|
||||
},
|
||||
'INNERTUBE_CONTEXT_CLIENT_NAME': 7,
|
||||
@ -460,6 +463,26 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
|
||||
'uploader': uploader,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _extract_thumbnails(data, *path_list, **kw_final_key):
|
||||
"""
|
||||
Extract thumbnails from thumbnails dict
|
||||
@param path_list: path list to level that contains 'thumbnails' key
|
||||
"""
|
||||
final_key = kw_final_key.get('final_key', 'thumbnails')
|
||||
|
||||
return traverse_obj(data, ((
|
||||
tuple(variadic(path) + (final_key, Ellipsis)
|
||||
for path in path_list or [()])), {
|
||||
'url': ('url', T(url_or_none),
|
||||
# Sometimes youtube gives a wrong thumbnail URL. See:
|
||||
# https://github.com/yt-dlp/yt-dlp/issues/233
|
||||
# https://github.com/ytdl-org/youtube-dl/issues/28023
|
||||
T(lambda u: update_url(u, query=None) if u and 'maxresdefault' in u else u)),
|
||||
'height': ('height', T(int_or_none)),
|
||||
'width': ('width', T(int_or_none)),
|
||||
}, T(lambda t: t if t.get('url') else None)))
|
||||
|
||||
def _search_results(self, query, params):
|
||||
data = {
|
||||
'context': {
|
||||
@ -1829,12 +1852,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|
||||
|
||||
if func_code:
|
||||
return jsi, player_id, func_code
|
||||
return self._extract_n_function_code_jsi(video_id, jsi, player_id)
|
||||
|
||||
func_name = self._extract_n_function_name(jscode)
|
||||
def _extract_n_function_code_jsi(self, video_id, jsi, player_id=None):
|
||||
|
||||
var_ay = self._search_regex(
|
||||
r'(?:[;\s]|^)\s*(var\s*[\w$]+\s*=\s*"[^"]+"\s*\.\s*split\("\{"\))(?=\s*[,;])',
|
||||
jsi.code, 'useful values', default='')
|
||||
|
||||
func_name = self._extract_n_function_name(jsi.code)
|
||||
|
||||
func_code = jsi.extract_function_code(func_name)
|
||||
if var_ay:
|
||||
func_code = (func_code[0], ';\n'.join((var_ay, func_code[1])))
|
||||
|
||||
self.cache.store('youtube-nsig', player_id, func_code)
|
||||
if player_id:
|
||||
self.cache.store('youtube-nsig', player_id, func_code)
|
||||
return jsi, player_id, func_code
|
||||
|
||||
def _extract_n_function_from_code(self, jsi, func_code):
|
||||
@ -3183,8 +3216,12 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
|
||||
expected_type=txt_or_none)
|
||||
|
||||
def _grid_entries(self, grid_renderer):
|
||||
for item in grid_renderer['items']:
|
||||
if not isinstance(item, dict):
|
||||
for item in traverse_obj(grid_renderer, ('items', Ellipsis, T(dict))):
|
||||
lockup_view_model = traverse_obj(item, ('lockupViewModel', T(dict)))
|
||||
if lockup_view_model:
|
||||
entry = self._extract_lockup_view_model(lockup_view_model)
|
||||
if entry:
|
||||
yield entry
|
||||
continue
|
||||
renderer = self._extract_grid_item_renderer(item)
|
||||
if not isinstance(renderer, dict):
|
||||
@ -3268,6 +3305,25 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
|
||||
continue
|
||||
yield self._extract_video(renderer)
|
||||
|
||||
def _extract_lockup_view_model(self, view_model):
|
||||
content_id = view_model.get('contentId')
|
||||
if not content_id:
|
||||
return
|
||||
content_type = view_model.get('contentType')
|
||||
if content_type not in ('LOCKUP_CONTENT_TYPE_PLAYLIST', 'LOCKUP_CONTENT_TYPE_PODCAST'):
|
||||
self.report_warning(
|
||||
'Unsupported lockup view model content type "{0}"{1}'.format(content_type, bug_reports_message()), only_once=True)
|
||||
return
|
||||
return merge_dicts(self.url_result(
|
||||
update_url_query('https://www.youtube.com/playlist', {'list': content_id}),
|
||||
ie=YoutubeTabIE.ie_key(), video_id=content_id), {
|
||||
'title': traverse_obj(view_model, (
|
||||
'metadata', 'lockupMetadataViewModel', 'title', 'content', T(compat_str))),
|
||||
'thumbnails': self._extract_thumbnails(view_model, (
|
||||
'contentImage', 'collectionThumbnailViewModel', 'primaryThumbnail',
|
||||
'thumbnailViewModel', 'image'), final_key='sources'),
|
||||
})
|
||||
|
||||
def _video_entry(self, video_renderer):
|
||||
video_id = video_renderer.get('videoId')
|
||||
if video_id:
|
||||
|
||||
@ -1,10 +1,12 @@
|
||||
# coding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import calendar
|
||||
import itertools
|
||||
import json
|
||||
import operator
|
||||
import re
|
||||
import time
|
||||
|
||||
from functools import update_wrapper, wraps
|
||||
|
||||
@ -12,8 +14,10 @@ from .utils import (
|
||||
error_to_compat_str,
|
||||
ExtractorError,
|
||||
float_or_none,
|
||||
int_or_none,
|
||||
js_to_json,
|
||||
remove_quotes,
|
||||
str_or_none,
|
||||
unified_timestamp,
|
||||
variadic,
|
||||
write_string,
|
||||
@ -150,6 +154,7 @@ def _js_to_primitive(v):
|
||||
)
|
||||
|
||||
|
||||
# more exact: yt-dlp/yt-dlp#12110
|
||||
def _js_toString(v):
|
||||
return (
|
||||
'undefined' if v is JS_Undefined
|
||||
@ -158,7 +163,7 @@ def _js_toString(v):
|
||||
else 'null' if v is None
|
||||
# bool <= int: do this first
|
||||
else ('false', 'true')[v] if isinstance(v, bool)
|
||||
else '{0:.7f}'.format(v).rstrip('.0') if isinstance(v, compat_numeric_types)
|
||||
else re.sub(r'(?<=\d)\.?0*$', '', '{0:.7f}'.format(v)) if isinstance(v, compat_numeric_types)
|
||||
else _js_to_primitive(v))
|
||||
|
||||
|
||||
@ -404,6 +409,7 @@ class JSInterpreter(object):
|
||||
class Exception(ExtractorError):
|
||||
def __init__(self, msg, *args, **kwargs):
|
||||
expr = kwargs.pop('expr', None)
|
||||
msg = str_or_none(msg, default='"None"')
|
||||
if expr is not None:
|
||||
msg = '{0} in: {1!r:.100}'.format(msg.rstrip(), expr)
|
||||
super(JSInterpreter.Exception, self).__init__(msg, *args, **kwargs)
|
||||
@ -431,6 +437,7 @@ class JSInterpreter(object):
|
||||
flags, _ = self.regex_flags(flags)
|
||||
# First, avoid https://github.com/python/cpython/issues/74534
|
||||
self.__self = None
|
||||
pattern_txt = str_or_none(pattern_txt) or '(?:)'
|
||||
self.__pattern_txt = pattern_txt.replace('[[', r'[\[')
|
||||
self.__flags = flags
|
||||
|
||||
@ -475,6 +482,73 @@ class JSInterpreter(object):
|
||||
flags |= cls.RE_FLAGS[ch]
|
||||
return flags, expr[idx + 1:]
|
||||
|
||||
class JS_Date(object):
|
||||
_t = None
|
||||
|
||||
@staticmethod
|
||||
def __ymd_etc(*args, **kw_is_utc):
|
||||
# args: year, monthIndex, day, hours, minutes, seconds, milliseconds
|
||||
is_utc = kw_is_utc.get('is_utc', False)
|
||||
|
||||
args = list(args[:7])
|
||||
args += [0] * (9 - len(args))
|
||||
args[1] += 1 # month 0..11 -> 1..12
|
||||
ms = args[6]
|
||||
for i in range(6, 9):
|
||||
args[i] = -1 # don't know
|
||||
if is_utc:
|
||||
args[-1] = 1
|
||||
# TODO: [MDN] When a segment overflows or underflows its expected
|
||||
# range, it usually "carries over to" or "borrows from" the higher segment.
|
||||
try:
|
||||
mktime = calendar.timegm if is_utc else time.mktime
|
||||
return mktime(time.struct_time(args)) * 1000 + ms
|
||||
except (OverflowError, ValueError):
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def UTC(cls, *args):
|
||||
t = cls.__ymd_etc(*args, is_utc=True)
|
||||
return _NaN if t is None else t
|
||||
|
||||
@staticmethod
|
||||
def parse(date_str, **kw_is_raw):
|
||||
is_raw = kw_is_raw.get('is_raw', False)
|
||||
|
||||
t = unified_timestamp(str_or_none(date_str), False)
|
||||
return int(t * 1000) if t is not None else t if is_raw else _NaN
|
||||
|
||||
@staticmethod
|
||||
def now(**kw_is_raw):
|
||||
is_raw = kw_is_raw.get('is_raw', False)
|
||||
|
||||
t = time.time()
|
||||
return int(t * 1000) if t is not None else t if is_raw else _NaN
|
||||
|
||||
def __init__(self, *args):
|
||||
if not args:
|
||||
args = [self.now(is_raw=True)]
|
||||
if len(args) == 1:
|
||||
if isinstance(args[0], JSInterpreter.JS_Date):
|
||||
self._t = int_or_none(args[0].valueOf(), default=None)
|
||||
else:
|
||||
arg_type = _js_typeof(args[0])
|
||||
if arg_type == 'string':
|
||||
self._t = self.parse(args[0], is_raw=True)
|
||||
elif arg_type == 'number':
|
||||
self._t = int(args[0])
|
||||
else:
|
||||
self._t = self.__ymd_etc(*args)
|
||||
|
||||
def toString(self):
|
||||
try:
|
||||
return time.strftime('%a %b %0d %Y %H:%M:%S %Z%z', self._t).rstrip()
|
||||
except TypeError:
|
||||
return "Invalid Date"
|
||||
|
||||
def valueOf(self):
|
||||
return _NaN if self._t is None else self._t
|
||||
|
||||
@classmethod
|
||||
def __op_chars(cls):
|
||||
op_chars = set(';,[')
|
||||
@ -599,14 +673,15 @@ class JSInterpreter(object):
|
||||
except Exception as e:
|
||||
raise self.Exception('Failed to evaluate {left_val!r:.50} {op} {right_val!r:.50}'.format(**locals()), expr, cause=e)
|
||||
|
||||
def _index(self, obj, idx, allow_undefined=True):
|
||||
def _index(self, obj, idx, allow_undefined=None):
|
||||
if idx == 'length' and isinstance(obj, list):
|
||||
return len(obj)
|
||||
try:
|
||||
return obj[int(idx)] if isinstance(obj, list) else obj[compat_str(idx)]
|
||||
except (TypeError, KeyError, IndexError) as e:
|
||||
if allow_undefined:
|
||||
# when is not allowed?
|
||||
# allow_undefined is None gives correct behaviour
|
||||
if allow_undefined or (
|
||||
allow_undefined is None and not isinstance(e, TypeError)):
|
||||
return JS_Undefined
|
||||
raise self.Exception('Cannot get index {idx!r:.100}'.format(**locals()), expr=repr(obj), cause=e)
|
||||
|
||||
@ -715,7 +790,7 @@ class JSInterpreter(object):
|
||||
|
||||
new_kw, _, obj = expr.partition('new ')
|
||||
if not new_kw:
|
||||
for klass, konstr in (('Date', lambda x: int(unified_timestamp(x, False) * 1000)),
|
||||
for klass, konstr in (('Date', lambda *x: self.JS_Date(*x).valueOf()),
|
||||
('RegExp', self.JS_RegExp),
|
||||
('Error', self.Exception)):
|
||||
if not obj.startswith(klass + '('):
|
||||
@ -1034,6 +1109,7 @@ class JSInterpreter(object):
|
||||
'String': compat_str,
|
||||
'Math': float,
|
||||
'Array': list,
|
||||
'Date': self.JS_Date,
|
||||
}
|
||||
obj = local_vars.get(variable)
|
||||
if obj in (JS_Undefined, None):
|
||||
@ -1086,6 +1162,8 @@ class JSInterpreter(object):
|
||||
assertion(len(argvals) == 2, 'takes two arguments')
|
||||
return argvals[0] ** argvals[1]
|
||||
raise self.Exception('Unsupported Math method ' + member, expr=expr)
|
||||
elif obj is self.JS_Date:
|
||||
return getattr(obj, member)(*argvals)
|
||||
|
||||
if member == 'split':
|
||||
assertion(len(argvals) <= 2, 'takes at most two arguments')
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user