Merge 525c46942deeeca2430fdd5717c82df4f6ac3422 into da7223d4aa42ff9fc680b0951d043dd03cec2d30

* improve extraction of global "useful data" Array from player JS
* also handle tv-player and add tests: thx seproDev (yt-dlp/yt-dlp#12684)

Co-Authored-By: sepro <sepro@sepr0.com>

README.md

@@ -1,29 +1,29 @@
 [![Build Status](https://github.com/ytdl-org/youtube-dl/workflows/CI/badge.svg)](https://github.com/ytdl-org/youtube-dl/actions?query=workflow%3ACI)
 
 
-youtube-dl - download videos from youtube.com or other video platforms
+youtube-dl - download videos from youtube.com or other video platforms.
 
-- [INSTALLATION](#installation)
+- [Installation](#installation)
-- [DESCRIPTION](#description)
+- [Description](#description)
-- [OPTIONS](#options)
+- [Options](#options)
-- [CONFIGURATION](#configuration)
+- [Configuration](#configuration)
-- [OUTPUT TEMPLATE](#output-template)
+- [Output template](#output-template)
-- [FORMAT SELECTION](#format-selection)
+- [Format selection](#format-selection)
-- [VIDEO SELECTION](#video-selection)
+- [Video selection](#video-selection)
 - [FAQ](#faq)
-- [DEVELOPER INSTRUCTIONS](#developer-instructions)
+- [Developer instructions](#developer-instructions)
-- [EMBEDDING YOUTUBE-DL](#embedding-youtube-dl)
+- [Embedding youtube-dl](#embedding-youtube-dl)
-- [BUGS](#bugs)
+- [Bugs](#bugs)
-- [COPYRIGHT](#copyright)
+- [Copyright](#copyright)
 
 # INSTALLATION
 
-To install it right away for all UNIX users (Linux, macOS, etc.), type:
+To install it right away for all Unix users (Linux, macOS, etc.), type:
 
     sudo curl -L https://yt-dl.org/downloads/latest/youtube-dl -o /usr/local/bin/youtube-dl
     sudo chmod a+rx /usr/local/bin/youtube-dl
 
-If you do not have curl, you can alternatively use a recent wget:
+If you do not have curl, you can alternatively use wget:
 
     sudo wget https://yt-dl.org/downloads/latest/youtube-dl -O /usr/local/bin/youtube-dl
     sudo chmod a+rx /usr/local/bin/youtube-dl
@@ -44,10 +44,10 @@ Or with [MacPorts](https://www.macports.org/):
 
     sudo port install youtube-dl
 
-Alternatively, refer to the [developer instructions](#developer-instructions) for how to check out and work with the git repository. For further options, including PGP signatures, see the [youtube-dl Download Page](https://ytdl-org.github.io/youtube-dl/download.html).
+Alternatively, refer to the [developer instructions](#developer-instructions) for how to check out and work with the Git repository. For further options, including PGP signatures, see the [youtube-dl Download Page](https://ytdl-org.github.io/youtube-dl/download.html).
 
 # DESCRIPTION
-**youtube-dl** is a command-line program to download videos from YouTube.com and a few more sites. It requires the Python interpreter, version 2.6, 2.7, or 3.2+, and it is not platform specific. It should work on your Unix box, on Windows or on macOS. It is released to the public domain, which means you can modify it, redistribute it or use it however you like.
+**youtube-dl** is a command-line program to download videos from YouTube.com and a few more sites. It requires the Python interpreter, version 2.6, 2.7, or 3.2+, and it is not platform specific. It should work on your Unix computer, on Windows or on macOS. It is released to the public domain, which means you can modify it, redistribute it or use it however you like.
 
     youtube-dl [OPTIONS] URL [URL...]
 
@@ -97,7 +97,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
                                          only)
     --no-color                           Do not emit color codes in output
 
-## Network Options:
+## Network options
     --proxy URL                          Use the specified HTTP/HTTPS/SOCKS
                                          proxy. To enable SOCKS proxy, specify a
                                          proper scheme. For example
@@ -110,7 +110,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     -4, --force-ipv4                     Make all connections via IPv4
     -6, --force-ipv6                     Make all connections via IPv6
 
-## Geo Restriction:
+## Geo restriction
     --geo-verification-proxy URL         Use this proxy to verify the IP address
                                          for some geo-restricted sites. The
                                          default proxy specified by --proxy (or
@@ -127,7 +127,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
                                          with explicitly provided IP block in
                                          CIDR notation
 
-## Video Selection:
+## Video selection
     --playlist-start NUMBER              Playlist video to start at (default is
                                          1)
     --playlist-end NUMBER                Playlist video to end at (default is
@@ -193,7 +193,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     --include-ads                        Download advertisements as well
                                          (experimental)
 
-## Download Options:
+## Download options
     -r, --limit-rate RATE                Maximum download rate in bytes per
                                          second (e.g. 50K or 4.2M)
     -R, --retries RETRIES                Number of retries (default is 10), or
@@ -239,7 +239,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     --external-downloader-args ARGS      Give these arguments to the external
                                          downloader
 
-## Filesystem Options:
+## Filesystem options
     -a, --batch-file FILE                File containing URLs to download ('-'
                                          for stdin), one URL per line. Lines
                                          starting with '#', ';' or ']' are
@@ -287,14 +287,14 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     --no-cache-dir                       Disable filesystem caching
     --rm-cache-dir                       Delete all filesystem cache files
 
-## Thumbnail Options:
+## Thumbnail options
     --write-thumbnail                    Write thumbnail image to disk
     --write-all-thumbnails               Write all thumbnail image formats to
                                          disk
     --list-thumbnails                    Simulate and list all available
                                          thumbnail formats
 
-## Verbosity / Simulation Options:
+## Verbosity / simulation options
     -q, --quiet                          Activate quiet mode
     --no-warnings                        Ignore warnings
     -s, --simulate                       Do not download the video and do not
@@ -336,7 +336,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     --no-call-home                       Do NOT contact the youtube-dl server
                                          for debugging
 
-## Workarounds:
+## Workarounds
     --encoding ENCODING                  Force the specified encoding
                                          (experimental)
     --no-check-certificate               Suppress HTTPS certificate validation
@@ -365,7 +365,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
                                          Must only be used along with --min-
                                          sleep-interval.
 
-## Video Format Options:
+## Video format options
     -f, --format FORMAT                  Video format code, see the "FORMAT
                                          SELECTION" for all the info
     --all-formats                        Download all available video formats
@@ -381,7 +381,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
                                          webm, flv. Ignored if no merge is
                                          required
 
-## Subtitle Options:
+## Subtitle options
     --write-sub                          Write subtitle file
     --write-auto-sub                     Write automatically generated subtitle
                                          file (YouTube only)
@@ -396,7 +396,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
                                          (optional) separated by commas, use
                                          --list-subs for available language tags
 
-## Authentication Options:
+## Authentication options
     -u, --username USERNAME              Login with this account ID
     -p, --password PASSWORD              Account password. If this option is
                                          left out, youtube-dl will ask
@@ -405,7 +405,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     -n, --netrc                          Use .netrc authentication data
     --video-password PASSWORD            Video password (vimeo, youku)
 
-## Adobe Pass Options:
+## Adobe Pass options
     --ap-mso MSO                         Adobe Pass multiple-system operator (TV
                                          provider) identifier, use --ap-list-mso
                                          for a list of available MSOs
@@ -416,7 +416,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     --ap-list-mso                        List all supported multiple-system
                                          operators
 
-## Post-processing Options:
+## Post-processing options
     -x, --extract-audio                  Convert video files to audio-only files
                                          (requires ffmpeg/avconv and
                                          ffprobe/avprobe)
@@ -480,7 +480,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
 
 # CONFIGURATION
 
-You can configure youtube-dl by placing any supported command line option to a configuration file. On Linux and macOS, the system wide configuration file is located at `/etc/youtube-dl.conf` and the user wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. Note that by default configuration file may not exist so you may need to create it yourself.
+You can configure youtube-dl by placing any supported command-line option to a configuration file. On Linux and macOS, the system-wide configuration file is located at `/etc/youtube-dl.conf` and the user-wide configuration file at `~/.config/youtube-dl/config`. On Windows, the user-wide configuration file locations are `%APPDATA%\youtube-dl\config.txt` or `C:\Users\<user name>\youtube-dl.conf`. Note that by default configuration file may not exist so you may need to create it yourself.
 
 For example, with the following configuration file youtube-dl will always extract the audio, not copy the mtime, use a proxy and save all videos under `Movies` directory in your home directory:
 ```
@@ -499,7 +499,7 @@ For example, with the following configuration file youtube-dl will always extrac
 -o ~/Movies/%(title)s.%(ext)s
 ```
 
-Note that options in configuration file are just the same options aka switches used in regular command line calls thus there **must be no whitespace** after `-` or `--`, e.g. `-o` or `--proxy` but not `- o` or `-- proxy`.
+Note that options in configuration file are just the same options aka switches used in regular command-line calls thus there **must be no whitespace** after `-` or `--`, e.g. `-o` or `--proxy` but not `- o` or `-- proxy`.
 
 You can use `--ignore-config` if you want to disable the configuration file for a particular youtube-dl run.
 
@@ -507,7 +507,7 @@ You can also use `--config-location` if you want to use custom configuration fil
 
 ### Authentication with `.netrc` file
 
-You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command line arguments on every youtube-dl execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](https://stackoverflow.com/tags/.netrc/info) on a per extractor basis. For that you will need to create a `.netrc` file in your `$HOME` and restrict permissions to read/write by only you:
+You may also want to configure automatic credentials storage for extractors that support authentication (by providing login and password with `--username` and `--password`) in order not to pass credentials as command-line arguments on every youtube-dl execution and prevent tracking plain text passwords in the shell command history. You can achieve this using a [`.netrc` file](https://stackoverflow.com/tags/.netrc/info) on a per extractor basis. For that you will need to create a `.netrc` file in your `$HOME` and restrict permissions to read/write by only you:
 ```
 touch $HOME/.netrc
 chmod a-rwx,u+rw $HOME/.netrc
@@ -534,7 +534,7 @@ The `-o` option allows users to indicate a template for the output file names.
 
 **tl;dr:** [navigate me to examples](#output-template-examples).
 
-The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "https://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [python string formatting operations](https://docs.python.org/2/library/stdtypes.html#string-formatting). For example, `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by formatting operations. Allowed names along with sequence type are:
+The basic usage is not to set any template arguments when downloading a single file, like in `youtube-dl -o funny_video.flv "https://some/video"`. However, it may contain special sequences that will be replaced when downloading each video. The special sequences may be formatted according to [Python string formatting operations](https://docs.python.org/2/library/stdtypes.html#string-formatting). For example, `%(NAME)s` or `%(NAME)05d`. To clarify, that is a percent symbol followed by a name in parentheses, followed by formatting operations. Allowed names along with sequence type are:
 
  - `id` (string): Video identifier
  - `title` (string): Video title
@@ -640,7 +640,7 @@ If you are using an output template inside a Windows batch file then you must es
 
 #### Output template examples
 
-Note that on Windows you may need to use double quotes instead of single.
+Note that on Windows you may need to use double quotes instead of single quotes.
 
 ```bash
 $ youtube-dl --get-filename -o '%(title)s.%(ext)s' BaW_jenozKc
@@ -733,7 +733,7 @@ If you want to preserve the old format selection behavior (prior to youtube-dl 2
 
 #### Format selection examples
 
-Note that on Windows you may need to use double quotes instead of single.
+Note that on Windows you may need to use double quotes instead of single quotes.
 
 ```bash
 # Download best mp4 format available or any other best if no mp4 available

test/test_jsinterp.py

@@ -11,6 +11,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import math
 import re
+import time
 
 from youtube_dl.compat import compat_str as str
 from youtube_dl.jsinterp import JS_Undefined, JSInterpreter
@@ -208,6 +209,34 @@ class TestJSInterpreter(unittest.TestCase):
         self._test(jsi, 86000, args=['12/31/1969 18:01:26 MDT'])
         # epoch 0
         self._test(jsi, 0, args=['1 January 1970 00:00:00 UTC'])
+        # undefined
+        self._test(jsi, NaN, args=[JS_Undefined])
+        # y,m,d, ... - may fail with older dates lacking DST data
+        jsi = JSInterpreter(
+            'function f() { return new Date(%s); }'
+            % ('2024, 5, 29, 2, 52, 12, 42',))
+        self._test(jsi, (
+            1719625932042                           # UK value
+            + (
+                + 3600                              # back to GMT
+                + (time.altzone if time.daylight    # host's DST
+                   else time.timezone)
+            ) * 1000))
+        # no arg
+        self.assertAlmostEqual(JSInterpreter(
+            'function f() { return new Date() - 0; }').call_function('f'),
+            time.time() * 1000, delta=100)
+        # Date.now()
+        self.assertAlmostEqual(JSInterpreter(
+            'function f() { return Date.now(); }').call_function('f'),
+            time.time() * 1000, delta=100)
+        # Date.parse()
+        jsi = JSInterpreter('function f(dt) { return Date.parse(dt); }')
+        self._test(jsi, 0, args=['1 January 1970 00:00:00 UTC'])
+        # Date.UTC()
+        jsi = JSInterpreter('function f() { return Date.UTC(%s); }'
+                            % ('1970, 0, 1, 0, 0, 0, 0',))
+        self._test(jsi, 0)
 
     def test_call(self):
         jsi = JSInterpreter('''
@@ -463,6 +492,14 @@ class TestJSInterpreter(unittest.TestCase):
         self._test('function f(){return NaN << 42}', 0)
         self._test('function f(){return "21.9" << 1}', 42)
         self._test('function f(){return 21 << 4294967297}', 42)
+        self._test('function f(){return true << "5";}', 32)
+        self._test('function f(){return true << true;}', 2)
+        self._test('function f(){return "19" & "21.9";}', 17)
+        self._test('function f(){return "19" & false;}', 0)
+        self._test('function f(){return "11.0" >> "2.1";}', 2)
+        self._test('function f(){return 5 ^ 9;}', 12)
+        self._test('function f(){return 0.0 << NaN}', 0)
+        self._test('function f(){return null << undefined}', 0)
 
     def test_negative(self):
         self._test('function f(){return 2    *    -2.0    ;}', -4)

test/test_youtube_signature.py

@@ -223,6 +223,42 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/9c6dfc4a/player_ias.vflset/en_US/base.js',
         'jbu7ylIosQHyJyJV', 'uwI0ESiynAmhNg',
     ),
+    (
+        'https://www.youtube.com/s/player/f6e09c70/player_ias.vflset/en_US/base.js',
+        'W9HJZKktxuYoDTqW', 'jHbbkcaxm54',
+    ),
+    (
+        'https://www.youtube.com/s/player/f6e09c70/player_ias_tce.vflset/en_US/base.js',
+        'W9HJZKktxuYoDTqW', 'jHbbkcaxm54',
+    ),
+    (
+        'https://www.youtube.com/s/player/643afba4/player_ias.vflset/en_US/base.js',
+        'W9HJZKktxuYoDTqW', 'larxUlagTRAcSw',
+    ),
+    (
+        'https://www.youtube.com/s/player/e7567ecf/player_ias_tce.vflset/en_US/base.js',
+        'Sy4aDGc0VpYRR9ew_', '5UPOT1VhoZxNLQ',
+    ),
+    (
+        'https://www.youtube.com/s/player/d50f54ef/player_ias_tce.vflset/en_US/base.js',
+        'Ha7507LzRmH3Utygtj', 'XFTb2HoeOE5MHg',
+    ),
+    (
+        'https://www.youtube.com/s/player/074a8365/player_ias_tce.vflset/en_US/base.js',
+        'Ha7507LzRmH3Utygtj', 'ufTsrE0IVYrkl8v',
+    ),
+    (
+        'https://www.youtube.com/s/player/643afba4/player_ias.vflset/en_US/base.js',
+        'N5uAlLqm0eg1GyHO', 'dCBQOejdq5s-ww',
+    ),
+    (
+        'https://www.youtube.com/s/player/69f581a5/tv-player-ias.vflset/tv-player-ias.js',
+        '-qIP447rVlTTwaZjY', 'KNcGOksBAvwqQg',
+    ),
+    (
+        'https://www.youtube.com/s/player/643afba4/tv-player-ias.vflset/tv-player-ias.js',
+        'ir9-V6cdbCiyKxhr', '2PL7ZDYAALMfmA',
+    ),
 ]
 
 
@@ -284,7 +320,7 @@ def t_factory(name, sig_func, url_pattern):
 
 
 def signature(jscode, sig_input):
-    func = YoutubeIE(FakeYDL())._parse_sig_js(jscode)
+    func = YoutubeIE(FakeYDL({'cachedir': False}))._parse_sig_js(jscode)
     src_sig = (
         compat_str(string.printable[:sig_input])
         if isinstance(sig_input, int) else sig_input)
@@ -292,9 +328,10 @@ def signature(jscode, sig_input):
 
 
 def n_sig(jscode, sig_input):
-    funcname = YoutubeIE(FakeYDL())._extract_n_function_name(jscode)
+    ie = YoutubeIE(FakeYDL({'cachedir': False}))
-    return JSInterpreter(jscode).call_function(
+    jsi = JSInterpreter(jscode)
-        funcname, sig_input, _ytdl_do_not_return=sig_input)
+    jsi, _, func_code = ie._extract_n_function_code_jsi(sig_input, jsi)
+    return ie._extract_n_function_from_code(jsi, func_code)(sig_input)
 
 
 make_sig_test = t_factory(

youtube_dl/__init__.py

@@ -18,7 +18,7 @@ from .compat import (
     compat_getpass,
     compat_register_utf8,
     compat_shlex_split,
-    workaround_optparse_bug9161,
+    _workaround_optparse_bug9161,
 )
 from .utils import (
     _UnsafeExtensionError,
@@ -50,7 +50,7 @@ def _real_main(argv=None):
     # Compatibility fix for Windows
     compat_register_utf8()
 
-    workaround_optparse_bug9161()
+    _workaround_optparse_bug9161()
 
     setproctitle('youtube-dl')
 

youtube_dl/compat.py

@@ -16,7 +16,6 @@ import os
 import platform
 import re
 import shlex
-import shutil
 import socket
 import struct
 import subprocess
@@ -24,11 +23,15 @@ import sys
 import types
 import xml.etree.ElementTree
 
+_IDENTITY = lambda x: x
+
 # naming convention
 # 'compat_' + Python3_name.replace('.', '_')
 # other aliases exist for convenience and/or legacy
+# wrap disposable test values in type() to reclaim storage
 
-# deal with critical unicode/str things first
+# deal with critical unicode/str things first:
+# compat_str, compat_basestring, compat_chr
 try:
     # Python 2
     compat_str, compat_basestring, compat_chr = (
@@ -39,18 +42,23 @@ except NameError:
         str, (str, bytes), chr
     )
 
-# casefold
+
+# compat_casefold
 try:
     compat_str.casefold
     compat_casefold = lambda s: s.casefold()
 except AttributeError:
     from .casefold import _casefold as compat_casefold
 
+
+# compat_collections_abc
 try:
     import collections.abc as compat_collections_abc
 except ImportError:
     import collections as compat_collections_abc
 
+
+# compat_urllib_request
 try:
     import urllib.request as compat_urllib_request
 except ImportError:  # Python 2
@@ -79,11 +87,15 @@ except TypeError:
     _add_init_method_arg(compat_urllib_request.Request)
     del _add_init_method_arg
 
+
+# compat_urllib_error
 try:
     import urllib.error as compat_urllib_error
 except ImportError:  # Python 2
     import urllib2 as compat_urllib_error
 
+
+# compat_urllib_parse
 try:
     import urllib.parse as compat_urllib_parse
 except ImportError:  # Python 2
@@ -98,17 +110,23 @@ except ImportError:  # Python 2
 compat_urlparse = compat_urllib_parse
 compat_urllib_parse_urlparse = compat_urllib_parse.urlparse
 
+
+# compat_urllib_response
 try:
     import urllib.response as compat_urllib_response
 except ImportError:  # Python 2
     import urllib as compat_urllib_response
 
+
+# compat_urllib_response.addinfourl
 try:
     compat_urllib_response.addinfourl.status
 except AttributeError:
     # .getcode() is deprecated in Py 3.
     compat_urllib_response.addinfourl.status = property(lambda self: self.getcode())
 
+
+# compat_http_cookiejar
 try:
     import http.cookiejar as compat_cookiejar
 except ImportError:  # Python 2
@@ -127,12 +145,16 @@ else:
     compat_cookiejar_Cookie = compat_cookiejar.Cookie
 compat_http_cookiejar_Cookie = compat_cookiejar_Cookie
 
+
+# compat_http_cookies
 try:
     import http.cookies as compat_cookies
 except ImportError:  # Python 2
     import Cookie as compat_cookies
 compat_http_cookies = compat_cookies
 
+
+# compat_http_cookies_SimpleCookie
 if sys.version_info[0] == 2 or sys.version_info < (3, 3):
     class compat_cookies_SimpleCookie(compat_cookies.SimpleCookie):
         def load(self, rawdata):
@@ -155,11 +177,15 @@ else:
     compat_cookies_SimpleCookie = compat_cookies.SimpleCookie
 compat_http_cookies_SimpleCookie = compat_cookies_SimpleCookie
 
+
+# compat_html_entities, probably useless now
 try:
     import html.entities as compat_html_entities
 except ImportError:  # Python 2
     import htmlentitydefs as compat_html_entities
 
+
+# compat_html_entities_html5
 try:  # Python >= 3.3
     compat_html_entities_html5 = compat_html_entities.html5
 except AttributeError:
@@ -2408,18 +2434,24 @@ except AttributeError:
     # Py < 3.1
     compat_http_client.HTTPResponse.getcode = lambda self: self.status
 
+
+# compat_urllib_HTTPError
 try:
     from urllib.error import HTTPError as compat_HTTPError
 except ImportError:  # Python 2
     from urllib2 import HTTPError as compat_HTTPError
 compat_urllib_HTTPError = compat_HTTPError
 
+
+# compat_urllib_request_urlretrieve
 try:
     from urllib.request import urlretrieve as compat_urlretrieve
 except ImportError:  # Python 2
     from urllib import urlretrieve as compat_urlretrieve
 compat_urllib_request_urlretrieve = compat_urlretrieve
 
+
+# compat_html_parser_HTMLParser, compat_html_parser_HTMLParseError
 try:
     from HTMLParser import (
         HTMLParser as compat_HTMLParser,
@@ -2432,22 +2464,33 @@ except ImportError:  # Python 3
         # HTMLParseError was deprecated in Python 3.3 and removed in
         # Python 3.5. Introducing dummy exception for Python >3.5 for compatible
         # and uniform cross-version exception handling
+
         class compat_HTMLParseError(Exception):
             pass
+
 compat_html_parser_HTMLParser = compat_HTMLParser
 compat_html_parser_HTMLParseError = compat_HTMLParseError
 
+
+# compat_subprocess_get_DEVNULL
 try:
     _DEVNULL = subprocess.DEVNULL
     compat_subprocess_get_DEVNULL = lambda: _DEVNULL
 except AttributeError:
     compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
 
+
+# compat_http_server
 try:
     import http.server as compat_http_server
 except ImportError:
     import BaseHTTPServer as compat_http_server
 
+
+# compat_urllib_parse_unquote_to_bytes,
+# compat_urllib_parse_unquote, compat_urllib_parse_unquote_plus,
+# compat_urllib_parse_urlencode,
+# compat_urllib_parse_parse_qs
 try:
     from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
     from urllib.parse import unquote as compat_urllib_parse_unquote
@@ -2598,6 +2641,8 @@ except ImportError:  # Python 2
 
 compat_urllib_parse_parse_qs = compat_parse_qs
 
+
+# compat_urllib_request_DataHandler
 try:
     from urllib.request import DataHandler as compat_urllib_request_DataHandler
 except ImportError:  # Python < 3.4
@@ -2632,16 +2677,20 @@ except ImportError:  # Python < 3.4
 
             return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)
 
+
+# compat_xml_etree_ElementTree_ParseError
 try:
     from xml.etree.ElementTree import ParseError as compat_xml_parse_error
 except ImportError:  # Python 2.6
     from xml.parsers.expat import ExpatError as compat_xml_parse_error
 compat_xml_etree_ElementTree_ParseError = compat_xml_parse_error
 
-etree = xml.etree.ElementTree
+
+# compat_xml_etree_ElementTree_Element
+_etree = xml.etree.ElementTree
 
 
-class _TreeBuilder(etree.TreeBuilder):
+class _TreeBuilder(_etree.TreeBuilder):
     def doctype(self, name, pubid, system):
         pass
 
@@ -2650,7 +2699,7 @@ try:
     # xml.etree.ElementTree.Element is a method in Python <=2.6 and
     # the following will crash with:
     #  TypeError: isinstance() arg 2 must be a class, type, or tuple of classes and types
-    isinstance(None, etree.Element)
+    isinstance(None, _etree.Element)
     from xml.etree.ElementTree import Element as compat_etree_Element
 except TypeError:  # Python <=2.6
     from xml.etree.ElementTree import _ElementInterface as compat_etree_Element
@@ -2658,12 +2707,12 @@ compat_xml_etree_ElementTree_Element = compat_etree_Element
 
 if sys.version_info[0] >= 3:
     def compat_etree_fromstring(text):
-        return etree.XML(text, parser=etree.XMLParser(target=_TreeBuilder()))
+        return _etree.XML(text, parser=_etree.XMLParser(target=_TreeBuilder()))
 else:
     # python 2.x tries to encode unicode strings with ascii (see the
     # XMLParser._fixtext method)
     try:
-        _etree_iter = etree.Element.iter
+        _etree_iter = _etree.Element.iter
     except AttributeError:  # Python <=2.6
         def _etree_iter(root):
             for el in root.findall('*'):
@@ -2675,27 +2724,29 @@ else:
     # 2.7 source
     def _XML(text, parser=None):
         if not parser:
-            parser = etree.XMLParser(target=_TreeBuilder())
+            parser = _etree.XMLParser(target=_TreeBuilder())
         parser.feed(text)
         return parser.close()
 
     def _element_factory(*args, **kwargs):
-        el = etree.Element(*args, **kwargs)
+        el = _etree.Element(*args, **kwargs)
         for k, v in el.items():
             if isinstance(v, bytes):
                 el.set(k, v.decode('utf-8'))
         return el
 
     def compat_etree_fromstring(text):
-        doc = _XML(text, parser=etree.XMLParser(target=_TreeBuilder(element_factory=_element_factory)))
+        doc = _XML(text, parser=_etree.XMLParser(target=_TreeBuilder(element_factory=_element_factory)))
         for el in _etree_iter(doc):
             if el.text is not None and isinstance(el.text, bytes):
                 el.text = el.text.decode('utf-8')
         return doc
 
-if hasattr(etree, 'register_namespace'):
+
-    compat_etree_register_namespace = etree.register_namespace
+# compat_xml_etree_register_namespace
-else:
+try:
+    compat_etree_register_namespace = _etree.register_namespace
+except AttributeError:
     def compat_etree_register_namespace(prefix, uri):
         """Register a namespace prefix.
         The registry is global, and any existing mapping for either the
@@ -2704,14 +2755,16 @@ else:
         attributes in this namespace will be serialized with prefix if possible.
         ValueError is raised if prefix is reserved or is invalid.
         """
-        if re.match(r"ns\d+$", prefix):
+        if re.match(r'ns\d+$', prefix):
-            raise ValueError("Prefix format reserved for internal use")
+            raise ValueError('Prefix format reserved for internal use')
-        for k, v in list(etree._namespace_map.items()):
+        for k, v in list(_etree._namespace_map.items()):
             if k == uri or v == prefix:
-                del etree._namespace_map[k]
+                del _etree._namespace_map[k]
-        etree._namespace_map[uri] = prefix
+        _etree._namespace_map[uri] = prefix
 compat_xml_etree_register_namespace = compat_etree_register_namespace
 
+
+# compat_xpath, compat_etree_iterfind
 if sys.version_info < (2, 7):
     # Here comes the crazy part: In 2.6, if the xpath is a unicode,
     # .//node does not match if a node is a direct child of . !
@@ -2898,7 +2951,6 @@ if sys.version_info < (2, 7):
         def __init__(self, root):
             self.root = root
 
-    ##
     # Generate all matching objects.
 
     def compat_etree_iterfind(elem, path, namespaces=None):
@@ -2933,13 +2985,15 @@ if sys.version_info < (2, 7):
 
 
 else:
-    compat_xpath = lambda xpath: xpath
     compat_etree_iterfind = lambda element, match: element.iterfind(match)
+    compat_xpath = _IDENTITY
 
 
+# compat_os_name
 compat_os_name = os._name if os.name == 'java' else os.name
 
 
+# compat_shlex_quote
 if compat_os_name == 'nt':
     def compat_shlex_quote(s):
         return s if re.match(r'^[-_\w./]+$', s) else '"%s"' % s.replace('"', '\\"')
@@ -2954,6 +3008,7 @@ else:
                 return "'" + s.replace("'", "'\"'\"'") + "'"
 
 
+# compat_shlex.split
 try:
     args = shlex.split('中文')
     assert (isinstance(args, list)
@@ -2969,6 +3024,7 @@ except (AssertionError, UnicodeEncodeError):
         return list(map(lambda s: s.decode('utf-8'), shlex.split(s, comments, posix)))
 
 
+# compat_ord
 def compat_ord(c):
     if isinstance(c, int):
         return c
@@ -2976,6 +3032,7 @@ def compat_ord(c):
         return ord(c)
 
 
+# compat_getenv, compat_os_path_expanduser, compat_setenv
 if sys.version_info >= (3, 0):
     compat_getenv = os.getenv
     compat_expanduser = os.path.expanduser
@@ -3063,6 +3120,7 @@ else:
 compat_os_path_expanduser = compat_expanduser
 
 
+# compat_os_path_realpath
 if compat_os_name == 'nt' and sys.version_info < (3, 8):
     # os.path.realpath on Windows does not follow symbolic links
     # prior to Python 3.8 (see https://bugs.python.org/issue9949)
@@ -3076,6 +3134,7 @@ else:
 compat_os_path_realpath = compat_realpath
 
 
+# compat_print
 if sys.version_info < (3, 0):
     def compat_print(s):
         from .utils import preferredencoding
@@ -3086,6 +3145,7 @@ else:
         print(s)
 
 
+# compat_getpass_getpass
 if sys.version_info < (3, 0) and sys.platform == 'win32':
     def compat_getpass(prompt, *args, **kwargs):
         if isinstance(prompt, compat_str):
@@ -3098,22 +3158,22 @@ else:
 compat_getpass_getpass = compat_getpass
 
 
+# compat_input
 try:
     compat_input = raw_input
 except NameError:  # Python 3
     compat_input = input
 
 
+# compat_kwargs
 # Python < 2.6.5 require kwargs to be bytes
 try:
-    def _testfunc(x):
+    (lambda x: x)(**{'x': 0})
-        pass
-    _testfunc(**{'x': 0})
 except TypeError:
     def compat_kwargs(kwargs):
         return dict((bytes(k), v) for k, v in kwargs.items())
 else:
-    compat_kwargs = lambda kwargs: kwargs
+    compat_kwargs = _IDENTITY
 
 
 # compat_numeric_types
@@ -3132,6 +3192,8 @@ except NameError:  # Python 3
 # compat_int
 compat_int = compat_integer_types[-1]
 
+
+# compat_socket_create_connection
 if sys.version_info < (2, 7):
     def compat_socket_create_connection(address, timeout, source_address=None):
         host, port = address
@@ -3158,6 +3220,7 @@ else:
     compat_socket_create_connection = socket.create_connection
 
 
+# compat_contextlib_suppress
 try:
     from contextlib import suppress as compat_contextlib_suppress
 except ImportError:
@@ -3205,7 +3268,7 @@ except AttributeError:
 
 # Fix https://github.com/ytdl-org/youtube-dl/issues/4223
 # See http://bugs.python.org/issue9161 for what is broken
-def workaround_optparse_bug9161():
+def _workaround_optparse_bug9161():
     op = optparse.OptionParser()
     og = optparse.OptionGroup(op, 'foo')
     try:
@@ -3224,9 +3287,10 @@ def workaround_optparse_bug9161():
         optparse.OptionGroup.add_option = _compat_add_option
 
 
-if hasattr(shutil, 'get_terminal_size'):  # Python >= 3.3
+# compat_shutil_get_terminal_size
-    compat_get_terminal_size = shutil.get_terminal_size
+try:
-else:
+    from shutil import get_terminal_size as compat_get_terminal_size  # Python >= 3.3
+except ImportError:
     _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])
 
     def compat_get_terminal_size(fallback=(80, 24)):
@@ -3256,27 +3320,33 @@ else:
                 columns = _columns
             if lines is None or lines <= 0:
                 lines = _lines
+
         return _terminal_size(columns, lines)
 
+compat_shutil_get_terminal_size = compat_get_terminal_size
 
+
+# compat_itertools_count
 try:
-    itertools.count(start=0, step=1)
+    type(itertools.count(start=0, step=1))
     compat_itertools_count = itertools.count
-except TypeError:  # Python 2.6
+except TypeError:  # Python 2.6 lacks step
     def compat_itertools_count(start=0, step=1):
         while True:
             yield start
             start += step
 
 
+# compat_tokenize_tokenize
 if sys.version_info >= (3, 0):
     from tokenize import tokenize as compat_tokenize_tokenize
 else:
     from tokenize import generate_tokens as compat_tokenize_tokenize
 
 
+# compat_struct_pack, compat_struct_unpack, compat_Struct
 try:
-    struct.pack('!I', 0)
+    type(struct.pack('!I', 0))
 except TypeError:
     # In Python 2.6 and 2.7.x < 2.7.7, struct requires a bytes argument
     # See https://bugs.python.org/issue19099
@@ -3308,8 +3378,10 @@ else:
         compat_Struct = struct.Struct
 
 
-# compat_map/filter() returning an iterator, supposedly the
+# builtins returning an iterator
-# same versioning as for zip below
+
+# compat_map, compat_filter
+# supposedly the same versioning as for zip below
 try:
     from future_builtins import map as compat_map
 except ImportError:
@@ -3326,6 +3398,7 @@ except ImportError:
     except ImportError:
         compat_filter = filter
 
+# compat_zip
 try:
     from future_builtins import zip as compat_zip
 except ImportError:  # not 2.6+ or is 3.x
@@ -3335,6 +3408,7 @@ except ImportError:  # not 2.6+ or is 3.x
         compat_zip = zip
 
 
+# compat_itertools_zip_longest
 # method renamed between Py2/3
 try:
     from itertools import zip_longest as compat_itertools_zip_longest
@@ -3342,7 +3416,8 @@ except ImportError:
     from itertools import izip_longest as compat_itertools_zip_longest
 
 
-# new class in collections
+# compat_collections_chain_map
+# collections.ChainMap: new class
 try:
     from collections import ChainMap as compat_collections_chain_map
     # Py3.3's ChainMap is deficient
@@ -3398,19 +3473,22 @@ except ImportError:
         def new_child(self, m=None, **kwargs):
             m = m or {}
             m.update(kwargs)
-            return compat_collections_chain_map(m, *self.maps)
+            # support inheritance !
+            return type(self)(m, *self.maps)
 
         @property
         def parents(self):
-            return compat_collections_chain_map(*(self.maps[1:]))
+            return type(self)(*(self.maps[1:]))
 
 
+# compat_re_Pattern, compat_re_Match
 # Pythons disagree on the type of a pattern (RegexObject, _sre.SRE_Pattern, Pattern, ...?)
 compat_re_Pattern = type(re.compile(''))
 # and on the type of a match
 compat_re_Match = type(re.match('a', 'a'))
 
 
+# compat_base64_b64decode
 if sys.version_info < (3, 3):
     def compat_b64decode(s, *args, **kwargs):
         if isinstance(s, compat_str):
@@ -3422,6 +3500,7 @@ else:
 compat_base64_b64decode = compat_b64decode
 
 
+# compat_ctypes_WINFUNCTYPE
 if platform.python_implementation() == 'PyPy' and sys.pypy_version_info < (5, 4, 0):
     # PyPy2 prior to version 5.4.0 expects byte strings as Windows function
     # names, see the original PyPy issue [1] and the youtube-dl one [2].
@@ -3440,6 +3519,7 @@ else:
         return ctypes.WINFUNCTYPE(*args, **kwargs)
 
 
+# compat_open
 if sys.version_info < (3, 0):
     # open(file, mode='r', buffering=- 1, encoding=None, errors=None, newline=None, closefd=True) not: opener=None
     def compat_open(file_, *args, **kwargs):
@@ -3467,18 +3547,28 @@ except AttributeError:
     def compat_datetime_timedelta_total_seconds(td):
         return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6
 
+
 # optional decompression packages
+# compat_brotli
 # PyPi brotli package implements 'br' Content-Encoding
 try:
     import brotli as compat_brotli
 except ImportError:
     compat_brotli = None
+# compat_ncompress
 # PyPi ncompress package implements 'compress' Content-Encoding
 try:
     import ncompress as compat_ncompress
 except ImportError:
     compat_ncompress = None
 
+# compat_zstandard
+# PyPi zstandard package implements 'zstd' Content-Encoding (RFC 8878 7.2)
+try:
+    import zstandard as compat_zstandard
+except ImportError:
+    compat_zstandard = None
+
 
 legacy = [
     'compat_HTMLParseError',
@@ -3495,6 +3585,7 @@ legacy = [
     'compat_getpass',
     'compat_parse_qs',
     'compat_realpath',
+    'compat_shlex_split',
     'compat_urllib_parse_parse_qs',
     'compat_urllib_parse_unquote',
     'compat_urllib_parse_unquote_plus',
@@ -3508,8 +3599,6 @@ legacy = [
 
 
 __all__ = [
-    'compat_html_parser_HTMLParseError',
-    'compat_html_parser_HTMLParser',
     'compat_Struct',
     'compat_base64_b64decode',
     'compat_basestring',
@@ -3518,13 +3607,9 @@ __all__ = [
     'compat_chr',
     'compat_collections_abc',
     'compat_collections_chain_map',
-    'compat_datetime_timedelta_total_seconds',
-    'compat_http_cookiejar',
-    'compat_http_cookiejar_Cookie',
-    'compat_http_cookies',
-    'compat_http_cookies_SimpleCookie',
     'compat_contextlib_suppress',
     'compat_ctypes_WINFUNCTYPE',
+    'compat_datetime_timedelta_total_seconds',
     'compat_etree_fromstring',
     'compat_etree_iterfind',
     'compat_filter',
@@ -3533,6 +3618,12 @@ __all__ = [
     'compat_getpass_getpass',
     'compat_html_entities',
     'compat_html_entities_html5',
+    'compat_html_parser_HTMLParseError',
+    'compat_html_parser_HTMLParser',
+    'compat_http_cookiejar',
+    'compat_http_cookiejar_Cookie',
+    'compat_http_cookies',
+    'compat_http_cookies_SimpleCookie',
     'compat_http_client',
     'compat_http_server',
     'compat_input',
@@ -3555,7 +3646,7 @@ __all__ = [
     'compat_register_utf8',
     'compat_setenv',
     'compat_shlex_quote',
-    'compat_shlex_split',
+    'compat_shutil_get_terminal_size',
     'compat_socket_create_connection',
     'compat_str',
     'compat_struct_pack',
@@ -3575,5 +3666,5 @@ __all__ = [
     'compat_xml_etree_register_namespace',
     'compat_xpath',
     'compat_zip',
-    'workaround_optparse_bug9161',
+    'compat_zstandard',
 ]

youtube_dl/extractor/youtube.py

@@ -27,6 +27,7 @@ from ..compat import (
 )
 from ..jsinterp import JSInterpreter
 from ..utils import (
+    bug_reports_message,
     clean_html,
     dict_get,
     error_to_compat_str,
@@ -65,6 +66,7 @@ from ..utils import (
     url_or_none,
     urlencode_postdata,
     urljoin,
+    variadic,
 )
 
 
@@ -89,12 +91,12 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'INNERTUBE_CONTEXT': {
                 'client': {
                     'clientName': 'IOS',
-                    'clientVersion': '19.45.4',
+                    'clientVersion': '20.10.4',
                     'deviceMake': 'Apple',
                     'deviceModel': 'iPhone16,2',
-                    'userAgent': 'com.google.ios.youtube/19.45.4 (iPhone16,2; U; CPU iOS 18_1_0 like Mac OS X;)',
+                    'userAgent': 'com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X;)',
                     'osName': 'iPhone',
-                    'osVersion': '18.1.0.22B83',
+                    'osVersion': '18.3.2.22D82',
                 },
             },
             'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
@@ -107,7 +109,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'INNERTUBE_CONTEXT': {
                 'client': {
                     'clientName': 'MWEB',
-                    'clientVersion': '2.20241202.07.00',
+                    'clientVersion': '2.20250311.03.00',
                     # mweb previously did not require PO Token with this UA
                     'userAgent': 'Mozilla/5.0 (iPad; CPU OS 16_7_10 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1,gzip(gfe)',
                 },
@@ -120,7 +122,8 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'INNERTUBE_CONTEXT': {
                 'client': {
                     'clientName': 'TVHTML5',
-                    'clientVersion': '7.20241201.18.00',
+                    'clientVersion': '7.20250312.16.00',
+                    'userAgent': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/Version',
                 },
             },
             'INNERTUBE_CONTEXT_CLIENT_NAME': 7,
@@ -130,7 +133,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'INNERTUBE_CONTEXT': {
                 'client': {
                     'clientName': 'WEB',
-                    'clientVersion': '2.20241126.01.00',
+                    'clientVersion': '2.20250312.04.00',
                 },
             },
             'INNERTUBE_CONTEXT_CLIENT_NAME': 1,
@@ -460,6 +463,26 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'uploader': uploader,
         }
 
+    @staticmethod
+    def _extract_thumbnails(data, *path_list, **kw_final_key):
+        """
+        Extract thumbnails from thumbnails dict
+        @param path_list: path list to level that contains 'thumbnails' key
+        """
+        final_key = kw_final_key.get('final_key', 'thumbnails')
+
+        return traverse_obj(data, ((
+            tuple(variadic(path) + (final_key, Ellipsis)
+                  for path in path_list or [()])), {
+            'url': ('url', T(url_or_none),
+                    # Sometimes youtube gives a wrong thumbnail URL. See:
+                    # https://github.com/yt-dlp/yt-dlp/issues/233
+                    # https://github.com/ytdl-org/youtube-dl/issues/28023
+                    T(lambda u: update_url(u, query=None) if u and 'maxresdefault' in u else u)),
+            'height': ('height', T(int_or_none)),
+            'width': ('width', T(int_or_none)),
+        }, T(lambda t: t if t.get('url') else None)))
+
     def _search_results(self, query, params):
         data = {
             'context': {
@@ -669,7 +692,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         'invidious': '|'.join(_INVIDIOUS_SITES),
     }
     _PLAYER_INFO_RE = (
-        r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
+        r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})//(?:tv-)?player',
         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
     )
@@ -1829,11 +1852,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         if func_code:
             return jsi, player_id, func_code
+        return self._extract_n_function_code_jsi(video_id, jsi, player_id)
 
-        func_name = self._extract_n_function_name(jscode)
+    def _extract_n_function_code_jsi(self, video_id, jsi, player_id=None):
+
+        var_ay = self._search_regex(
+            r'(?:[;\s]|^)\s*(var\s*[\w$]+\s*=\s*"(?:\\"|[^"])+"\s*\.\s*split\("\W+"\))(?=\s*[,;])',
+            jsi.code, 'useful values', default='')
+
+        func_name = self._extract_n_function_name(jsi.code)
 
         func_code = jsi.extract_function_code(func_name)
+        if var_ay:
+            func_code = (func_code[0], ';\n'.join((var_ay, func_code[1])))
 
+        if player_id:
             self.cache.store('youtube-nsig', player_id, func_code)
         return jsi, player_id, func_code
 
@@ -3183,8 +3216,12 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             expected_type=txt_or_none)
 
     def _grid_entries(self, grid_renderer):
-        for item in grid_renderer['items']:
+        for item in traverse_obj(grid_renderer, ('items', Ellipsis, T(dict))):
-            if not isinstance(item, dict):
+            lockup_view_model = traverse_obj(item, ('lockupViewModel', T(dict)))
+            if lockup_view_model:
+                entry = self._extract_lockup_view_model(lockup_view_model)
+                if entry:
+                    yield entry
                 continue
             renderer = self._extract_grid_item_renderer(item)
             if not isinstance(renderer, dict):
@@ -3268,6 +3305,25 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                 continue
             yield self._extract_video(renderer)
 
+    def _extract_lockup_view_model(self, view_model):
+        content_id = view_model.get('contentId')
+        if not content_id:
+            return
+        content_type = view_model.get('contentType')
+        if content_type not in ('LOCKUP_CONTENT_TYPE_PLAYLIST', 'LOCKUP_CONTENT_TYPE_PODCAST'):
+            self.report_warning(
+                'Unsupported lockup view model content type "{0}"{1}'.format(content_type, bug_reports_message()), only_once=True)
+            return
+        return merge_dicts(self.url_result(
+            update_url_query('https://www.youtube.com/playlist', {'list': content_id}),
+            ie=YoutubeTabIE.ie_key(), video_id=content_id), {
+                'title': traverse_obj(view_model, (
+                    'metadata', 'lockupMetadataViewModel', 'title', 'content', T(compat_str))),
+                'thumbnails': self._extract_thumbnails(view_model, (
+                    'contentImage', 'collectionThumbnailViewModel', 'primaryThumbnail',
+                    'thumbnailViewModel', 'image'), final_key='sources'),
+        })
+
     def _video_entry(self, video_renderer):
         video_id = video_renderer.get('videoId')
         if video_id:

youtube_dl/jsinterp.py

@@ -1,10 +1,12 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import calendar
 import itertools
 import json
 import operator
 import re
+import time
 
 from functools import update_wrapper, wraps
 
@@ -12,8 +14,10 @@ from .utils import (
     error_to_compat_str,
     ExtractorError,
     float_or_none,
+    int_or_none,
     js_to_json,
     remove_quotes,
+    str_or_none,
     unified_timestamp,
     variadic,
     write_string,
@@ -150,6 +154,7 @@ def _js_to_primitive(v):
     )
 
 
+# more exact: yt-dlp/yt-dlp#12110
 def _js_toString(v):
     return (
         'undefined' if v is JS_Undefined
@@ -158,7 +163,7 @@ def _js_toString(v):
         else 'null' if v is None
         # bool <= int: do this first
         else ('false', 'true')[v] if isinstance(v, bool)
-        else '{0:.7f}'.format(v).rstrip('.0') if isinstance(v, compat_numeric_types)
+        else re.sub(r'(?<=\d)\.?0*$', '', '{0:.7f}'.format(v)) if isinstance(v, compat_numeric_types)
         else _js_to_primitive(v))
 
 
@@ -404,6 +409,7 @@ class JSInterpreter(object):
     class Exception(ExtractorError):
         def __init__(self, msg, *args, **kwargs):
             expr = kwargs.pop('expr', None)
+            msg = str_or_none(msg, default='"None"')
             if expr is not None:
                 msg = '{0} in: {1!r:.100}'.format(msg.rstrip(), expr)
             super(JSInterpreter.Exception, self).__init__(msg, *args, **kwargs)
@@ -431,6 +437,7 @@ class JSInterpreter(object):
                 flags, _ = self.regex_flags(flags)
             # First, avoid https://github.com/python/cpython/issues/74534
             self.__self = None
+            pattern_txt = str_or_none(pattern_txt) or '(?:)'
             self.__pattern_txt = pattern_txt.replace('[[', r'[\[')
             self.__flags = flags
 
@@ -475,6 +482,73 @@ class JSInterpreter(object):
                 flags |= cls.RE_FLAGS[ch]
             return flags, expr[idx + 1:]
 
+    class JS_Date(object):
+        _t = None
+
+        @staticmethod
+        def __ymd_etc(*args, **kw_is_utc):
+            # args: year, monthIndex, day, hours, minutes, seconds, milliseconds
+            is_utc = kw_is_utc.get('is_utc', False)
+
+            args = list(args[:7])
+            args += [0] * (9 - len(args))
+            args[1] += 1  # month 0..11 -> 1..12
+            ms = args[6]
+            for i in range(6, 9):
+                args[i] = -1  # don't know
+            if is_utc:
+                args[-1] = 1
+            # TODO: [MDN] When a segment overflows or underflows its expected
+            # range, it usually "carries over to" or "borrows from" the higher segment.
+            try:
+                mktime = calendar.timegm if is_utc else time.mktime
+                return mktime(time.struct_time(args)) * 1000 + ms
+            except (OverflowError, ValueError):
+                return None
+
+        @classmethod
+        def UTC(cls, *args):
+            t = cls.__ymd_etc(*args, is_utc=True)
+            return _NaN if t is None else t
+
+        @staticmethod
+        def parse(date_str, **kw_is_raw):
+            is_raw = kw_is_raw.get('is_raw', False)
+
+            t = unified_timestamp(str_or_none(date_str), False)
+            return int(t * 1000) if t is not None else t if is_raw else _NaN
+
+        @staticmethod
+        def now(**kw_is_raw):
+            is_raw = kw_is_raw.get('is_raw', False)
+
+            t = time.time()
+            return int(t * 1000) if t is not None else t if is_raw else _NaN
+
+        def __init__(self, *args):
+            if not args:
+                args = [self.now(is_raw=True)]
+            if len(args) == 1:
+                if isinstance(args[0], JSInterpreter.JS_Date):
+                    self._t = int_or_none(args[0].valueOf(), default=None)
+                else:
+                    arg_type = _js_typeof(args[0])
+                    if arg_type == 'string':
+                        self._t = self.parse(args[0], is_raw=True)
+                    elif arg_type == 'number':
+                        self._t = int(args[0])
+            else:
+                self._t = self.__ymd_etc(*args)
+
+        def toString(self):
+            try:
+                return time.strftime('%a %b %0d %Y %H:%M:%S %Z%z', self._t).rstrip()
+            except TypeError:
+                return "Invalid Date"
+
+        def valueOf(self):
+            return _NaN if self._t is None else self._t
+
     @classmethod
     def __op_chars(cls):
         op_chars = set(';,[')
@@ -599,14 +673,15 @@ class JSInterpreter(object):
         except Exception as e:
             raise self.Exception('Failed to evaluate {left_val!r:.50} {op} {right_val!r:.50}'.format(**locals()), expr, cause=e)
 
-    def _index(self, obj, idx, allow_undefined=True):
+    def _index(self, obj, idx, allow_undefined=None):
         if idx == 'length' and isinstance(obj, list):
             return len(obj)
         try:
             return obj[int(idx)] if isinstance(obj, list) else obj[compat_str(idx)]
         except (TypeError, KeyError, IndexError) as e:
-            if allow_undefined:
+            # allow_undefined is None gives correct behaviour
-                # when is not allowed?
+            if allow_undefined or (
+                    allow_undefined is None and not isinstance(e, TypeError)):
                 return JS_Undefined
             raise self.Exception('Cannot get index {idx!r:.100}'.format(**locals()), expr=repr(obj), cause=e)
 
@@ -715,7 +790,7 @@ class JSInterpreter(object):
 
         new_kw, _, obj = expr.partition('new ')
         if not new_kw:
-            for klass, konstr in (('Date', lambda x: int(unified_timestamp(x, False) * 1000)),
+            for klass, konstr in (('Date', lambda *x: self.JS_Date(*x).valueOf()),
                                   ('RegExp', self.JS_RegExp),
                                   ('Error', self.Exception)):
                 if not obj.startswith(klass + '('):
@@ -1034,6 +1109,7 @@ class JSInterpreter(object):
                     'String': compat_str,
                     'Math': float,
                     'Array': list,
+                    'Date': self.JS_Date,
                 }
                 obj = local_vars.get(variable)
                 if obj in (JS_Undefined, None):
@@ -1086,6 +1162,8 @@ class JSInterpreter(object):
                         assertion(len(argvals) == 2, 'takes two arguments')
                         return argvals[0] ** argvals[1]
                     raise self.Exception('Unsupported Math method ' + member, expr=expr)
+                elif obj is self.JS_Date:
+                    return getattr(obj, member)(*argvals)
 
                 if member == 'split':
                     assertion(len(argvals) <= 2, 'takes at most two arguments')