Merge 3091daa834 into e0727e4ab6

Fixes 21792b88b7 (commitcomment-140705274), thx: vonProteus

test/test_downloader_external.py

@@ -18,6 +18,7 @@ from test.helper import (
 )
 from youtube_dl import YoutubeDL
 from youtube_dl.compat import (
+    compat_contextlib_suppress,
     compat_http_cookiejar_Cookie,
     compat_http_server,
     compat_kwargs,
@@ -35,6 +36,9 @@ from youtube_dl.downloader.external import (
     HttpieFD,
     WgetFD,
 )
+from youtube_dl.postprocessor import (
+    FFmpegPostProcessor,
+)
 import threading
 
 TEST_SIZE = 10 * 1024
@@ -227,7 +231,17 @@ class TestAria2cFD(unittest.TestCase):
             self.assertIn('--load-cookies=%s' % downloader._cookies_tempfile, cmd)
 
 
-@ifExternalFDAvailable(FFmpegFD)
+# Handle delegated availability
+def ifFFmpegFDAvailable(externalFD):
+    # raise SkipTest, or set False!
+    avail = ifExternalFDAvailable(externalFD) and False
+    with compat_contextlib_suppress(Exception):
+        avail = FFmpegPostProcessor(downloader=None).available
+    return unittest.skipUnless(
+        avail, externalFD.get_basename() + ' not found')
+
+
+@ifFFmpegFDAvailable(FFmpegFD)
 class TestFFmpegFD(unittest.TestCase):
     _args = []
 

youtube_dl/compat.py

@@ -2421,29 +2421,26 @@ except ImportError:  # Python 2
 compat_urllib_request_urlretrieve = compat_urlretrieve
 
 try:
+    from HTMLParser import (
+        HTMLParser as compat_HTMLParser,
+        HTMLParseError as compat_HTMLParseError)
+except ImportError:  # Python 3
     from html.parser import HTMLParser as compat_HTMLParser
-except ImportError:  # Python 2
-    from HTMLParser import HTMLParser as compat_HTMLParser
-compat_html_parser_HTMLParser = compat_HTMLParser
-
-try:  # Python 2
-    from HTMLParser import HTMLParseError as compat_HTMLParseError
-except ImportError:  # Python <3.4
     try:
         from html.parser import HTMLParseError as compat_HTMLParseError
     except ImportError:  # Python >3.4
-
+        # HTMLParseError was deprecated in Python 3.3 and removed in
-        # HTMLParseError has been deprecated in Python 3.3 and removed in
         # Python 3.5. Introducing dummy exception for Python >3.5 for compatible
         # and uniform cross-version exception handling
         class compat_HTMLParseError(Exception):
             pass
+compat_html_parser_HTMLParser = compat_HTMLParser
 compat_html_parser_HTMLParseError = compat_HTMLParseError
 
 try:
-    from subprocess import DEVNULL
+    _DEVNULL = subprocess.DEVNULL
-    compat_subprocess_get_DEVNULL = lambda: DEVNULL
+    compat_subprocess_get_DEVNULL = lambda: _DEVNULL
-except ImportError:
+except AttributeError:
     compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
 
 try:
@@ -2943,6 +2940,51 @@ else:
     compat_socket_create_connection = socket.create_connection
 
 
+try:
+    from contextlib import suppress as compat_contextlib_suppress
+except ImportError:
+    class compat_contextlib_suppress(object):
+        _exceptions = None
+
+        def __init__(self, *exceptions):
+            super(compat_contextlib_suppress, self).__init__()
+            # TODO: [Base]ExceptionGroup (3.12+)
+            self._exceptions = exceptions
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, exc_type, exc_val, exc_tb):
+            return exc_val is not None and isinstance(exc_val, self._exceptions or tuple())
+
+
+# subprocess.Popen context manager
+# avoids leaking handles if .communicate() is not called
+try:
+    _Popen = subprocess.Popen
+    # check for required context manager attributes
+    _Popen.__enter__ and _Popen.__exit__
+    compat_subprocess_Popen = _Popen
+except AttributeError:
+    # not a context manager - make one
+    from contextlib import contextmanager
+
+    @contextmanager
+    def compat_subprocess_Popen(*args, **kwargs):
+        popen = None
+        try:
+            popen = _Popen(*args, **kwargs)
+            yield popen
+        finally:
+            if popen:
+                for f in (popen.stdin, popen.stdout, popen.stderr):
+                    if f:
+                        # repeated .close() is OK, but just in case
+                        with compat_contextlib_suppress(EnvironmentError):
+                            f.close()
+                popen.wait()
+
+
 # Fix https://github.com/ytdl-org/youtube-dl/issues/4223
 # See http://bugs.python.org/issue9161 for what is broken
 def workaround_optparse_bug9161():
@@ -3263,6 +3305,7 @@ __all__ = [
     'compat_http_cookiejar_Cookie',
     'compat_http_cookies',
     'compat_http_cookies_SimpleCookie',
+    'compat_contextlib_suppress',
     'compat_ctypes_WINFUNCTYPE',
     'compat_etree_fromstring',
     'compat_filter',
@@ -3298,6 +3341,7 @@ __all__ = [
     'compat_struct_pack',
     'compat_struct_unpack',
     'compat_subprocess_get_DEVNULL',
+    'compat_subprocess_Popen',
     'compat_tokenize_tokenize',
     'compat_urllib_error',
     'compat_urllib_parse',

youtube_dl/downloader/external.py

@@ -11,8 +11,14 @@ from .common import FileDownloader
 from ..compat import (
     compat_setenv,
     compat_str,
+    compat_subprocess_Popen,
 )
-from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
+
+try:
+    from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
+except ImportError:
+    FFmpegPostProcessor = None
+
 from ..utils import (
     cli_option,
     cli_valueless_option,
@@ -361,13 +367,14 @@ class FFmpegFD(ExternalFD):
 
     @classmethod
     def available(cls):
-        return FFmpegPostProcessor().available
+        # actual availability can only be confirmed for an instance
+        return bool(FFmpegPostProcessor)
 
     def _call_downloader(self, tmpfilename, info_dict):
-        url = info_dict['url']
+        # `downloader` means the parent `YoutubeDL`
-        ffpp = FFmpegPostProcessor(downloader=self)
+        ffpp = FFmpegPostProcessor(downloader=self.ydl)
         if not ffpp.available:
-            self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
+            self.report_error('ffmpeg required for download but no ffmpeg (nor avconv) executable could be found. Please install one.')
             return False
         ffpp.check_version()
 
@@ -396,6 +403,7 @@ class FFmpegFD(ExternalFD):
         # if end_time:
         #     args += ['-t', compat_str(end_time - start_time)]
 
+        url = info_dict['url']
         cookies = self.ydl.cookiejar.get_cookies_for_url(url)
         if cookies:
             args.extend(['-cookies', ''.join(
@@ -483,21 +491,25 @@ class FFmpegFD(ExternalFD):
 
         self._debug_cmd(args)
 
-        proc = subprocess.Popen(args, stdin=subprocess.PIPE, env=env)
+        # From [1], a PIPE opened in Popen() should be closed, unless
-        try:
+        # .communicate() is called. Avoid leaking any PIPEs by using Popen
-            retval = proc.wait()
+        # as a context manager (newer Python 3.x and compat)
-        except BaseException as e:
+        # Fixes "Resource Warning" in test/test_downloader_external.py
-            # subprocess.run would send the SIGKILL signal to ffmpeg and the
+        # [1] https://devpress.csdn.net/python/62fde12d7e66823466192e48.html
-            # mp4 file couldn't be played, but if we ask ffmpeg to quit it
+        with compat_subprocess_Popen(args, stdin=subprocess.PIPE, env=env) as proc:
-            # produces a file that is playable (this is mostly useful for live
+            try:
-            # streams). Note that Windows is not affected and produces playable
+                retval = proc.wait()
-            # files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
+            except BaseException as e:
-            if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32':
+                # subprocess.run would send the SIGKILL signal to ffmpeg and the
-                process_communicate_or_kill(proc, b'q')
+                # mp4 file couldn't be played, but if we ask ffmpeg to quit it
-            else:
+                # produces a file that is playable (this is mostly useful for live
-                proc.kill()
+                # streams). Note that Windows is not affected and produces playable
-                proc.wait()
+                # files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
-            raise
+                if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32':
+                    process_communicate_or_kill(proc, b'q')
+                else:
+                    proc.kill()
+                raise
         return retval
 
 

youtube_dl/extractor/newgrounds.py

@@ -5,7 +5,6 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     extract_attributes,
-    int_or_none,
     parse_duration,
     parse_filesize,
     unified_timestamp,
@@ -27,15 +26,15 @@ class NewgroundsIE(InfoExtractor):
             'duration': 143,
         },
     }, {
-        'url': 'https://www.newgrounds.com/portal/view/673111',
+        'url': 'https://www.newgrounds.com/portal/view/850292',
-        'md5': '3394735822aab2478c31b1004fe5e5bc',
+        'md5': 'bb7cacf45e1b4d648e2dac2d79284d67',
         'info_dict': {
-            'id': '673111',
+            'id': '850292',
             'ext': 'mp4',
-            'title': 'Dancin',
+            'title': 'Timeless (2021)',
-            'uploader': 'Squirrelman82',
+            'uploader': 'Kevuhn',
-            'timestamp': 1460256780,
+            'timestamp': 1657896960,
-            'upload_date': '20160410',
+            'upload_date': '20220715',
         },
     }, {
         # source format unavailable, additional mp4 formats
@@ -44,7 +43,7 @@ class NewgroundsIE(InfoExtractor):
             'id': '689400',
             'ext': 'mp4',
             'title': 'ZTV News Episode 8',
-            'uploader': 'BennettTheSage',
+            'uploader': 'ZONE-SAMA',
             'timestamp': 1487965140,
             'upload_date': '20170224',
         },
@@ -58,40 +57,46 @@ class NewgroundsIE(InfoExtractor):
 
         webpage = self._download_webpage(url, media_id)
 
+        paths = url.split('/')
+        if paths[-3] == 'audio':
+            isAudio = True
+        else:
+            isAudio = False
+
+        if isAudio:
+            media_url = self._parse_json(self._search_regex(
+                r'"url"\s*:\s*("[^"]+"),', webpage, ''), media_id)
+
+            uploader = self._html_search_regex(
+                (r'(?s)<h4[^>]*>(.+?)</h4>.*?<em>\s*Author\s*</em>',
+                 r'(?:Author|Writer)\s*<a[^>]+>([^<]+)'), webpage, 'uploader',
+                fatal=False)
+
+            formats = [{
+                'url': media_url,
+                'format_id': 'source',
+                'quality': 1,
+            }]
+
+        else:
+            media_url = 'https://www.newgrounds.com/portal/video/' + media_id
+            media = self._download_json(media_url, media_id, headers={'X-Requested-With': 'XMLHttpRequest'})
+
+            uploader = media['author']
+
+            sources = media['sources']
+
+            formats = []
+            for source in sources:
+                for i in range(len(sources[source])):
+                    formats.append({
+                        'url': sources[source][i]['src'],
+                        'format_id': source,
+                        'height': int(source[:-2])  # 1080p -> 1080
+                    })
+
         title = self._html_search_regex(
             r'<title>([^>]+)</title>', webpage, 'title')
-
-        media_url = self._parse_json(self._search_regex(
-            r'"url"\s*:\s*("[^"]+"),', webpage, ''), media_id)
-
-        formats = [{
-            'url': media_url,
-            'format_id': 'source',
-            'quality': 1,
-        }]
-
-        max_resolution = int_or_none(self._search_regex(
-            r'max_resolution["\']\s*:\s*(\d+)', webpage, 'max resolution',
-            default=None))
-        if max_resolution:
-            url_base = media_url.rpartition('.')[0]
-            for resolution in (360, 720, 1080):
-                if resolution > max_resolution:
-                    break
-                formats.append({
-                    'url': '%s.%dp.mp4' % (url_base, resolution),
-                    'format_id': '%dp' % resolution,
-                    'height': resolution,
-                })
-
-        self._check_formats(formats, media_id)
-        self._sort_formats(formats)
-
-        uploader = self._html_search_regex(
-            (r'(?s)<h4[^>]*>(.+?)</h4>.*?<em>\s*Author\s*</em>',
-             r'(?:Author|Writer)\s*<a[^>]+>([^<]+)'), webpage, 'uploader',
-            fatal=False)
-
         timestamp = unified_timestamp(self._html_search_regex(
             (r'<dt>\s*Uploaded\s*</dt>\s*<dd>([^<]+</dd>\s*<dd>[^<]+)',
              r'<dt>\s*Uploaded\s*</dt>\s*<dd>([^<]+)'), webpage, 'timestamp',
@@ -99,7 +104,6 @@ class NewgroundsIE(InfoExtractor):
         duration = parse_duration(self._search_regex(
             r'(?s)<dd>\s*Song\s*</dd>\s*<dd>.+?</dd>\s*<dd>([^<]+)', webpage,
             'duration', default=None))
-
         filesize_approx = parse_filesize(self._html_search_regex(
             r'(?s)<dd>\s*Song\s*</dd>\s*<dd>(.+?)</dd>', webpage, 'filesize',
             default=None))
@@ -109,6 +113,9 @@ class NewgroundsIE(InfoExtractor):
         if '<dd>Song' in webpage:
             formats[0]['vcodec'] = 'none'
 
+        self._check_formats(formats, media_id)
+        self._sort_formats(formats)
+
         return {
             'id': media_id,
             'title': title,

youtube_dl/extractor/youtube.py

@@ -1647,10 +1647,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         except JSInterpreter.Exception as e:
             self.report_warning(
                 '%s (%s %s)' % (
-                    self.__ie_msg(
+                    'Unable to decode n-parameter: download likely to be throttled',
-                        'Unable to decode n-parameter: download likely to be throttled'),
                     error_to_compat_str(e),
-                    traceback.format_exc()))
+                    traceback.format_exc()),
+                video_id=video_id)
             return
 
         self.write_debug('Decrypted nsig {0} => {1}'.format(n, ret))

youtube_dl/postprocessor/ffmpeg.py

@@ -74,8 +74,11 @@ class FFmpegPostProcessor(PostProcessor):
         return FFmpegPostProcessor(downloader)._versions
 
     def _determine_executables(self):
-        programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
+        # ordered to match prefer_ffmpeg!
+        convs = ['ffmpeg', 'avconv']
+        probes = ['ffprobe', 'avprobe']
         prefer_ffmpeg = True
+        programs = convs + probes
 
         def get_ffmpeg_version(path):
             ver = get_exe_version(path, args=['-version'])
@@ -96,6 +99,7 @@ class FFmpegPostProcessor(PostProcessor):
 
         self._paths = None
         self._versions = None
+        location = None
         if self._downloader:
             prefer_ffmpeg = self._downloader.params.get('prefer_ffmpeg', True)
             location = self._downloader.params.get('ffmpeg_location')
@@ -118,33 +122,21 @@ class FFmpegPostProcessor(PostProcessor):
                     location = os.path.dirname(os.path.abspath(location))
                     if basename in ('ffmpeg', 'ffprobe'):
                         prefer_ffmpeg = True
+        self._paths = dict(
+            (p, p if location is None else os.path.join(location, p))
+            for p in programs)
+        self._versions = dict(
+            x for x in (
+                (p, get_ffmpeg_version(self._paths[p])) for p in programs)
+            if x[1] is not None)
 
-                self._paths = dict(
+        basenames = [None, None]
-                    (p, os.path.join(location, p)) for p in programs)
+        for i, progs in enumerate((convs, probes)):
-                self._versions = dict(
+            for p in progs[::-1 if prefer_ffmpeg is False else 1]:
-                    (p, get_ffmpeg_version(self._paths[p])) for p in programs)
+                if self._versions.get(p):
-        if self._versions is None:
+                    basenames[i] = p
-            self._versions = dict(
+                    break
-                (p, get_ffmpeg_version(p)) for p in programs)
+        self.basename, self.probe_basename = basenames
-            self._paths = dict((p, p) for p in programs)
-
-        if prefer_ffmpeg is False:
-            prefs = ('avconv', 'ffmpeg')
-        else:
-            prefs = ('ffmpeg', 'avconv')
-        for p in prefs:
-            if self._versions[p]:
-                self.basename = p
-                break
-
-        if prefer_ffmpeg is False:
-            prefs = ('avprobe', 'ffprobe')
-        else:
-            prefs = ('ffprobe', 'avprobe')
-        for p in prefs:
-            if self._versions[p]:
-                self.probe_basename = p
-                break
 
     @property
     def available(self):

youtube_dl/utils.py

@@ -45,6 +45,7 @@ from .compat import (
     compat_casefold,
     compat_chr,
     compat_collections_abc,
+    compat_contextlib_suppress,
     compat_cookiejar,
     compat_ctypes_WINFUNCTYPE,
     compat_datetime_timedelta_total_seconds,
@@ -1855,25 +1856,18 @@ def write_json_file(obj, fn):
     try:
         with tf:
             json.dump(obj, tf)
-        if sys.platform == 'win32':
+        with compat_contextlib_suppress(OSError):
-            # Need to remove existing file on Windows, else os.rename raises
+            if sys.platform == 'win32':
-            # WindowsError or FileExistsError.
+                # Need to remove existing file on Windows, else os.rename raises
-            try:
+                # WindowsError or FileExistsError.
                 os.unlink(fn)
-            except OSError:
-                pass
-        try:
             mask = os.umask(0)
             os.umask(mask)
             os.chmod(tf.name, 0o666 & ~mask)
-        except OSError:
-            pass
         os.rename(tf.name, fn)
     except Exception:
-        try:
+        with compat_contextlib_suppress(OSError):
             os.remove(tf.name)
-        except OSError:
-            pass
         raise
 
 
@@ -2033,14 +2027,13 @@ def extract_attributes(html_element):
     NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
     but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
     """
-    parser = HTMLAttributeParser()
+    ret = None
-    try:
+    # Older Python may throw HTMLParseError in case of malformed HTML (and on .close()!)
-        parser.feed(html_element)
+    with compat_contextlib_suppress(compat_HTMLParseError):
-        parser.close()
+        with contextlib.closing(HTMLAttributeParser()) as parser:
-    # Older Python may throw HTMLParseError in case of malformed HTML
+            parser.feed(html_element)
-    except compat_HTMLParseError:
+            ret = parser.attrs
-        pass
+    return ret or {}
-    return parser.attrs
 
 
 def clean_html(html):
@@ -2241,7 +2234,8 @@ def _htmlentity_transform(entity_with_semicolon):
             numstr = '0%s' % numstr
         else:
             base = 10
-        # See https://github.com/ytdl-org/youtube-dl/issues/7518
+        # See https://github.com/ytdl-org/youtube-dl/issues/7518\
+        # Also, weirdly, compat_contextlib_suppress fails here in 2.6
         try:
             return compat_chr(int(numstr, base))
         except ValueError:
@@ -2348,11 +2342,9 @@ def make_HTTPS_handler(params, **kwargs):
         # Some servers may (wrongly) reject requests if ALPN extension is not sent. See:
         # https://github.com/python/cpython/issues/85140
         # https://github.com/yt-dlp/yt-dlp/issues/3878
-        try:
+        with compat_contextlib_suppress(AttributeError, NotImplementedError):
+            # fails for Python < 2.7.10, not ssl.HAS_ALPN
             ctx.set_alpn_protocols(ALPN_PROTOCOLS)
-        except (AttributeError, NotImplementedError):
-            # Python < 2.7.10, not ssl.HAS_ALPN
-            pass
 
     opts_no_check_certificate = params.get('nocheckcertificate', False)
     if hasattr(ssl, 'create_default_context'):  # Python >= 3.4 or 2.7.9
@@ -2362,12 +2354,10 @@ def make_HTTPS_handler(params, **kwargs):
             context.check_hostname = False
             context.verify_mode = ssl.CERT_NONE
 
-        try:
+        with compat_contextlib_suppress(TypeError):
+            # Fails with Python 2.7.8 (create_default_context present
+            # but HTTPSHandler has no context=)
             return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
-        except TypeError:
-            # Python 2.7.8
-            # (create_default_context present but HTTPSHandler has no context=)
-            pass
 
     if sys.version_info < (3, 2):
         return YoutubeDLHTTPSHandler(params, **kwargs)
@@ -2381,15 +2371,24 @@ def make_HTTPS_handler(params, **kwargs):
         return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
 
 
-def bug_reports_message():
+def bug_reports_message(before=';'):
     if ytdl_is_updateable():
         update_cmd = 'type  youtube-dl -U  to update'
     else:
-        update_cmd = 'see  https://yt-dl.org/update  on how to update'
+        update_cmd = 'see  https://github.com/ytdl-org/youtube-dl/#user-content-installation  on how to update'
-    msg = '; please report this issue on https://yt-dl.org/bug .'
+
-    msg += ' Make sure you are using the latest version; %s.' % update_cmd
+    msg = (
-    msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
+        'please report this issue on https://github.com/ytdl-org/youtube-dl/issues ,'
-    return msg
+        ' using the appropriate issue template.'
+        ' Make sure you are using the latest version; %s.'
+        ' Be sure to call youtube-dl with the --verbose option and include the complete output.'
+    ) % update_cmd
+
+    before = (before or '').rstrip()
+    if not before or before.endswith(('.', '!', '?')):
+        msg = msg[0].title() + msg[1:]
+
+    return (before + ' ' if before else '') + msg
 
 
 class YoutubeDLError(Exception):
@@ -3176,12 +3175,10 @@ def parse_iso8601(date_str, delimiter='T', timezone=None):
     if timezone is None:
         timezone, date_str = extract_timezone(date_str)
 
-    try:
+    with compat_contextlib_suppress(ValueError):
         date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
         dt = datetime.datetime.strptime(date_str, date_format) - timezone
         return calendar.timegm(dt.timetuple())
-    except ValueError:
-        pass
 
 
 def date_formats(day_first=True):
@@ -3201,17 +3198,13 @@ def unified_strdate(date_str, day_first=True):
     _, date_str = extract_timezone(date_str)
 
     for expression in date_formats(day_first):
-        try:
+        with compat_contextlib_suppress(ValueError):
             upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
-        except ValueError:
-            pass
     if upload_date is None:
         timetuple = email.utils.parsedate_tz(date_str)
         if timetuple:
-            try:
+            with compat_contextlib_suppress(ValueError):
                 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
-            except ValueError:
-                pass
     if upload_date is not None:
         return compat_str(upload_date)
 
@@ -3240,11 +3233,9 @@ def unified_timestamp(date_str, day_first=True):
         date_str = m.group(1)
 
     for expression in date_formats(day_first):
-        try:
+        with compat_contextlib_suppress(ValueError):
             dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta)
             return calendar.timegm(dt.timetuple())
-        except ValueError:
-            pass
     timetuple = email.utils.parsedate_tz(date_str)
     if timetuple:
         return calendar.timegm(timetuple) + pm_delta * 3600 - compat_datetime_timedelta_total_seconds(timezone)