[YouPorn] Improve extraction

* use video ID from page
* detect unwatchable videos
* improve duration extraction
* fix count extraction and support large values
* detect and remnove SEO spam boilerplate description

test/test_age_restriction.py

@@ -47,7 +47,7 @@ class TestAgeRestriction(unittest.TestCase):
     def test_youporn(self):
         self._assert_restricted(
             'https://www.youporn.com/watch/16715086/sex-ed-in-detention-18-asmr/',
-            '16715086.mp4', 2, old_age=25)
+            '48260111.mp4', 2, old_age=25)
 
 
 if __name__ == '__main__':

test/test_download.py

@@ -10,7 +10,6 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from test.helper import (
     assertGreaterEqual,
-    assertLessEqual,
     expect_warnings,
     get_params,
     gettestcases,
@@ -123,10 +122,7 @@ def generator(test_case, tname):
         params['outtmpl'] = tname + '_' + params['outtmpl']
         if is_playlist and 'playlist' not in test_case:
             params.setdefault('extract_flat', 'in_playlist')
-            params.setdefault('playlistend',
-                              test_case['playlist_maxcount'] + 1
-                              if test_case.get('playlist_maxcount')
-                              else test_case.get('playlist_mincount'))
+            params.setdefault('playlistend', test_case.get('playlist_mincount'))
             params.setdefault('skip_download', True)
 
         ydl = YoutubeDL(params, auto_init=False)
@@ -194,14 +190,6 @@ def generator(test_case, tname):
                     'Expected at least %d in playlist %s, but got only %d' % (
                         test_case['playlist_mincount'], test_case['url'],
                         len(res_dict['entries'])))
-            if 'playlist_maxcount' in test_case:
-                assertLessEqual(
-                    self,
-                    len(res_dict['entries']),
-                    test_case['playlist_maxcount'],
-                    'Expected at most %d in playlist %s, but got %d' % (
-                        test_case['playlist_maxcount'], test_case['url'],
-                        len(res_dict['entries'])))
             if 'playlist_count' in test_case:
                 self.assertEqual(
                     len(res_dict['entries']),

test/test_downloader_external.py

@@ -18,7 +18,6 @@ from test.helper import (
 )
 from youtube_dl import YoutubeDL
 from youtube_dl.compat import (
-    compat_contextlib_suppress,
     compat_http_cookiejar_Cookie,
     compat_http_server,
     compat_kwargs,
@@ -36,9 +35,6 @@ from youtube_dl.downloader.external import (
     HttpieFD,
     WgetFD,
 )
-from youtube_dl.postprocessor import (
-    FFmpegPostProcessor,
-)
 import threading
 
 TEST_SIZE = 10 * 1024
@@ -231,17 +227,7 @@ class TestAria2cFD(unittest.TestCase):
             self.assertIn('--load-cookies=%s' % downloader._cookies_tempfile, cmd)
 
 
-# Handle delegated availability
-def ifFFmpegFDAvailable(externalFD):
-    # raise SkipTest, or set False!
-    avail = ifExternalFDAvailable(externalFD) and False
-    with compat_contextlib_suppress(Exception):
-        avail = FFmpegPostProcessor(downloader=None).available
-    return unittest.skipUnless(
-        avail, externalFD.get_basename() + ' not found')
-
-
-@ifFFmpegFDAvailable(FFmpegFD)
+@ifExternalFDAvailable(FFmpegFD)
 class TestFFmpegFD(unittest.TestCase):
     _args = []
 

youtube_dl/compat.py

@@ -2421,26 +2421,29 @@ except ImportError:  # Python 2
 compat_urllib_request_urlretrieve = compat_urlretrieve
 
 try:
-    from HTMLParser import (
-        HTMLParser as compat_HTMLParser,
-        HTMLParseError as compat_HTMLParseError)
-except ImportError:  # Python 3
     from html.parser import HTMLParser as compat_HTMLParser
+except ImportError:  # Python 2
+    from HTMLParser import HTMLParser as compat_HTMLParser
+compat_html_parser_HTMLParser = compat_HTMLParser
+
+try:  # Python 2
+    from HTMLParser import HTMLParseError as compat_HTMLParseError
+except ImportError:  # Python <3.4
     try:
         from html.parser import HTMLParseError as compat_HTMLParseError
     except ImportError:  # Python >3.4
-        # HTMLParseError was deprecated in Python 3.3 and removed in
+
+        # HTMLParseError has been deprecated in Python 3.3 and removed in
         # Python 3.5. Introducing dummy exception for Python >3.5 for compatible
         # and uniform cross-version exception handling
         class compat_HTMLParseError(Exception):
             pass
-compat_html_parser_HTMLParser = compat_HTMLParser
 compat_html_parser_HTMLParseError = compat_HTMLParseError
 
 try:
-    _DEVNULL = subprocess.DEVNULL
-    compat_subprocess_get_DEVNULL = lambda: _DEVNULL
-except AttributeError:
+    from subprocess import DEVNULL
+    compat_subprocess_get_DEVNULL = lambda: DEVNULL
+except ImportError:
     compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
 
 try:
@@ -2940,51 +2943,6 @@ else:
     compat_socket_create_connection = socket.create_connection
 
 
-try:
-    from contextlib import suppress as compat_contextlib_suppress
-except ImportError:
-    class compat_contextlib_suppress(object):
-        _exceptions = None
-
-        def __init__(self, *exceptions):
-            super(compat_contextlib_suppress, self).__init__()
-            # TODO: [Base]ExceptionGroup (3.12+)
-            self._exceptions = exceptions
-
-        def __enter__(self):
-            return self
-
-        def __exit__(self, exc_type, exc_val, exc_tb):
-            return exc_val is not None and isinstance(exc_val, self._exceptions or tuple())
-
-
-# subprocess.Popen context manager
-# avoids leaking handles if .communicate() is not called
-try:
-    _Popen = subprocess.Popen
-    # check for required context manager attributes
-    _Popen.__enter__ and _Popen.__exit__
-    compat_subprocess_Popen = _Popen
-except AttributeError:
-    # not a context manager - make one
-    from contextlib import contextmanager
-
-    @contextmanager
-    def compat_subprocess_Popen(*args, **kwargs):
-        popen = None
-        try:
-            popen = _Popen(*args, **kwargs)
-            yield popen
-        finally:
-            if popen:
-                for f in (popen.stdin, popen.stdout, popen.stderr):
-                    if f:
-                        # repeated .close() is OK, but just in case
-                        with compat_contextlib_suppress(EnvironmentError):
-                            f.close()
-                popen.wait()
-
-
 # Fix https://github.com/ytdl-org/youtube-dl/issues/4223
 # See http://bugs.python.org/issue9161 for what is broken
 def workaround_optparse_bug9161():
@@ -3305,7 +3263,6 @@ __all__ = [
     'compat_http_cookiejar_Cookie',
     'compat_http_cookies',
     'compat_http_cookies_SimpleCookie',
-    'compat_contextlib_suppress',
     'compat_ctypes_WINFUNCTYPE',
     'compat_etree_fromstring',
     'compat_filter',
@@ -3341,7 +3298,6 @@ __all__ = [
     'compat_struct_pack',
     'compat_struct_unpack',
     'compat_subprocess_get_DEVNULL',
-    'compat_subprocess_Popen',
     'compat_tokenize_tokenize',
     'compat_urllib_error',
     'compat_urllib_parse',

youtube_dl/downloader/external.py

@@ -11,14 +11,8 @@ from .common import FileDownloader
 from ..compat import (
     compat_setenv,
     compat_str,
-    compat_subprocess_Popen,
 )
-
-try:
-    from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
-except ImportError:
-    FFmpegPostProcessor = None
-
+from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
 from ..utils import (
     cli_option,
     cli_valueless_option,
@@ -367,14 +361,13 @@ class FFmpegFD(ExternalFD):
 
     @classmethod
     def available(cls):
-        # actual availability can only be confirmed for an instance
-        return bool(FFmpegPostProcessor)
+        return FFmpegPostProcessor().available
 
     def _call_downloader(self, tmpfilename, info_dict):
-        # `downloader` means the parent `YoutubeDL`
-        ffpp = FFmpegPostProcessor(downloader=self.ydl)
+        url = info_dict['url']
+        ffpp = FFmpegPostProcessor(downloader=self)
         if not ffpp.available:
-            self.report_error('ffmpeg required for download but no ffmpeg (nor avconv) executable could be found. Please install one.')
+            self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
             return False
         ffpp.check_version()
 
@@ -403,7 +396,6 @@ class FFmpegFD(ExternalFD):
         # if end_time:
         #     args += ['-t', compat_str(end_time - start_time)]
 
-        url = info_dict['url']
         cookies = self.ydl.cookiejar.get_cookies_for_url(url)
         if cookies:
             args.extend(['-cookies', ''.join(
@@ -491,25 +483,21 @@ class FFmpegFD(ExternalFD):
 
         self._debug_cmd(args)
 
-        # From [1], a PIPE opened in Popen() should be closed, unless
-        # .communicate() is called. Avoid leaking any PIPEs by using Popen
-        # as a context manager (newer Python 3.x and compat)
-        # Fixes "Resource Warning" in test/test_downloader_external.py
-        # [1] https://devpress.csdn.net/python/62fde12d7e66823466192e48.html
-        with compat_subprocess_Popen(args, stdin=subprocess.PIPE, env=env) as proc:
-            try:
-                retval = proc.wait()
-            except BaseException as e:
-                # subprocess.run would send the SIGKILL signal to ffmpeg and the
-                # mp4 file couldn't be played, but if we ask ffmpeg to quit it
-                # produces a file that is playable (this is mostly useful for live
-                # streams). Note that Windows is not affected and produces playable
-                # files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
-                if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32':
-                    process_communicate_or_kill(proc, b'q')
-                else:
-                    proc.kill()
-                raise
+        proc = subprocess.Popen(args, stdin=subprocess.PIPE, env=env)
+        try:
+            retval = proc.wait()
+        except BaseException as e:
+            # subprocess.run would send the SIGKILL signal to ffmpeg and the
+            # mp4 file couldn't be played, but if we ask ffmpeg to quit it
+            # produces a file that is playable (this is mostly useful for live
+            # streams). Note that Windows is not affected and produces playable
+            # files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
+            if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32':
+                process_communicate_or_kill(proc, b'q')
+            else:
+                proc.kill()
+                proc.wait()
+            raise
         return retval
 
 

youtube_dl/extractor/extractors.py

@@ -1653,15 +1653,7 @@ from .younow import (
     YouNowChannelIE,
     YouNowMomentIE,
 )
-from .youporn import (
-    YouPornIE,
-    YouPornCategoryIE,
-    YouPornChannelIE,
-    YouPornCollectionIE,
-    YouPornStarIE,
-    YouPornTagIE,
-    YouPornVideosIE,
-)
+from .youporn import YouPornIE
 from .yourporn import YourPornIE
 from .yourupload import YourUploadIE
 from .youtube import (

youtube_dl/extractor/vimeo.py

@@ -673,8 +673,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
             raise
 
         if '//player.vimeo.com/video/' in url:
-            config = self._search_json(
-                r'\b(?:playerC|c)onfig\s*=', webpage, 'info section', video_id)
+            config = self._parse_json(self._search_regex(
+                r'(?s)\b(?:playerC|c)onfig\s*=\s*({.+?})\s*[;\n]', webpage, 'info section'), video_id)
             if config.get('view') == 4:
                 config = self._verify_player_video_password(
                     redirect_url, video_id, headers)

youtube_dl/extractor/youporn.py

@@ -1,37 +1,26 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import itertools
 import re
-from time import sleep
 
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
     extract_attributes,
     ExtractorError,
-    get_element_by_class,
     get_element_by_id,
     int_or_none,
     merge_dicts,
     parse_count,
-    parse_qs,
     T,
     traverse_obj,
     unified_strdate,
     url_or_none,
-    urljoin,
 )
 
 
 class YouPornIE(InfoExtractor):
-    _VALID_URL = (
-        r'youporn:(?P<id>\d+)',
-        r'''(?x)
-            https?://(?:www\.)?youporn\.com/(?:watch|embed)/(?P<id>\d+)
-            (?:/(?:(?P<display_id>[^/?#&]+)/?)?)?(?:[#?]|$)
-    '''
-    )
+    _VALID_URL = r'https?://(?:www\.)?youporn\.com/(?:watch|embed)/(?P<id>\d+)(?:/(?P<display_id>[^/?#&]+))?'
     _EMBED_REGEX = [r'<iframe[^>]+\bsrc=["\'](?P<url>(?:https?:)?//(?:www\.)?youporn\.com/embed/\d+)']
     _TESTS = [{
         'url': 'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
@@ -87,7 +76,7 @@ class YouPornIE(InfoExtractor):
     }, {
         'url': 'https://www.youporn.com/watch/16290308/tinderspecial-trailer1/',
         'info_dict': {
-            'id': '16290308',
+            'id': '46949121',
             'age_limit': 18,
             'categories': [],
             'description': None,  # SEO spam using title removed
@@ -114,16 +103,16 @@ class YouPornIE(InfoExtractor):
         return list(yield_urls())
 
     def _real_extract(self, url):
-        # A different video ID (data-video-id) is hidden in the page but
-        # never seems to be used
-        video_id, display_id = self._match_valid_url(url).group('id', 'display_id')
-        url = 'http://www.youporn.com/watch/%s' % (video_id,)
+        # Actual video ID is hidden in the page
+        display_id = self._match_valid_url(url).group('id', 'display_id')
+        url = 'http://www.youporn.com/watch/%s' % (display_id[0],)
+        display_id = display_id[1] or display_id[0]
         webpage = self._download_webpage(
-            url, video_id, headers={'Cookie': 'age_verified=1'})
+            url, display_id, headers={'Cookie': 'age_verified=1'})
 
         watchable = self._search_regex(
             r'''(<div\s[^>]*\bid\s*=\s*('|")?watch-container(?(2)\2|(?!-)\b)[^>]*>)''',
-            webpage, 'watchability', default=None)
+            webpage, 'video id', default=None)
         if not watchable:
             msg = re.split(r'\s{4}', clean_html(get_element_by_id(
                 'mainContent', webpage)) or '')[0]
@@ -131,11 +120,15 @@ class YouPornIE(InfoExtractor):
                 ('%s says: %s' % (self.IE_NAME, msg))
                 if msg else 'Video unavailable: no reason found',
                 expected=True)
-        # internal ID ?
-        # video_id = extract_attributes(watchable).get('data-video-id')
+        video_id = extract_attributes(watchable).get('data-video-id')
+        if not video_id:
+            # last chance saloon
+            video_id = self._search_regex(
+                r'''\bcurrentVideoId\s*:\s*('|")(?P<id>\d+)\1''',
+                webpage, 'video id', group='id')
 
         playervars = self._search_json(
-            r'\bplayervars\s*:', webpage, 'playervars', video_id)
+            r'\bplayervars\s*:', webpage, 'playervars', display_id)
 
         def get_fmt(x):
             v_url = url_or_none(x.get('videoUrl'))
@@ -150,7 +143,7 @@ class YouPornIE(InfoExtractor):
             if f not in defs_by_format:
                 return []
             return self._download_json(
-                defs_by_format[f]['videoUrl'], video_id, '{0}-formats'.format(f))
+                defs_by_format[f]['videoUrl'], display_id, '{0}-formats'.format(f))
 
         formats = []
         # Try to extract only the actual master m3u8 first, avoiding the duplicate single resolution "master" m3u8s
@@ -254,441 +247,8 @@ class YouPornIE(InfoExtractor):
             'age_limit': age_limit,
             'formats': formats,
         })
-        # Remove promotional non-description
+        # Remove promotionalm non-description
         if result.get('description', '').startswith(
                 'Watch %s online' % (result['title'],)):
             del result['description']
         return result
-
-
-class YouPornListBase(InfoExtractor):
-    # pattern in '.title-text' element of page section containing videos
-    _PLAYLIST_TITLEBAR_RE = r'\s+[Vv]ideos\s*$'
-    _PAGE_RETRY_COUNT = 0  # ie, no retry
-    _PAGE_RETRY_DELAY = 2  # seconds
-
-    def _get_next_url(self, url, pl_id, html):
-        return urljoin(url, self._search_regex(
-            r'''<a\s[^>]*?\bhref\s*=\s*("|')(?P<url>(?:(?!\1)[^>])+)\1''',
-            get_element_by_id('next', html) or '', 'next page',
-            group='url', default=None))
-
-    @classmethod
-    def _get_title_from_slug(cls, title_slug):
-        return re.sub(r'[_-]', ' ', title_slug)
-
-    def _entries(self, url, pl_id, html=None, page_num=None):
-
-        # separates page sections
-        PLAYLIST_SECTION_RE = (
-            r'''<div\s[^>]*\bclass\s*=\s*('|")(?:[\w$-]+\s+|\s)*?title-bar(?:\s+[\w$-]+|\s)*\1[^>]*>'''
-        )
-        # contains video link
-        VIDEO_URL_RE = r'''(?x)
-            <div\s[^>]*\bdata-video-id\s*=\s*('|")\d+\1[^>]*>\s*
-            (?:<div\b[\s\S]+?</div>\s*)*
-            <a\s[^>]*\bhref\s*=\s*('|")(?P<url>(?:(?!\2)[^>])+)\2
-        '''
-
-        def yield_pages(url, html=html, page_num=page_num):
-            fatal = not html
-            for pnum in itertools.count(start=page_num or 1):
-                if not html:
-                    html = self._download_webpage(
-                        url, pl_id, note='Downloading page %d' % pnum,
-                        fatal=fatal)
-                if not html:
-                    break
-                fatal = False
-                yield (url, html, pnum)
-                # explicit page: extract just that page
-                if page_num is not None:
-                    break
-                next_url = self._get_next_url(url, pl_id, html)
-                if not next_url or next_url == url:
-                    break
-                url, html = next_url, None
-
-        def retry_page(msg, tries_left, page_data):
-            if tries_left <= 0:
-                return
-            self.report_warning(msg, pl_id)
-            sleep(self._PAGE_RETRY_DELAY)
-            return next(
-                yield_pages(page_data[0], page_num=page_data[2]), None)
-
-        def yield_entries(html):
-            for frag in re.split(PLAYLIST_SECTION_RE, html):
-                if not frag:
-                    continue
-                t_text = get_element_by_class('title-text', frag or '')
-                if not (t_text and re.search(self._PLAYLIST_TITLEBAR_RE, t_text)):
-                    continue
-                for m in re.finditer(VIDEO_URL_RE, frag):
-                    video_url = urljoin(url, m.group('url'))
-                    if video_url:
-                        yield self.url_result(video_url)
-
-        last_first_url = None
-        for page_data in yield_pages(url, html=html, page_num=page_num):
-            # page_data: url, html, page_num
-            first_url = None
-            tries_left = self._PAGE_RETRY_COUNT + 1
-            while tries_left > 0:
-                tries_left -= 1
-                for from_ in yield_entries(page_data[1]):
-                    # may get the same page twice instead of empty page
-                    # or (site bug) intead of actual next page
-                    if not first_url:
-                        first_url = from_['url']
-                        if first_url == last_first_url:
-                            # sometimes (/porntags/) the site serves the previous page
-                            # instead but may provide the correct page after a delay
-                            page_data = retry_page(
-                                'Retrying duplicate page...', tries_left, page_data)
-                            if page_data:
-                                first_url = None
-                                break
-                            continue
-                    yield from_
-                else:
-                    if not first_url and 'no-result-paragarph1' in page_data[1]:
-                        page_data = retry_page(
-                            'Retrying empty page...', tries_left, page_data)
-                        if page_data:
-                            continue
-                    else:
-                        # success/failure
-                        break
-            # may get an infinite (?) sequence of empty pages
-            if not first_url:
-                break
-            last_first_url = first_url
-
-    def _real_extract(self, url, html=None):
-        # exceptionally, id may be None
-        m_dict = self._match_valid_url(url).groupdict()
-        pl_id, page_type, sort = (m_dict.get(k) for k in ('id', 'type', 'sort'))
-
-        qs = parse_qs(url)
-        for q, v in qs.items():
-            if v:
-                qs[q] = v[-1]
-            else:
-                del qs[q]
-
-        base_id = pl_id or 'YouPorn'
-        title = self._get_title_from_slug(base_id)
-        if page_type:
-            title = '%s %s' % (page_type.capitalize(), title)
-        base_id = [base_id.lower()]
-        if sort is None:
-            title += ' videos'
-        else:
-            title = '%s videos by %s' % (title, re.sub(r'[_-]', ' ', sort))
-            base_id.append(sort)
-        if qs:
-            ps = ['%s=%s' % item for item in sorted(qs.items())]
-            title += ' (%s)' % ','.join(ps)
-            base_id.extend(ps)
-        pl_id = '/'.join(base_id)
-
-        return self.playlist_result(
-            self._entries(url, pl_id, html=html,
-                          page_num=int_or_none(qs.get('page'))),
-            playlist_id=pl_id, playlist_title=title)
-
-
-class YouPornCategoryIE(YouPornListBase):
-    IE_DESC = 'YouPorn category, with sorting, filtering and pagination'
-    _VALID_URL = r'''(?x)
-        https?://(?:www\.)?youporn\.com/
-        (?P<type>category)/(?P<id>[^/?#&]+)
-        (?:/(?P<sort>popular|views|rating|time|duration))?/?(?:[#?]|$)
-    '''
-    _TESTS = [{
-        'note': 'Full list with pagination',
-        'url': 'https://www.youporn.com/category/lingerie/popular/',
-        'info_dict': {
-            'id': 'lingerie/popular',
-            'title': 'Category lingerie videos by popular',
-        },
-        'playlist_mincount': 39,
-    }, {
-        'note': 'Filtered paginated list with single page result',
-        'url': 'https://www.youporn.com/category/lingerie/duration/?min_minutes=10',
-        'info_dict': {
-            'id': 'lingerie/duration/min_minutes=10',
-            'title': 'Category lingerie videos by duration (min_minutes=10)',
-        },
-        'playlist_maxcount': 30,
-    }, {
-        'note': 'Single page of full list',
-        'url': 'https://www.youporn.com/category/lingerie/popular?page=1',
-        'info_dict': {
-            'id': 'lingerie/popular/page=1',
-            'title': 'Category lingerie videos by popular (page=1)',
-        },
-        'playlist_count': 30,
-    }]
-
-
-class YouPornChannelIE(YouPornListBase):
-    IE_DESC = 'YouPorn channel, with sorting and pagination'
-    _VALID_URL = r'''(?x)
-        https?://(?:www\.)?youporn\.com/
-        (?P<type>channel)/(?P<id>[^/?#&]+)
-        (?:/(?P<sort>rating|views|duration))?/?(?:[#?]|$)
-    '''
-    _TESTS = [{
-        'note': 'Full list with pagination',
-        'url': 'https://www.youporn.com/channel/x-feeds/',
-        'info_dict': {
-            'id': 'x-feeds',
-            'title': 'Channel X-Feeds videos',
-        },
-        'playlist_mincount': 37,
-    }, {
-        'note': 'Single page of full list (no filters here)',
-        'url': 'https://www.youporn.com/channel/x-feeds/duration?page=1',
-        'info_dict': {
-            'id': 'x-feeds/duration/page=1',
-            'title': 'Channel X-Feeds videos by duration (page=1)',
-        },
-        'playlist_count': 24,
-    }]
-
-    @staticmethod
-    def _get_title_from_slug(title_slug):
-        return re.sub(r'_', ' ', title_slug).title()
-
-
-class YouPornCollectionIE(YouPornListBase):
-    IE_DESC = 'YouPorn collection (user playlist), with sorting and pagination'
-    _VALID_URL = r'''(?x)
-        https?://(?:www\.)?youporn\.com/
-        (?P<type>collection)s/videos/(?P<id>\d+)
-        (?:/(?P<sort>rating|views|time|duration))?/?(?:[#?]|$)
-    '''
-    _PLAYLIST_TITLEBAR_RE = r'^\s*Videos\s+in\s'
-    _TESTS = [{
-        'note': 'Full list with pagination',
-        'url': 'https://www.youporn.com/collections/videos/33044251/',
-        'info_dict': {
-            'id': '33044251',
-            'title': 'Collection Sexy Lips videos',
-            'uploader': 'ph-littlewillyb',
-        },
-        'playlist_mincount': 50,
-    }, {
-        'note': 'Single page of full list (no filters here)',
-        'url': 'https://www.youporn.com/collections/videos/33044251/time?page=1',
-        'info_dict': {
-            'id': '33044251/time/page=1',
-            'title': 'Collection Sexy Lips videos by time (page=1)',
-            'uploader': 'ph-littlewillyb',
-        },
-        'playlist_count': 20,
-    }]
-
-    def _real_extract(self, url):
-        pl_id = self._match_id(url)
-        html = self._download_webpage(url, pl_id)
-        playlist = super(YouPornCollectionIE, self)._real_extract(url, html=html)
-        infos = re.sub(r'\s+', ' ', clean_html(get_element_by_class(
-            'collection-infos', html)) or '')
-        title, uploader = self._search_regex(
-            r'^\s*Collection: (?P<title>.+?) \d+ VIDEOS \d+ VIEWS \d+ days LAST UPDATED From: (?P<uploader>[\w_-]+)',
-            infos, 'title/uploader', group=('title', 'uploader'), default=(None, None))
-
-        return merge_dicts({
-            'title': playlist['title'].replace(playlist['id'].split('/')[0], title),
-            'uploader': uploader,
-        }, playlist) if title else playlist
-
-
-class YouPornTagIE(YouPornListBase):
-    IE_DESC = 'YouPorn tag (porntags), with sorting, filtering and pagination'
-    _VALID_URL = r'''(?x)
-        https?://(?:www\.)?youporn\.com/
-        porn(?P<type>tag)s/(?P<id>[^/?#&]+)
-        (?:/(?P<sort>views|rating|time|duration))?/?(?:[#?]|$)
-    '''
-    _PLAYLIST_TITLEBAR_RE = r'^\s*Videos\s+tagged\s'
-    _PAGE_RETRY_COUNT = 1
-    _TESTS = [{
-        'note': 'Full list with pagination',
-        'url': 'https://www.youporn.com/porntags/austrian',
-        'info_dict': {
-            'id': 'austrian',
-            'title': 'Tag austrian videos',
-        },
-        'playlist_mincount': 35,
-        'expected_warnings': ['Retrying duplicate page'],
-    }, {
-        'note': 'Filtered paginated list with single page result',
-        'url': 'https://www.youporn.com/porntags/austrian/duration/?min_minutes=10',
-        'info_dict': {
-            'id': 'austrian/duration/min_minutes=10',
-            'title': 'Tag austrian videos by duration (min_minutes=10)',
-        },
-        # number of videos per page is (row x col) 2x3 + 6x4 + 2, or + 3,
-        # or more, varying with number of ads; let's set max as 9x4
-        # NB col 1 may not be shown in non-JS page with site CSS and zoom 100%
-        'playlist_maxcount': 32,
-        'expected_warnings': ['Retrying duplicate page', 'Retrying empty page'],
-    }, {
-        'note': 'Single page of full list',
-        'url': 'https://www.youporn.com/porntags/austrian/?page=1',
-        'info_dict': {
-            'id': 'austrian/page=1',
-            'title': 'Tag austrian videos (page=1)',
-        },
-        'playlist_mincount': 32,
-        'playlist_maxcount': 34,
-        'expected_warnings': ['Retrying duplicate page', 'Retrying empty page'],
-    }]
-
-    # YP tag navigation is broken, loses sort
-    def _get_next_url(self, url, pl_id, html):
-        next_url = super(YouPornTagIE, self)._get_next_url(url, pl_id, html)
-        if next_url:
-            n = self._match_valid_url(next_url)
-            if n:
-                s = n.groupdict().get('sort')
-            if s:
-                u = self._match_valid_url(url)
-                if u:
-                    u = u.groupdict().get('sort')
-                    if s and not u:
-                        n = n.end('sort')
-                        next_url = next_url[:n] + '/' + u + next_url[n:]
-        return next_url
-
-
-class YouPornStarIE(YouPornListBase):
-    IE_DESC = 'YouPorn Pornstar, with description, sorting and pagination'
-    _VALID_URL = r'''(?x)
-        https?://(?:www\.)?youporn\.com/
-        (?P<type>pornstar)/(?P<id>[^/?#&]+)
-        (?:/(?P<sort>rating|views|duration))?/?(?:[#?]|$)
-    '''
-    _PLAYLIST_TITLEBAR_RE = r'^\s*Videos\s+[fF]eaturing\s'
-    _TESTS = [{
-        'note': 'Full list with pagination',
-        'url': 'https://www.youporn.com/pornstar/daynia/',
-        'info_dict': {
-            'id': 'daynia',
-            'title': 'Pornstar Daynia videos',
-            'description': r're:Daynia Rank \d+ Videos \d+ Views [\d,.]+ .+ Subscribers \d+',
-        },
-        'playlist_mincount': 45,
-    }, {
-        'note': 'Single page of full list (no filters here)',
-        'url': 'https://www.youporn.com/pornstar/daynia/?page=1',
-        'info_dict': {
-            'id': 'daynia/page=1',
-            'title': 'Pornstar Daynia videos (page=1)',
-            'description': 're:.{180,}',
-        },
-        'playlist_count': 26,
-    }]
-
-    @staticmethod
-    def _get_title_from_slug(title_slug):
-        return re.sub(r'_', ' ', title_slug).title()
-
-    def _real_extract(self, url):
-        pl_id = self._match_id(url)
-        html = self._download_webpage(url, pl_id)
-        playlist = super(YouPornStarIE, self)._real_extract(url, html=html)
-        INFO_ELEMENT_RE = r'''(?x)
-            <div\s[^>]*\bclass\s*=\s*('|")(?:[\w$-]+\s+|\s)*?pornstar-info-wrapper(?:\s+[\w$-]+|\s)*\1[^>]*>
-            (?P<info>[\s\S]+?)(?:</div>\s*){6,}
-        '''
-
-        infos = self._search_regex(INFO_ELEMENT_RE, html, 'infos', group='info', default='')
-        if infos:
-            infos = re.sub(
-                r'(?:\s*nl=nl)+\s*', ' ',
-                re.sub(r'(?u)\s+', ' ', clean_html(
-                    re.sub('\n', 'nl=nl', infos)))).replace('ribe Subsc', '')
-
-        return merge_dicts({
-            'description': infos.strip() or None,
-        }, playlist)
-
-
-class YouPornVideosIE(YouPornListBase):
-    IE_DESC = 'YouPorn video (browse) playlists, with sorting, filtering and pagination'
-    _VALID_URL = r'''(?x)
-        https?://(?:www\.)?youporn\.com/
-            (?:(?P<id>browse)/)?
-            (?P<sort>(?(id)
-                (?:duration|rating|time|views)|
-                (?:most_(?:favou?rit|view)ed|recommended|top_rated)?))
-            (?:[/#?]|$)
-    '''
-    _PLAYLIST_TITLEBAR_RE = r'\s+(?:[Vv]ideos|VIDEOS)\s*$'
-    _TESTS = [{
-        'note': 'Full list with pagination (too long for test)',
-        'url': 'https://www.youporn.com/',
-        'info_dict': {
-            'id': 'youporn',
-            'title': 'YouPorn videos',
-        },
-        'only_matching': True,
-    }, {
-        'note': 'Full list with pagination (too long for test)',
-        'url': 'https://www.youporn.com/recommended',
-        'info_dict': {
-            'id': 'youporn/recommended',
-            'title': 'YouPorn videos by recommended',
-        },
-        'only_matching': True,
-    }, {
-        'note': 'Full list with pagination (too long for test)',
-        'url': 'https://www.youporn.com/top_rated',
-        'info_dict': {
-            'id': 'youporn/top_rated',
-            'title': 'YouPorn videos by top rated',
-        },
-        'only_matching': True,
-    }, {
-        'note': 'Full list with pagination (too long for test)',
-        'url': 'https://www.youporn.com/browse/time',
-        'info_dict': {
-            'id': 'browse/time',
-            'title': 'YouPorn videos by time',
-        },
-        'only_matching': True,
-    }, {
-        'note': 'Filtered paginated list with single page result',
-        'url': 'https://www.youporn.com/most_favorited/?res=VR&max_minutes=2',
-        'info_dict': {
-            'id': 'youporn/most_favorited/max_minutes=2/res=VR',
-            'title': 'YouPorn videos by most favorited (max_minutes=2,res=VR)',
-        },
-        'playlist_mincount': 10,
-        'playlist_maxcount': 28,
-    }, {
-        'note': 'Filtered paginated list with several pages',
-        'url': 'https://www.youporn.com/most_favorited/?res=VR&max_minutes=5',
-        'info_dict': {
-            'id': 'youporn/most_favorited/max_minutes=5/res=VR',
-            'title': 'YouPorn videos by most favorited (max_minutes=5,res=VR)',
-        },
-        'playlist_mincount': 45,
-    }, {
-        'note': 'Single page of full list',
-        'url': 'https://www.youporn.com/browse/time?page=1',
-        'info_dict': {
-            'id': 'browse/time/page=1',
-            'title': 'YouPorn videos by time (page=1)',
-        },
-        'playlist_count': 36,
-    }]
-
-    @staticmethod
-    def _get_title_from_slug(title_slug):
-        return 'YouPorn' if title_slug == 'browse' else title_slug

youtube_dl/extractor/youtube.py

@@ -1647,10 +1647,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         except JSInterpreter.Exception as e:
             self.report_warning(
                 '%s (%s %s)' % (
-                    'Unable to decode n-parameter: download likely to be throttled',
+                    self.__ie_msg(
+                        'Unable to decode n-parameter: download likely to be throttled'),
                     error_to_compat_str(e),
-                    traceback.format_exc()),
-                video_id=video_id)
+                    traceback.format_exc()))
             return
 
         self.write_debug('Decrypted nsig {0} => {1}'.format(n, ret))

youtube_dl/postprocessor/ffmpeg.py

@@ -74,11 +74,8 @@ class FFmpegPostProcessor(PostProcessor):
         return FFmpegPostProcessor(downloader)._versions
 
     def _determine_executables(self):
-        # ordered to match prefer_ffmpeg!
-        convs = ['ffmpeg', 'avconv']
-        probes = ['ffprobe', 'avprobe']
+        programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
         prefer_ffmpeg = True
-        programs = convs + probes
 
         def get_ffmpeg_version(path):
             ver = get_exe_version(path, args=['-version'])
@@ -99,7 +96,6 @@ class FFmpegPostProcessor(PostProcessor):
 
         self._paths = None
         self._versions = None
-        location = None
         if self._downloader:
             prefer_ffmpeg = self._downloader.params.get('prefer_ffmpeg', True)
             location = self._downloader.params.get('ffmpeg_location')
@@ -122,21 +118,33 @@ class FFmpegPostProcessor(PostProcessor):
                     location = os.path.dirname(os.path.abspath(location))
                     if basename in ('ffmpeg', 'ffprobe'):
                         prefer_ffmpeg = True
-        self._paths = dict(
-            (p, p if location is None else os.path.join(location, p))
-            for p in programs)
-        self._versions = dict(
-            x for x in (
-                (p, get_ffmpeg_version(self._paths[p])) for p in programs)
-            if x[1] is not None)
 
-        basenames = [None, None]
-        for i, progs in enumerate((convs, probes)):
-            for p in progs[::-1 if prefer_ffmpeg is False else 1]:
-                if self._versions.get(p):
-                    basenames[i] = p
-                    break
-        self.basename, self.probe_basename = basenames
+                self._paths = dict(
+                    (p, os.path.join(location, p)) for p in programs)
+                self._versions = dict(
+                    (p, get_ffmpeg_version(self._paths[p])) for p in programs)
+        if self._versions is None:
+            self._versions = dict(
+                (p, get_ffmpeg_version(p)) for p in programs)
+            self._paths = dict((p, p) for p in programs)
+
+        if prefer_ffmpeg is False:
+            prefs = ('avconv', 'ffmpeg')
+        else:
+            prefs = ('ffmpeg', 'avconv')
+        for p in prefs:
+            if self._versions[p]:
+                self.basename = p
+                break
+
+        if prefer_ffmpeg is False:
+            prefs = ('avprobe', 'ffprobe')
+        else:
+            prefs = ('ffprobe', 'avprobe')
+        for p in prefs:
+            if self._versions[p]:
+                self.probe_basename = p
+                break
 
     @property
     def available(self):

youtube_dl/utils.py

@@ -45,7 +45,6 @@ from .compat import (
     compat_casefold,
     compat_chr,
     compat_collections_abc,
-    compat_contextlib_suppress,
     compat_cookiejar,
     compat_ctypes_WINFUNCTYPE,
     compat_datetime_timedelta_total_seconds,
@@ -1856,18 +1855,25 @@ def write_json_file(obj, fn):
     try:
         with tf:
             json.dump(obj, tf)
-        with compat_contextlib_suppress(OSError):
-            if sys.platform == 'win32':
-                # Need to remove existing file on Windows, else os.rename raises
-                # WindowsError or FileExistsError.
+        if sys.platform == 'win32':
+            # Need to remove existing file on Windows, else os.rename raises
+            # WindowsError or FileExistsError.
+            try:
                 os.unlink(fn)
+            except OSError:
+                pass
+        try:
             mask = os.umask(0)
             os.umask(mask)
             os.chmod(tf.name, 0o666 & ~mask)
+        except OSError:
+            pass
         os.rename(tf.name, fn)
     except Exception:
-        with compat_contextlib_suppress(OSError):
+        try:
             os.remove(tf.name)
+        except OSError:
+            pass
         raise
 
 
@@ -2027,13 +2033,14 @@ def extract_attributes(html_element):
     NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
     but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
     """
-    ret = None
-    # Older Python may throw HTMLParseError in case of malformed HTML (and on .close()!)
-    with compat_contextlib_suppress(compat_HTMLParseError):
-        with contextlib.closing(HTMLAttributeParser()) as parser:
-            parser.feed(html_element)
-            ret = parser.attrs
-    return ret or {}
+    parser = HTMLAttributeParser()
+    try:
+        parser.feed(html_element)
+        parser.close()
+    # Older Python may throw HTMLParseError in case of malformed HTML
+    except compat_HTMLParseError:
+        pass
+    return parser.attrs
 
 
 def clean_html(html):
@@ -2234,8 +2241,7 @@ def _htmlentity_transform(entity_with_semicolon):
             numstr = '0%s' % numstr
         else:
             base = 10
-        # See https://github.com/ytdl-org/youtube-dl/issues/7518\
-        # Also, weirdly, compat_contextlib_suppress fails here in 2.6
+        # See https://github.com/ytdl-org/youtube-dl/issues/7518
         try:
             return compat_chr(int(numstr, base))
         except ValueError:
@@ -2342,9 +2348,11 @@ def make_HTTPS_handler(params, **kwargs):
         # Some servers may (wrongly) reject requests if ALPN extension is not sent. See:
         # https://github.com/python/cpython/issues/85140
         # https://github.com/yt-dlp/yt-dlp/issues/3878
-        with compat_contextlib_suppress(AttributeError, NotImplementedError):
-            # fails for Python < 2.7.10, not ssl.HAS_ALPN
+        try:
             ctx.set_alpn_protocols(ALPN_PROTOCOLS)
+        except (AttributeError, NotImplementedError):
+            # Python < 2.7.10, not ssl.HAS_ALPN
+            pass
 
     opts_no_check_certificate = params.get('nocheckcertificate', False)
     if hasattr(ssl, 'create_default_context'):  # Python >= 3.4 or 2.7.9
@@ -2354,10 +2362,12 @@ def make_HTTPS_handler(params, **kwargs):
             context.check_hostname = False
             context.verify_mode = ssl.CERT_NONE
 
-        with compat_contextlib_suppress(TypeError):
-            # Fails with Python 2.7.8 (create_default_context present
-            # but HTTPSHandler has no context=)
+        try:
             return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
+        except TypeError:
+            # Python 2.7.8
+            # (create_default_context present but HTTPSHandler has no context=)
+            pass
 
     if sys.version_info < (3, 2):
         return YoutubeDLHTTPSHandler(params, **kwargs)
@@ -2371,24 +2381,15 @@ def make_HTTPS_handler(params, **kwargs):
         return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
 
 
-def bug_reports_message(before=';'):
+def bug_reports_message():
     if ytdl_is_updateable():
         update_cmd = 'type  youtube-dl -U  to update'
     else:
-        update_cmd = 'see  https://github.com/ytdl-org/youtube-dl/#user-content-installation  on how to update'
-
-    msg = (
-        'please report this issue on https://github.com/ytdl-org/youtube-dl/issues ,'
-        ' using the appropriate issue template.'
-        ' Make sure you are using the latest version; %s.'
-        ' Be sure to call youtube-dl with the --verbose option and include the complete output.'
-    ) % update_cmd
-
-    before = (before or '').rstrip()
-    if not before or before.endswith(('.', '!', '?')):
-        msg = msg[0].title() + msg[1:]
-
-    return (before + ' ' if before else '') + msg
+        update_cmd = 'see  https://yt-dl.org/update  on how to update'
+    msg = '; please report this issue on https://yt-dl.org/bug .'
+    msg += ' Make sure you are using the latest version; %s.' % update_cmd
+    msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
+    return msg
 
 
 class YoutubeDLError(Exception):
@@ -3175,10 +3176,12 @@ def parse_iso8601(date_str, delimiter='T', timezone=None):
     if timezone is None:
         timezone, date_str = extract_timezone(date_str)
 
-    with compat_contextlib_suppress(ValueError):
+    try:
         date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
         dt = datetime.datetime.strptime(date_str, date_format) - timezone
         return calendar.timegm(dt.timetuple())
+    except ValueError:
+        pass
 
 
 def date_formats(day_first=True):
@@ -3198,13 +3201,17 @@ def unified_strdate(date_str, day_first=True):
     _, date_str = extract_timezone(date_str)
 
     for expression in date_formats(day_first):
-        with compat_contextlib_suppress(ValueError):
+        try:
             upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
+        except ValueError:
+            pass
     if upload_date is None:
         timetuple = email.utils.parsedate_tz(date_str)
         if timetuple:
-            with compat_contextlib_suppress(ValueError):
+            try:
                 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
+            except ValueError:
+                pass
     if upload_date is not None:
         return compat_str(upload_date)
 
@@ -3233,9 +3240,11 @@ def unified_timestamp(date_str, day_first=True):
         date_str = m.group(1)
 
     for expression in date_formats(day_first):
-        with compat_contextlib_suppress(ValueError):
+        try:
             dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta)
             return calendar.timegm(dt.timetuple())
+        except ValueError:
+            pass
     timetuple = email.utils.parsedate_tz(date_str)
     if timetuple:
         return calendar.timegm(timetuple) + pm_delta * 3600 - compat_datetime_timedelta_total_seconds(timezone)