Merge 0c1db866dbf9eaff9f3b934d5660d81d8d8eaf6d into 673277e510ebd996b62a2fcc76169bf3cce29910

* [YouTube] Extract playlist items from LOCKUP_VIEW_MODEL_...
* resolves #33073
* thx seproDev (yt-dlp/yt-dlp#11615)

Co-authored-by: sepro <sepro@sepr0.com>

test/formatselection/criteria_longside_shortside.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+# coding: utf-8
+"""Tests module for longside/shortside format selector."""
+
+from __future__ import unicode_literals
+
+# Allow direct execution
+if __name__ == '__main__':
+    import os
+    import sys
+    repo_dir = os.path.abspath(os.path.join(__file__, '../' * 3))
+    sys.path.insert(0, repo_dir)
+
+import unittest
+
+from youtube_dl.extractor import YoutubeIE
+from test.test_YoutubeDL import YDL, TEST_URL, _make_result
+
+
+default_common_video_properties = {
+    'url': TEST_URL,
+}
+
+
+def prepare_formats_info_dict(sizes, common={}):
+    """Convert sizes (id, width, height) to info_dict."""
+    def make_one_format(size):
+        d = default_common_video_properties.copy()
+        (d['format_id'], d['width'], d['height']) = size
+        d.update(common)
+        return d
+
+    info_dict = _make_result([make_one_format(size) for size in sizes])
+    return info_dict
+
+
+def pick_format_ids(sizes, criteria):
+    """Check which size(s) match the criteria. Return their IDs."""
+    ydl = YDL({'format': criteria})
+    yie = YoutubeIE(ydl)
+    info_dict = prepare_formats_info_dict(sizes)
+    yie._sort_formats(info_dict['formats'])
+    ydl.process_ie_result(info_dict.copy())
+    picked_ids = [info['format_id'] for info in ydl.downloaded_info_dicts]
+    return picked_ids
+
+
+class TestFormatSelection(unittest.TestCase):
+    """Tests class for longside/shortside format selector."""
+
+    def test_fmtsel_criteria_longside_shortside(self):
+        """Find largest video within upper limits."""
+        # Feature request: https://github.com/ytdl-org/youtube-dl/issues/30737
+
+        crit = {'tendency': '', 'short': '', 'long': ''}
+
+        def verify_fmtsel(sizes, want):
+            crit_all = crit['tendency'] + crit['short'] + crit['long']
+            picked_ids = pick_format_ids(sizes, crit_all)
+            self.assertEqual(','.join(picked_ids), want)
+
+        sizes_h = [
+            ('A', 256, 144,),
+            ('B', 426, 240,),
+            ('C', 640, 360,),
+            ('D', 854, 480,),
+        ]
+        sizes_v = [(id, h, w) for (id, w, h) in sizes_h]
+        self.assertEqual(sizes_v, [
+            # This list is non-authoritative, merely for readers' reference.
+            ('A', 144, 256,),
+            ('B', 240, 426,),
+            ('C', 360, 640,),
+            ('D', 480, 854,),
+        ])
+
+        # def size_by_id(sizes, id):
+        #     return next((s for s in sizes if s[0] == id))
+
+        def verify_all_shapes_same(expected_id):
+            verify_fmtsel(sizes_h, expected_id)
+            verify_fmtsel(sizes_v, expected_id)
+
+        # First, test with no criteria (still empty from initialization above):
+        verify_fmtsel(sizes_h, 'A')
+
+        crit['tendency'] = 'best'
+        verify_all_shapes_same('D')
+
+        crit['long'] = '[longside<=720]'
+        verify_all_shapes_same('C')
+
+        crit['long'] = '[longside<=420]'
+        verify_all_shapes_same('A')
+
+        def shortside_group_1(long, best):
+            crit['long'] = long
+            crit['short'] = '[shortside<=720]'
+            verify_all_shapes_same(best)
+
+            crit['short'] = '[shortside<=420]'
+            verify_all_shapes_same('C')
+
+            crit['short'] = '[shortside<=360]'
+            verify_all_shapes_same('C')
+
+            crit['short'] = '[shortside<360]'
+            verify_all_shapes_same('B')
+
+        shortside_group_1(long='', best='D')
+        shortside_group_1(long='[longside<=720]', best='C')
+
+
+if __name__ == '__main__':
+    unittest.main()

test/test_YoutubeDL.py

@@ -136,6 +136,11 @@ class TestFormatSelection(unittest.TestCase):
         ]
         info_dict = _make_result(formats)
 
+        ydl = YDL({'format': ''}) # no criteria => anything goes
+        ydl.process_ie_result(info_dict.copy())
+        downloaded = ydl.downloaded_info_dicts[0]
+        self.assertEqual(downloaded['format_id'], '35')
+
         ydl = YDL({'format': '20/47'})
         ydl.process_ie_result(info_dict.copy())
         downloaded = ydl.downloaded_info_dicts[0]

youtube_dl/YoutubeDL.py

@@ -361,6 +361,7 @@ class YoutubeDL(object):
         'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count',
         'average_rating', 'comment_count', 'age_limit',
         'start_time', 'end_time',
+        'longside', 'shortside',
         'chapter_number', 'season_number', 'episode_number',
         'track_number', 'disc_number', 'release_year',
         'playlist_index',
@@ -1227,7 +1228,7 @@ class YoutubeDL(object):
             '!=': operator.ne,
         }
         operator_rex = re.compile(r'''(?x)\s*
-            (?P<key>width|height|tbr|abr|vbr|asr|filesize|filesize_approx|fps)
+            (?P<key>width|height|shortside|longside|tbr|abr|vbr|asr|filesize|filesize_approx|fps)
             \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
             (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
             $
@@ -1309,6 +1310,9 @@ class YoutubeDL(object):
                 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
             return SyntaxError(message)
 
+        if not format_spec:
+            format_spec = 'worst'
+
         PICKFIRST = 'PICKFIRST'
         MERGE = 'MERGE'
         SINGLE = 'SINGLE'
@@ -1515,6 +1519,8 @@ class YoutubeDL(object):
                                                 formats_info[1].get('format_id')),
                         'width': formats_info[0].get('width'),
                         'height': formats_info[0].get('height'),
+                        'longside': formats_info[0].get('longside'),
+                        'shortside': formats_info[0].get('shortside'),
                         'resolution': formats_info[0].get('resolution'),
                         'fps': formats_info[0].get('fps'),
                         'vcodec': formats_info[0].get('vcodec'),
@@ -1666,6 +1672,17 @@ class YoutubeDL(object):
         sanitize_string_field(info_dict, 'id')
         sanitize_numeric_fields(info_dict)
 
+        def add_calculated_video_proprties(fmt):
+            if type(fmt) is not dict: return
+            dims = [fmt.get(side) for side in ('width', 'height',)]
+            dims = [n for n in dims if n is not None]
+            if len(dims):
+                fmt['shortside'] = min(iter(dims))
+                fmt['longside'] = max(iter(dims))
+
+        for fmt in [info_dict] + (info_dict.get('formats') or []):
+            add_calculated_video_proprties(fmt)
+
         if 'playlist' not in info_dict:
             # It isn't part of a playlist
             info_dict['playlist'] = None

youtube_dl/extractor/youtube.py

@@ -27,6 +27,7 @@ from ..compat import (
 )
 from ..jsinterp import JSInterpreter
 from ..utils import (
+    bug_reports_message,
     clean_html,
     dict_get,
     error_to_compat_str,
@@ -65,6 +66,7 @@ from ..utils import (
     url_or_none,
     urlencode_postdata,
     urljoin,
+    variadic,
 )
 
 
@@ -460,6 +462,26 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'uploader': uploader,
         }
 
+    @staticmethod
+    def _extract_thumbnails(data, *path_list, **kw_final_key):
+        """
+        Extract thumbnails from thumbnails dict
+        @param path_list: path list to level that contains 'thumbnails' key
+        """
+        final_key = kw_final_key.get('final_key', 'thumbnails')
+
+        return traverse_obj(data, ((
+            tuple(variadic(path) + (final_key, Ellipsis)
+                  for path in path_list or [()])), {
+            'url': ('url', T(url_or_none),
+                    # Sometimes youtube gives a wrong thumbnail URL. See:
+                    # https://github.com/yt-dlp/yt-dlp/issues/233
+                    # https://github.com/ytdl-org/youtube-dl/issues/28023
+                    T(lambda u: update_url(u, query=None) if u and 'maxresdefault' in u else u)),
+            'height': ('height', T(int_or_none)),
+            'width': ('width', T(int_or_none)),
+        }, T(lambda t: t if t.get('url') else None)))
+
     def _search_results(self, query, params):
         data = {
             'context': {
@@ -3183,8 +3205,12 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             expected_type=txt_or_none)
 
     def _grid_entries(self, grid_renderer):
-        for item in grid_renderer['items']:
-            if not isinstance(item, dict):
+        for item in traverse_obj(grid_renderer, ('items', Ellipsis, T(dict))):
+            lockup_view_model = traverse_obj(item, ('lockupViewModel', T(dict)))
+            if lockup_view_model:
+                entry = self._extract_lockup_view_model(lockup_view_model)
+                if entry:
+                    yield entry
                 continue
             renderer = self._extract_grid_item_renderer(item)
             if not isinstance(renderer, dict):
@@ -3268,6 +3294,25 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                 continue
             yield self._extract_video(renderer)
 
+    def _extract_lockup_view_model(self, view_model):
+        content_id = view_model.get('contentId')
+        if not content_id:
+            return
+        content_type = view_model.get('contentType')
+        if content_type not in ('LOCKUP_CONTENT_TYPE_PLAYLIST', 'LOCKUP_CONTENT_TYPE_PODCAST'):
+            self.report_warning(
+                'Unsupported lockup view model content type "{0}"{1}'.format(content_type, bug_reports_message()), only_once=True)
+            return
+        return merge_dicts(self.url_result(
+            update_url_query('https://www.youtube.com/playlist', {'list': content_id}),
+            ie=YoutubeTabIE.ie_key(), video_id=content_id), {
+                'title': traverse_obj(view_model, (
+                    'metadata', 'lockupMetadataViewModel', 'title', 'content', T(compat_str))),
+                'thumbnails': self._extract_thumbnails(view_model, (
+                    'contentImage', 'collectionThumbnailViewModel', 'primaryThumbnail',
+                    'thumbnailViewModel', 'image'), final_key='sources'),
+        })
+
     def _video_entry(self, video_renderer):
         video_id = video_renderer.get('videoId')
         if video_id: