Merge 80d03d7e69 into c5098961b0

youtube_dl/extractor/sbs.py

@@ -13,6 +13,7 @@ from ..utils import (
     merge_dicts,
     parse_duration,
     parse_iso8601,
+    T,
     traverse_obj,
     update_url_query,
     url_or_none,
@@ -35,7 +36,7 @@ class SBSIE(InfoExtractor):
                 <meta\s+property="og:video"\s+content=|
                 <iframe[^>]+?src=
             )
-            (["\'])(?P<url>https?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1''']
+            ("|\')(?P<url>https?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1''']
 
     _TESTS = [{
         # Exceptional unrestricted show for testing, thanks SBS,
@@ -54,8 +55,8 @@ class SBSIE(InfoExtractor):
             'timestamp': 1408613220,
             'upload_date': '20140821',
             'uploader': 'SBSC',
-            'tags': None,
+            'tags': 'mincount:10',
-            'categories': None,
+            'categories': 'count:2',
         },
         'expected_warnings': ['Unable to download JSON metadata'],
     }, {
@@ -94,18 +95,14 @@ class SBSIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    # change default entry_protocol kwarg for _extract_smil_formats()
+    # TODO: ..._and_subtitles()
     def _extract_m3u8_formats(self, m3u8_url, video_id, *args, **kwargs):
-        # ext, entry_protocol, preference, m3u8_id, note, errnote, fatal,
+        # ext, entry_protocol, ...
-        # live, data, headers, query
+        entry_protocol = kwargs.get('entry_protocol')
-        entry_protocol = args[1] if len(args) > 1 else kwargs.get('entry_protocol')
+        if not entry_protocol and len(args) <= 1:
-        if not entry_protocol:
+            kwargs['entry_protocol'] = 'm3u8_native'
-            entry_protocol = 'm3u8_native'
+            kwargs = compat_kwargs(kwargs)
-            if len(args) > 1:
-                args = list(args)
-                args[1] = entry_protocol
-            else:
-                kwargs['entry_protocol'] = entry_protocol
-                kwargs = compat_kwargs(kwargs)
 
         return super(SBSIE, self)._extract_m3u8_formats(m3u8_url, video_id, *args, **kwargs)
 
@@ -144,8 +141,8 @@ class SBSIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         # get media links directly though later metadata may contain contentUrl
-        smil_url = self._get_smil_url(video_id)
+        formats, subtitles = self._extract_smil_formats(  # self._extract_smil_formats_and_subtitles(
-        formats = self._extract_smil_formats(smil_url, video_id, fatal=False) or []
+            self._get_smil_url(video_id), video_id, fatal=False), {}
 
         if not formats:
             urlh = self._request_webpage(
@@ -160,16 +157,16 @@ class SBSIE(InfoExtractor):
 
         # try for metadata from the same source
         player_data = self._get_player_data(video_id, fatal=False)
-        media = traverse_obj(player_data, 'video_object', expected_type=dict) or {}
+        media = traverse_obj(player_data, 'video_object', T(dict)) or {}
+
         # get, or add, metadata from catalogue
         media.update(self._call_api(video_id, 'mpx-media/' + video_id, fatal=not media))
 
-        # utils candidate for use with traverse_obj()
         def txt_or_none(s):
             return (s.strip() or None) if isinstance(s, compat_str) else None
 
         # expected_type fn for thumbs
-        def xlate_thumb(t):
+        def mk_thumb(t):
             u = url_or_none(t.get('contentUrl'))
             return u and {
                 'id': t.get('name'),
@@ -185,51 +182,41 @@ class SBSIE(InfoExtractor):
                 result = parse_duration(d)
             return result
 
-        def traverse_media(*args, **kwargs):
-            nkwargs = None
-            if 'expected_type' not in kwargs:
-                kwargs['expected_type'] = txt_or_none
-                nkwargs = kwargs
-            if 'get_all' not in kwargs:
-                kwargs['get_all'] = False
-                nkwargs = kwargs
-            if nkwargs:
-                kwargs = compat_kwargs(nkwargs)
-            return traverse_obj(media, *args, **kwargs)
-
         # For named episodes, use the catalogue's title to set episode, rather than generic 'Episode N'.
-        if traverse_media('partOfSeries', expected_type=dict):
+        if traverse_obj(media, ('partOfSeries', T(dict))):
-            media['epName'] = traverse_media('title')
+            media['epName'] = traverse_obj(media, 'title')
 
-        return merge_dicts(*reversed(({
+        str = txt_or_none  # instant compat
+        return merge_dicts({
             'id': video_id,
-        }, dict((k, traverse_media(v)) for k, v in {
+        }, traverse_obj(media, {
-            'title': 'name',
+            'title': ('name', T(str)),
-            'description': 'description',
+            'description': ('description', T(str)),
-            'channel': ('taxonomy', 'channel', 'name'),
+            'channel': ('taxonomy', 'channel', 'name', T(str)),
-            'series': ((('partOfSeries', 'name'), 'seriesTitle')),
+            'series': ((('partOfSeries', 'name'), 'seriesTitle'), T(str)),
-            'series_id': ((('partOfSeries', 'uuid'), 'seriesID')),
+            'series_id': ((('partOfSeries', 'uuid'), 'seriesID'), T(str)),
-            'episode': 'epName',
+            'season_number': (('partOfSeries', None), 'seasonNumber', T(int_or_none)),
-        }.items()), {
+            'episode': ('epName', T(str)),
-            'season_number': traverse_media((('partOfSeries', None), 'seasonNumber'), expected_type=int_or_none),
+            'episode_number': ('episodeNumber', T(int_or_none)),
-            'episode_number': traverse_media('episodeNumber', expected_type=int_or_none),
+            'timestamp': (('datePublished', ('publication', 'startDate')), T(parse_iso8601)),
-            'timestamp': traverse_media('datePublished', ('publication', 'startDate'),
+            'release_year': ('releaseYear', T(int_or_none)),
-                                        expected_type=parse_iso8601),
+            'duration': ('duration', T(really_parse_duration)),
-            'release_year': traverse_media('releaseYear', expected_type=int_or_none),
+            'is_live': ('liveStream', T(bool)),
-            'duration': traverse_media('duration', expected_type=really_parse_duration),
+            'age_limit': ('classificationID', 'contentRating',
-            'is_live': traverse_media('liveStream', expected_type=bool),
+                          T(lambda x: self.AUS_TV_PARENTAL_GUIDELINES.get(x, '').upper() or None)),  # dict.get is unhashable in py3.7
-            'age_limit': self.AUS_TV_PARENTAL_GUIDELINES.get(traverse_media(
+        }, get_all=False), traverse_obj(media, {
-                'classificationID', 'contentRating', default='').upper()),
+            'categories': ((('genres', Ellipsis),
-            'categories': traverse_media(
+                            ('taxonomy', ((('genre', 'subgenre'), Ellipsis, 'name'), 'useType'))),
-                ('genres', Ellipsis), ('taxonomy', ('genre', 'subgenre'), 'name'),
+                           T(str)),
-                get_all=True) or None,
+            'tags': ((((('keywords',),
-            'tags': traverse_media(
+                        ('consumerAdviceTexts', ('sbsSubCertification', 'consumerAdvice'))),
-                (('consumerAdviceTexts', ('sbsSubCertification', 'consumerAdvice')), Ellipsis),
+                       Ellipsis),
-                get_all=True) or None,
+                     ('taxonomy', ('era', 'location', 'section', 'subject', 'theme'),
-            'thumbnails': traverse_media(('thumbnails', Ellipsis),
+                      Ellipsis, 'name')),
-                                         expected_type=xlate_thumb, get_all=True),
+                     T(str)),
+            'thumbnails': ('thumbnails', lambda _, v: v['contentUrl'], T(mk_thumb)),
+        }), {
             'formats': formats,
-            # TODO: _extract_smil_formats_and_subtitles()
+            'subtitles': subtitles,
-            # 'subtitles': subtitles,
             'uploader': 'SBSC',
-        })))
+        }, rev=True)