Merge 80d03d7e69c6462ad54d51f58a858b1af0ba18a6 into 3eb8d22ddb8982ca4fb56bb7a8d6517538bf14c6

* thx seproDev, yt-dlp/yt-dlp/pull/12694/commits/ae69e3c
* also add documenting comment in `InfoExtractor`

youtube_dl/extractor/bokecc.py

@@ -32,7 +32,7 @@ class BokeCCBaseIE(InfoExtractor):
 
 
 class BokeCCIE(BokeCCBaseIE):
-    _IE_DESC = 'CC视频'
+    IE_DESC = 'CC视频'
     _VALID_URL = r'https?://union\.bokecc\.com/playvideo\.bo\?(?P<query>.*)'
 
     _TESTS = [{

youtube_dl/extractor/cloudy.py

@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class CloudyIE(InfoExtractor):
-    _IE_DESC = 'cloudy.ec'
+    IE_DESC = 'cloudy.ec'
     _VALID_URL = r'https?://(?:www\.)?cloudy\.ec/(?:v/|embed\.php\?.*?\bid=)(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'https://www.cloudy.ec/v/af511e2527aac',

youtube_dl/extractor/common.py

@@ -422,6 +422,8 @@ class InfoExtractor(object):
     _GEO_COUNTRIES = None
     _GEO_IP_BLOCKS = None
     _WORKING = True
+    # supply this in public subclasses: used in supported sites list, etc
+    # IE_DESC = 'short description of IE'
 
     def __init__(self, downloader=None):
         """Constructor. Receives an optional downloader."""

youtube_dl/extractor/itv.py

@@ -35,15 +35,6 @@ from ..utils import (
 
 class ITVBaseIE(InfoExtractor):
 
-    def _search_nextjs_data(self, webpage, video_id, **kw):
-        transform_source = kw.pop('transform_source', None)
-        fatal = kw.pop('fatal', True)
-        return self._parse_json(
-            self._search_regex(
-                r'''<script\b[^>]+\bid=('|")__NEXT_DATA__\1[^>]*>(?P<js>[^<]+)</script>''',
-                webpage, 'next.js data', group='js', fatal=fatal, **kw),
-            video_id, transform_source=transform_source, fatal=fatal)
-
     def __handle_request_webpage_error(self, err, video_id=None, errnote=None, fatal=True):
         if errnote is False:
             return False
@@ -109,7 +100,9 @@ class ITVBaseIE(InfoExtractor):
 
 class ITVIE(ITVBaseIE):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/(?:(?P<w>watch)|hub)/[^/]+/(?(w)[\w-]+/)(?P<id>\w+)'
-    _IE_DESC = 'ITVX'
+    IE_DESC = 'ITVX'
+    _WORKING = False
+
     _TESTS = [{
         'note': 'Hub URLs redirect to ITVX',
         'url': 'https://www.itv.com/hub/liar/2a4547a0012',
@@ -270,7 +263,7 @@ class ITVIE(ITVBaseIE):
                 'ext': determine_ext(href, 'vtt'),
             })
 
-        next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default='{}')
+        next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default={})
         video_data.update(traverse_obj(next_data, ('props', 'pageProps', ('title', 'episode')), expected_type=dict)[0] or {})
         title = traverse_obj(video_data, 'headerTitle', 'episodeTitle')
         info = self._og_extract(webpage, require_title=not title)
@@ -323,7 +316,7 @@ class ITVIE(ITVBaseIE):
 
 class ITVBTCCIE(ITVBaseIE):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/(?!(?:watch|hub)/)(?:[^/]+/)+(?P<id>[^/?#&]+)'
-    _IE_DESC = 'ITV articles: News, British Touring Car Championship'
+    IE_DESC = 'ITV articles: News, British Touring Car Championship'
     _TESTS = [{
         'note': 'British Touring Car Championship',
         'url': 'https://www.itv.com/btcc/articles/btcc-2018-all-the-action-from-brands-hatch',

youtube_dl/extractor/sbs.py

@@ -2,32 +2,63 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_kwargs,
+    compat_str,
+)
 from ..utils import (
-    smuggle_url,
-    ExtractorError,
+    HEADRequest,
+    float_or_none,
+    int_or_none,
+    merge_dicts,
+    parse_duration,
+    parse_iso8601,
+    T,
+    traverse_obj,
+    update_url_query,
+    url_or_none,
 )
 
 
 class SBSIE(InfoExtractor):
     IE_DESC = 'sbs.com.au'
-    _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/(?:ondemand(?:/video/(?:single/)?|.*?\bplay=|/watch/)|news/(?:embeds/)?video/)(?P<id>[0-9]+)'
+    _VALID_URL = r'''(?x)
+        https?://(?:www\.)?sbs\.com\.au/(?:
+            ondemand(?:
+                /video/(?:single/)?|
+                /(?:movie|tv-program)/[^/]+/|
+                /(?:tv|news)-series/(?:[^/]+/){3}|
+                .*?\bplay=|/watch/
+            )|news/(?:embeds/)?video/
+        )(?P<id>[0-9]+)'''
+    _EMBED_REGEX = [r'''(?x)]
+            (?:
+                <meta\s+property="og:video"\s+content=|
+                <iframe[^>]+?src=
+            )
+            ("|\')(?P<url>https?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1''']
 
     _TESTS = [{
-        # Original URL is handled by the generic IE which finds the iframe:
-        # http://www.sbs.com.au/thefeed/blog/2014/08/21/dingo-conservation
+        # Exceptional unrestricted show for testing, thanks SBS,
+        # from an iframe of this page, handled by the generic IE, now 404:
+        # http://www.sbs.com.au/thefeed/blog/2014/08/21/dingo-conservation, but replaced by
+        # https://www.sbs.com.au/programs/video/320403011771/Dingo-Conservation-The-Feed
         'url': 'http://www.sbs.com.au/ondemand/video/single/320403011771/?source=drupal&vertical=thefeed',
-        'md5': '3150cf278965eeabb5b4cea1c963fe0a',
+        'md5': 'e49d0290cb4f40d893b8dfe760dce6b0',
         'info_dict': {
-            'id': '_rFBPRPO4pMR',
+            'id': '320403011771',  # formerly '_rFBPRPO4pMR', no longer found
             'ext': 'mp4',
             'title': 'Dingo Conservation (The Feed)',
             'description': 'md5:f250a9856fca50d22dec0b5b8015f8a5',
-            'thumbnail': r're:http://.*\.jpg',
+            'thumbnail': r're:https?://.*\.jpg',
             'duration': 308,
             'timestamp': 1408613220,
             'upload_date': '20140821',
             'uploader': 'SBSC',
+            'tags': 'mincount:10',
+            'categories': 'count:2',
         },
+        'expected_warnings': ['Unable to download JSON metadata'],
     }, {
         'url': 'http://www.sbs.com.au/ondemand/video/320403011771/Dingo-Conservation-The-Feed',
         'only_matching': True,
@@ -46,33 +77,146 @@ class SBSIE(InfoExtractor):
     }, {
         'url': 'https://www.sbs.com.au/ondemand/watch/1698704451971',
         'only_matching': True,
+    }, {
+        'url': 'https://www.sbs.com.au/ondemand/movie/coherence/1469404227931',
+        'only_matching': True,
+    }, {
+        'note': 'Live stream',
+        'url': 'https://www.sbs.com.au/ondemand/video/1726824003663/sbs-24x7-live-stream-nsw',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.sbs.com.au/ondemand/news-series/dateline/dateline-2022/dateline-s2022-ep26/2072245827515',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.sbs.com.au/ondemand/tv-series/the-handmaids-tale/season-5/the-handmaids-tale-s5-ep1/2065631811776',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.sbs.com.au/ondemand/tv-program/autun-romes-forgotten-sister/2116212803602',
+        'only_matching': True,
     }]
 
+    # change default entry_protocol kwarg for _extract_smil_formats()
+    # TODO: ..._and_subtitles()
+    def _extract_m3u8_formats(self, m3u8_url, video_id, *args, **kwargs):
+        # ext, entry_protocol, ...
+        entry_protocol = kwargs.get('entry_protocol')
+        if not entry_protocol and len(args) <= 1:
+            kwargs['entry_protocol'] = 'm3u8_native'
+            kwargs = compat_kwargs(kwargs)
+
+        return super(SBSIE, self)._extract_m3u8_formats(m3u8_url, video_id, *args, **kwargs)
+
+    _GEO_COUNTRIES = ['AU']
+    # naming for exportability
+    AUS_TV_PARENTAL_GUIDELINES = {
+        'P': 0,
+        'C': 7,
+        'G': 0,
+        'PG': 0,
+        'M': 14,
+        'MA15+': 15,
+        'AV15+': 15,
+        'MAV15+': 15,
+        'R18+': 18,
+        'NC': 0,  # not classified (unofficial, used by SBS)
+    }
+    _PLAYER_API = 'https://www.sbs.com.au/api/v3'
+    _CATALOGUE_API = 'https://catalogue.pr.sbsod.com/'
+    _VOD_BASE_URL = 'https://sbs-vod-prod-01.akamaized.net/'
+
+    def _call_api(self, video_id, path, query=None, data=None, headers=None, fatal=True):
+        return self._download_json(update_url_query(
+            self._CATALOGUE_API + path, query),
+            video_id, headers=headers or {}, fatal=fatal) or {}
+
+    def _get_smil_url(self, video_id):
+        return update_url_query(
+            self._PLAYER_API + 'video_smil', {'id': video_id})
+
+    def _get_player_data(self, video_id, headers=None, fatal=False):
+        return self._download_json(update_url_query(
+            self._PLAYER_API + 'video_stream', {'id': video_id, 'context': 'tv'}),
+            video_id, headers=headers or {}, fatal=fatal) or {}
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        player_params = self._download_json(
-            'http://www.sbs.com.au/api/video_pdkvars/id/%s?form=json' % video_id, video_id)
+        # get media links directly though later metadata may contain contentUrl
+        formats, subtitles = self._extract_smil_formats(  # self._extract_smil_formats_and_subtitles(
+            self._get_smil_url(video_id), video_id, fatal=False), {}
 
-        error = player_params.get('error')
-        if error:
-            error_message = 'Sorry, The video you are looking for does not exist.'
-            video_data = error.get('results') or {}
-            error_code = error.get('errorCode')
-            if error_code == 'ComingSoon':
-                error_message = '%s is not yet available.' % video_data.get('title', '')
-            elif error_code in ('Forbidden', 'intranetAccessOnly'):
-                error_message = 'Sorry, This video cannot be accessed via this website'
-            elif error_code == 'Expired':
-                error_message = 'Sorry, %s is no longer available.' % video_data.get('title', '')
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, error_message), expected=True)
+        if not formats:
+            urlh = self._request_webpage(
+                HEADRequest(self._VOD_BASE_URL), video_id,
+                note='Checking geo-restriction', fatal=False, expected_status=403)
+            if urlh:
+                error_reasons = urlh.headers.get_all('x-error-reason') or []
+                if 'geo-blocked' in error_reasons:
+                    self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
 
-        urls = player_params['releaseUrls']
-        theplatform_url = (urls.get('progressive') or urls.get('html')
-                           or urls.get('standard') or player_params['relatedItemsURL'])
+        self._sort_formats(formats)
 
-        return {
-            '_type': 'url_transparent',
-            'ie_key': 'ThePlatform',
-            'id': video_id,
-            'url': smuggle_url(self._proto_relative_url(theplatform_url), {'force_smil_url': True}),
+        # try for metadata from the same source
+        player_data = self._get_player_data(video_id, fatal=False)
+        media = traverse_obj(player_data, 'video_object', T(dict)) or {}
+
+        # get, or add, metadata from catalogue
+        media.update(self._call_api(video_id, 'mpx-media/' + video_id, fatal=not media))
+
+        def txt_or_none(s):
+            return (s.strip() or None) if isinstance(s, compat_str) else None
+
+        # expected_type fn for thumbs
+        def mk_thumb(t):
+            u = url_or_none(t.get('contentUrl'))
+            return u and {
+                'id': t.get('name'),
+                'url': u,
+                'width': int_or_none(t.get('width')),
+                'height': int_or_none(t.get('height')),
             }
+
+        # may be numeric or timecoded
+        def really_parse_duration(d):
+            result = float_or_none(d)
+            if result is None:
+                result = parse_duration(d)
+            return result
+
+        # For named episodes, use the catalogue's title to set episode, rather than generic 'Episode N'.
+        if traverse_obj(media, ('partOfSeries', T(dict))):
+            media['epName'] = traverse_obj(media, 'title')
+
+        str = txt_or_none  # instant compat
+        return merge_dicts({
+            'id': video_id,
+        }, traverse_obj(media, {
+            'title': ('name', T(str)),
+            'description': ('description', T(str)),
+            'channel': ('taxonomy', 'channel', 'name', T(str)),
+            'series': ((('partOfSeries', 'name'), 'seriesTitle'), T(str)),
+            'series_id': ((('partOfSeries', 'uuid'), 'seriesID'), T(str)),
+            'season_number': (('partOfSeries', None), 'seasonNumber', T(int_or_none)),
+            'episode': ('epName', T(str)),
+            'episode_number': ('episodeNumber', T(int_or_none)),
+            'timestamp': (('datePublished', ('publication', 'startDate')), T(parse_iso8601)),
+            'release_year': ('releaseYear', T(int_or_none)),
+            'duration': ('duration', T(really_parse_duration)),
+            'is_live': ('liveStream', T(bool)),
+            'age_limit': ('classificationID', 'contentRating',
+                          T(lambda x: self.AUS_TV_PARENTAL_GUIDELINES.get(x, '').upper() or None)),  # dict.get is unhashable in py3.7
+        }, get_all=False), traverse_obj(media, {
+            'categories': ((('genres', Ellipsis),
+                            ('taxonomy', ((('genre', 'subgenre'), Ellipsis, 'name'), 'useType'))),
+                           T(str)),
+            'tags': ((((('keywords',),
+                        ('consumerAdviceTexts', ('sbsSubCertification', 'consumerAdvice'))),
+                       Ellipsis),
+                     ('taxonomy', ('era', 'location', 'section', 'subject', 'theme'),
+                      Ellipsis, 'name')),
+                     T(str)),
+            'thumbnails': ('thumbnails', lambda _, v: v['contentUrl'], T(mk_thumb)),
+        }), {
+            'formats': formats,
+            'subtitles': subtitles,
+            'uploader': 'SBSC',
+        }, rev=True)

youtube_dl/extractor/senateisvp.py

@@ -47,7 +47,7 @@ class SenateISVPIE(InfoExtractor):
         ['vetaff', '76462', 'http://vetaff-f.akamaihd.net'],
         ['arch', '', 'http://ussenate-f.akamaihd.net/']
     ]
-    _IE_NAME = 'senate.gov'
+    IE_NAME = 'senate.gov'
     _VALID_URL = r'https?://(?:www\.)?senate\.gov/isvp/?\?(?P<qs>.+)'
     _TESTS = [{
         'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',

youtube_dl/jsinterp.py

@@ -686,6 +686,8 @@ class JSInterpreter(object):
             raise self.Exception('Cannot get index {idx!r:.100}'.format(**locals()), expr=repr(obj), cause=e)
 
     def _dump(self, obj, namespace):
+        if obj is JS_Undefined:
+            return 'undefined'
         try:
             return json.dumps(obj)
         except TypeError: