Merge efad43c197775eb3ef30b048c8d7b83c1ccbb57c into 3eb8d22ddb8982ca4fb56bb7a8d6517538bf14c6

* thx seproDev, yt-dlp/yt-dlp/pull/12694/commits/ae69e3c
* also add documenting comment in `InfoExtractor`

youtube_dl/extractor/bokecc.py

@@ -32,7 +32,7 @@ class BokeCCBaseIE(InfoExtractor):
 
 
 class BokeCCIE(BokeCCBaseIE):
-    _IE_DESC = 'CC视频'
+    IE_DESC = 'CC视频'
     _VALID_URL = r'https?://union\.bokecc\.com/playvideo\.bo\?(?P<query>.*)'
 
     _TESTS = [{

youtube_dl/extractor/cloudy.py

@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class CloudyIE(InfoExtractor):
-    _IE_DESC = 'cloudy.ec'
+    IE_DESC = 'cloudy.ec'
     _VALID_URL = r'https?://(?:www\.)?cloudy\.ec/(?:v/|embed\.php\?.*?\bid=)(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'https://www.cloudy.ec/v/af511e2527aac',

youtube_dl/extractor/common.py

@@ -422,6 +422,8 @@ class InfoExtractor(object):
     _GEO_COUNTRIES = None
     _GEO_IP_BLOCKS = None
     _WORKING = True
+    # supply this in public subclasses: used in supported sites list, etc
+    # IE_DESC = 'short description of IE'
 
     def __init__(self, downloader=None):
         """Constructor. Receives an optional downloader."""

youtube_dl/extractor/itv.py

@@ -35,15 +35,6 @@ from ..utils import (
 
 class ITVBaseIE(InfoExtractor):
 
-    def _search_nextjs_data(self, webpage, video_id, **kw):
-        transform_source = kw.pop('transform_source', None)
-        fatal = kw.pop('fatal', True)
-        return self._parse_json(
-            self._search_regex(
-                r'''<script\b[^>]+\bid=('|")__NEXT_DATA__\1[^>]*>(?P<js>[^<]+)</script>''',
-                webpage, 'next.js data', group='js', fatal=fatal, **kw),
-            video_id, transform_source=transform_source, fatal=fatal)
-
     def __handle_request_webpage_error(self, err, video_id=None, errnote=None, fatal=True):
         if errnote is False:
             return False
@@ -109,7 +100,9 @@ class ITVBaseIE(InfoExtractor):
 
 class ITVIE(ITVBaseIE):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/(?:(?P<w>watch)|hub)/[^/]+/(?(w)[\w-]+/)(?P<id>\w+)'
-    _IE_DESC = 'ITVX'
+    IE_DESC = 'ITVX'
+    _WORKING = False
+
     _TESTS = [{
         'note': 'Hub URLs redirect to ITVX',
         'url': 'https://www.itv.com/hub/liar/2a4547a0012',
@@ -270,7 +263,7 @@ class ITVIE(ITVBaseIE):
                 'ext': determine_ext(href, 'vtt'),
             })
 
-        next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default='{}')
+        next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default={})
         video_data.update(traverse_obj(next_data, ('props', 'pageProps', ('title', 'episode')), expected_type=dict)[0] or {})
         title = traverse_obj(video_data, 'headerTitle', 'episodeTitle')
         info = self._og_extract(webpage, require_title=not title)
@@ -323,7 +316,7 @@ class ITVIE(ITVBaseIE):
 
 class ITVBTCCIE(ITVBaseIE):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/(?!(?:watch|hub)/)(?:[^/]+/)+(?P<id>[^/?#&]+)'
-    _IE_DESC = 'ITV articles: News, British Touring Car Championship'
+    IE_DESC = 'ITV articles: News, British Touring Car Championship'
     _TESTS = [{
         'note': 'British Touring Car Championship',
         'url': 'https://www.itv.com/btcc/articles/btcc-2018-all-the-action-from-brands-hatch',

youtube_dl/extractor/joj.py

@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import re
+import json
 
 from .common import InfoExtractor
 from ..compat import compat_str
@@ -21,7 +22,8 @@ class JojIE(InfoExtractor):
                     (?P<id>[^/?#^]+)
                 '''
     _TESTS = [{
-        'url': 'https://media.joj.sk/embed/a388ec4c-6019-4a4a-9312-b1bee194e932',
+        'url':
+        'https://media.joj.sk/embed/a388ec4c-6019-4a4a-9312-b1bee194e932',
         'info_dict': {
             'id': 'a388ec4c-6019-4a4a-9312-b1bee194e932',
             'ext': 'mp4',
@@ -64,16 +66,20 @@ class JojIE(InfoExtractor):
                 r'(?s)(?:src|bitrates)\s*=\s*({.+?});', webpage, 'bitrates',
                 default='{}'),
             video_id, transform_source=js_to_json, fatal=False)
+        quality = self._search_regex(
+            r'var qualityLabels = ({.*?});', webpage, 'quality', default=None)
+        quality = json.loads(quality)
 
         formats = []
         for format_url in try_get(bitrates, lambda x: x['mp4'], list) or []:
             if isinstance(format_url, compat_str):
                 height = self._search_regex(
-                    r'(\d+)[pP]\.', format_url, 'height', default=None)
+                    r'-(\d+)p?\.', format_url, 'height', default=None)
                 formats.append({
                     'url': format_url,
                     'format_id': '%sp' % height if height else None,
                     'height': int(height),
+                    'resolution': quality.get(compat_str(height)+'p')
                 })
         if not formats:
             playlist = self._download_xml(
@@ -104,5 +110,5 @@ class JojIE(InfoExtractor):
             'title': title,
             'thumbnail': thumbnail,
             'duration': duration,
-            'formats': formats,
+            'formats': formats
         }

youtube_dl/extractor/markiza.py

@@ -1,21 +1,23 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+# update 15.01.2022 jastrab
 import re
+import json
 
 from .common import InfoExtractor
-from ..compat import compat_str
 from ..utils import (
     orderedSet,
-    parse_duration,
-    try_get,
+    url_or_none,
+    determine_ext
 )
 
 
 class MarkizaIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?videoarchiv\.markiza\.sk/(?:video/(?:[^/]+/)*|embed/)(?P<id>\d+)(?:[_/]|$)'
+    _VALID_URL = r'https:\/\/(?:www\.)?videoarchiv\.markiza\.sk\/(?:video\/(?:[^\/]+\/)*|embed\/)epizoda\/(?P<id>\d+)(?:[\_\/\-]|$)'
     _TESTS = [{
-        'url': 'http://videoarchiv.markiza.sk/video/oteckovia/84723_oteckovia-109',
+        'url': 'http://videoarchiv.markiza.sk/video/oteckovia/\
+            84723_oteckovia-109',
         'md5': 'ada4e9fad038abeed971843aa028c7b0',
         'info_dict': {
             'id': '139078',
@@ -26,54 +28,89 @@ class MarkizaIE(InfoExtractor):
             'duration': 2760,
         },
     }, {
-        'url': 'http://videoarchiv.markiza.sk/video/televizne-noviny/televizne-noviny/85430_televizne-noviny',
+        'url': ' https://videoarchiv.markiza.sk/video/laska-na-prenajom/epizoda/58779-seria-1-epizoda-14',
         'info_dict': {
             'id': '85430',
             'title': 'Televízne noviny',
         },
         'playlist_count': 23,
     }, {
-        'url': 'http://videoarchiv.markiza.sk/video/oteckovia/84723',
+        'url': 'https://videoarchiv.markiza.sk/video/oteckovia/84723',
         'only_matching': True,
     }, {
-        'url': 'http://videoarchiv.markiza.sk/video/84723',
+        'url': 'https://videoarchiv.markiza.sk/video/84723',
         'only_matching': True,
     }, {
-        'url': 'http://videoarchiv.markiza.sk/video/filmy/85190_kamenak',
+        'url': 'https://videoarchiv.markiza.sk/video/filmy/85190_kamenak',
         'only_matching': True,
     }, {
-        'url': 'http://videoarchiv.markiza.sk/video/reflex/zo-zakulisia/84651_pribeh-alzbetky',
+        'url': 'https://videoarchiv.markiza.sk/video/reflex/zo-zakulisia/84651_pribeh-alzbetky',
         'only_matching': True,
     }, {
-        'url': 'http://videoarchiv.markiza.sk/embed/85295',
+        'url': 'https://videoarchiv.markiza.sk/embed/85295',
         'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        embed = self._search_regex(
+            r'<iframe src="(https:\/\/media.*?)" style="',
+            webpage, 'embed', fatal=False)
+        webpage = self._download_webpage(embed, video_id)
+        data = self._search_regex(
+            r'processAdTagModifier\((\{.*)\), {"video":',
+            webpage, 'embed', fatal=False)
+        data2 = self._search_regex(
+            r'processAdTagModifier\(\{.*\), ({"video":.*)\);',
+            webpage, 'embed', fatal=False)
+        data = re.sub('\\\\/', '/', data)
+        data2 = re.sub('\\\\/', '/', data2)
+        info = json.loads(data)
+        info2 = json.loads(data2)
 
-        data = self._download_json(
-            'http://videoarchiv.markiza.sk/json/video_jwplayer7.json',
-            video_id, query={'id': video_id})
-
-        info = self._parse_jwplayer_data(data, m3u8_id='hls', mpd_id='dash')
-
-        if info.get('_type') == 'playlist':
-            info.update({
-                'id': video_id,
-                'title': try_get(
-                    data, lambda x: x['details']['name'], compat_str),
-            })
-        else:
-            info['duration'] = parse_duration(
-                try_get(data, lambda x: x['details']['duration'], compat_str))
-        return info
+        formats = []
+        for format_id, format_list in info['tracks'].items():
+            if not isinstance(format_list, list):
+                format_list = [format_list]
+            for format_dict in format_list:
+                if not isinstance(format_dict, dict):
+                    continue
+                format_url = url_or_none(format_dict.get('src'))
+                format_type = format_dict.get('type')
+                ext = determine_ext(format_url)
+                if (format_type == 'application/x-mpegURL'
+                        or format_id == 'HLS' or ext == 'm3u8'):
+                    formats.extend(self._extract_m3u8_formats(
+                        format_url, video_id, 'mp4',
+                        entry_protocol='m3u8_native', m3u8_id='hls',
+                        fatal=False))
+                elif (format_type == 'application/dash+xml'
+                      or format_id == 'DASH' or ext == 'mpd'):
+                    formats.extend(self._extract_mpd_formats(
+                        format_url, video_id, mpd_id='dash', fatal=False))
+                else:
+                    formats.append({
+                        'url': format_url,
+                    })
+        thumbnail = info.get('plugins').get('thumbnails').get('url')
+        thumbnail = re.sub('$Num$', '001', thumbnail)
+        duration = info.get('duration')
+        title2 = info2.get('video').get('title')
+        title = info2.get('video').get('custom').get('show_title')
+        return {
+            'id': video_id,
+            'title': title + ' - ' + title2,
+            'thumbnail': thumbnail,
+            'duration': duration,
+            'formats': formats
+        }
 
 
 class MarkizaPageIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?(?:(?:[^/]+\.)?markiza|tvnoviny)\.sk/(?:[^/]+/)*(?P<id>\d+)_'
     _TESTS = [{
-        'url': 'http://www.markiza.sk/soubiz/zahranicny/1923705_oteckovia-maju-svoj-den-ti-slavni-nie-su-o-nic-menej-rozkosni',
+        'url': 'https://www.markiza.sk/soubiz/zahranicny/1923705_oteckovia-maju-svoj-den-ti-slavni-nie-su-o-nic-menej-rozkosni',
         'md5': 'ada4e9fad038abeed971843aa028c7b0',
         'info_dict': {
             'id': '139355',
@@ -87,29 +124,30 @@ class MarkizaPageIE(InfoExtractor):
             'skip_download': True,
         },
     }, {
-        'url': 'http://dajto.markiza.sk/filmy-a-serialy/1774695_frajeri-vo-vegas',
+        'url': 'https://dajto.markiza.sk/filmy-a-serialy/1774695_frajeri-vo-vegas',
         'only_matching': True,
     }, {
-        'url': 'http://superstar.markiza.sk/aktualne/1923870_to-je-ale-telo-spevacka-ukazala-sexy-postavicku-v-bikinach',
+        'url': 'https://superstar.markiza.sk/aktualne/\
+        1923870_to-je-ale-telo-spevacka-ukazala-sexy-postavicku-v-bikinach',
         'only_matching': True,
     }, {
-        'url': 'http://hybsa.markiza.sk/aktualne/1923790_uzasna-atmosfera-na-hybsa-v-poprade-superstaristi-si-prve-koncerty-pred-davom-ludi-poriadne-uzili',
+        'url': 'https://hybsa.markiza.sk/aktualne/1923790_uzasna-atmosfera-na-hybsa-v-poprade-superstaristi-si-prve-koncerty-pred-davom-ludi-poriadne-uzili',
         'only_matching': True,
     }, {
-        'url': 'http://doma.markiza.sk/filmy/1885250_moja-vysnivana-svadba',
+        'url': 'https://doma.markiza.sk/filmy/1885250_moja-vysnivana-svadba',
         'only_matching': True,
     }, {
-        'url': 'http://www.tvnoviny.sk/domace/1923887_po-smrti-manzela-ju-cakalo-poriadne-prekvapenie',
+        'url': 'https://www.tvnoviny.sk/domace/1923887_po-smrti-manzela-ju-cakalo-poriadne-prekvapenie',
         'only_matching': True,
     }]
 
     @classmethod
     def suitable(cls, url):
-        return False if MarkizaIE.suitable(url) else super(MarkizaPageIE, cls).suitable(url)
+        return False if MarkizaIE.suitable(url) else \
+            super(MarkizaPageIE, cls).suitable(url)
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
-
         webpage = self._download_webpage(
             # Downloading for some hosts (e.g. dajto, doma) fails with 500
             # although everything seems to be OK, so considering 500
@@ -117,7 +155,8 @@ class MarkizaPageIE(InfoExtractor):
             url, playlist_id, expected_status=500)
 
         entries = [
-            self.url_result('http://videoarchiv.markiza.sk/video/%s' % video_id)
+            self.url_result(
+                'http://videoarchiv.markiza.sk/video/%s' % video_id)
             for video_id in orderedSet(re.findall(
                 r'(?:initPlayer_|data-entity=["\']|id=["\']player_)(\d+)',
                 webpage))]

youtube_dl/extractor/rtvs.py

@@ -1,7 +1,13 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..utils import (
+    url_or_none,
+    determine_ext
+)
 
 
 class RTVSIE(InfoExtractor):
@@ -26,7 +32,8 @@ class RTVSIE(InfoExtractor):
             'id': '63118',
             'ext': 'mp4',
             'title': 'Amaro Džives - Náš deň',
-            'description': 'Galavečer pri príležitosti Medzinárodného dňa Rómov.'
+            'description':
+            'Galavečer pri príležitosti Medzinárodného dňa Rómov.'
         },
         'params': {
             'skip_download': True,
@@ -36,12 +43,66 @@ class RTVSIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        if url.find('/radio/') != -1:
+            a2 = url.split('/')[-1]
+            a1 = url.split('/')[-2]
+            embed = self._download_webpage(
+                "https://www.rtvs.sk/embed/radio/archive/%s/%s" % (a1, a2),
+                video_id)
+            audio_id = re.search('audio5f.json?id=(?P<id>[^\"]+)', embed)
+            audio_id = audio_id.group('id')
+            info = self._download_json(
+                "https://www.rtvs.sk/json/audio5f.json?id=%s" % audio_id,
+                audio_id)
 
-        playlist_url = self._search_regex(
-            r'playlist["\']?\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
-            'playlist url', group='url')
+            formats = []
+            formats.append({
+                'url': info['playlist'][0]['sources'][0]['src'],
+                'format_id':  None,
+                'height': 0
+                })
+            info = info['playlist'][0]
+            return {
+                'id': audio_id,
+                'title': info.get('title'),
+                'thumbnail': info.get('image'),
+                'formats': formats
+            }
+        else:
+            info = self._download_json(
+                "https://www.rtvs.sk/json/archive5f.json?id=%s" % video_id,
+                video_id)
+            info = info.get('clip')
 
-        data = self._download_json(
-            playlist_url, video_id, 'Downloading playlist')[0]
-        return self._parse_jwplayer_data(data, video_id=video_id)
+            formats = []
+            for format_id, format_list in info.items():
+                if not isinstance(format_list, list):
+                    format_list = [format_list]
+                for format_dict in format_list:
+                    if not isinstance(format_dict, dict):
+                        continue
+                    format_url = url_or_none(format_dict.get('src'))
+                    format_type = format_dict.get('type')
+                    ext = determine_ext(format_url)
+                    if (format_type == 'application/x-mpegURL'
+                            or format_id == 'HLS' or ext == 'm3u8'):
+                        formats.extend(self._extract_m3u8_formats(
+                            format_url, video_id, 'mp4',
+                            entry_protocol='m3u8_native', m3u8_id='hls',
+                            fatal=False))
+                    elif (format_type == 'application/dash+xml'
+                          or format_id == 'DASH' or ext == 'mpd'):
+                        pass
+                    else:
+                        formats.append({
+                            'url': format_url,
+                        })
+            formats = sorted(formats, key=lambda i: i['tbr'])
+            dt = info.get('datetime_create')
+            return {
+                'id': video_id,
+                'title': info.get('title') + '-' + dt[:10],
+                'thumbnail': info.get('image'),
+                'description': info.get('description'),
+                'formats': formats
+            }

youtube_dl/extractor/senateisvp.py

@@ -47,7 +47,7 @@ class SenateISVPIE(InfoExtractor):
         ['vetaff', '76462', 'http://vetaff-f.akamaihd.net'],
         ['arch', '', 'http://ussenate-f.akamaihd.net/']
     ]
-    _IE_NAME = 'senate.gov'
+    IE_NAME = 'senate.gov'
     _VALID_URL = r'https?://(?:www\.)?senate\.gov/isvp/?\?(?P<qs>.+)'
     _TESTS = [{
         'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',

youtube_dl/jsinterp.py

@@ -686,6 +686,8 @@ class JSInterpreter(object):
             raise self.Exception('Cannot get index {idx!r:.100}'.format(**locals()), expr=repr(obj), cause=e)
 
     def _dump(self, obj, namespace):
+        if obj is JS_Undefined:
+            return 'undefined'
         try:
             return json.dumps(obj)
         except TypeError: