Merge 4b1c8309573a4c6e3d5e883cc64338a8e19d6778 into 3eb8d22ddb8982ca4fb56bb7a8d6517538bf14c6

* thx seproDev, yt-dlp/yt-dlp/pull/12694/commits/ae69e3c
* also add documenting comment in `InfoExtractor`

youtube_dl/extractor/bokecc.py

@@ -32,7 +32,7 @@ class BokeCCBaseIE(InfoExtractor):
 
 
 class BokeCCIE(BokeCCBaseIE):
-    _IE_DESC = 'CC视频'
+    IE_DESC = 'CC视频'
     _VALID_URL = r'https?://union\.bokecc\.com/playvideo\.bo\?(?P<query>.*)'
 
     _TESTS = [{

youtube_dl/extractor/cloudy.py

@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class CloudyIE(InfoExtractor):
-    _IE_DESC = 'cloudy.ec'
+    IE_DESC = 'cloudy.ec'
     _VALID_URL = r'https?://(?:www\.)?cloudy\.ec/(?:v/|embed\.php\?.*?\bid=)(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'https://www.cloudy.ec/v/af511e2527aac',

youtube_dl/extractor/common.py

@@ -422,6 +422,8 @@ class InfoExtractor(object):
     _GEO_COUNTRIES = None
     _GEO_IP_BLOCKS = None
     _WORKING = True
+    # supply this in public subclasses: used in supported sites list, etc
+    # IE_DESC = 'short description of IE'
 
     def __init__(self, downloader=None):
         """Constructor. Receives an optional downloader."""

youtube_dl/extractor/extractors.py

@@ -1069,7 +1069,7 @@ from .rtve import RTVEALaCartaIE, RTVELiveIE, RTVEInfantilIE, RTVELiveIE, RTVETe
 from .rtvnh import RTVNHIE
 from .rtvs import RTVSIE
 from .ruhd import RUHDIE
-from .rumble import RumbleEmbedIE
+from .rumble import RumbleIE
 from .rutube import (
     RutubeIE,
     RutubeChannelIE,
@@ -1439,6 +1439,7 @@ from .urort import UrortIE
 from .urplay import URPlayIE
 from .usanetwork import USANetworkIE
 from .usatoday import USATodayIE
+from .usawatchdog import UsaWatchdogIE
 from .ustream import UstreamIE, UstreamChannelIE
 from .ustudio import (
     UstudioIE,

youtube_dl/extractor/generic.py

@@ -132,6 +132,7 @@ from .kinja import KinjaEmbedIE
 from .arcpublishing import ArcPublishingIE
 from .medialaan import MedialaanIE
 from .simplecast import SimplecastIE
+from .rumble import RumbleIE
 
 
 class GenericIE(InfoExtractor):
@@ -3518,6 +3519,10 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 zype_urls, video_id, video_title, ie=ZypeIE.ie_key())
 
+        rumble_urls = RumbleIE.rumble_embedded_id(webpage)
+        if rumble_urls is not None:
+            return self.playlist_result(rumble_urls) if len(rumble_urls) > 1 else rumble_urls[0]
+
         # Look for HTML5 media
         entries = self._parse_html5_media_entries(url, webpage, video_id, m3u8_id='hls')
         if entries:

youtube_dl/extractor/itv.py

@@ -35,15 +35,6 @@ from ..utils import (
 
 class ITVBaseIE(InfoExtractor):
 
-    def _search_nextjs_data(self, webpage, video_id, **kw):
-        transform_source = kw.pop('transform_source', None)
-        fatal = kw.pop('fatal', True)
-        return self._parse_json(
-            self._search_regex(
-                r'''<script\b[^>]+\bid=('|")__NEXT_DATA__\1[^>]*>(?P<js>[^<]+)</script>''',
-                webpage, 'next.js data', group='js', fatal=fatal, **kw),
-            video_id, transform_source=transform_source, fatal=fatal)
-
     def __handle_request_webpage_error(self, err, video_id=None, errnote=None, fatal=True):
         if errnote is False:
             return False
@@ -109,7 +100,9 @@ class ITVBaseIE(InfoExtractor):
 
 class ITVIE(ITVBaseIE):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/(?:(?P<w>watch)|hub)/[^/]+/(?(w)[\w-]+/)(?P<id>\w+)'
-    _IE_DESC = 'ITVX'
+    IE_DESC = 'ITVX'
+    _WORKING = False
+
     _TESTS = [{
         'note': 'Hub URLs redirect to ITVX',
         'url': 'https://www.itv.com/hub/liar/2a4547a0012',
@@ -270,7 +263,7 @@ class ITVIE(ITVBaseIE):
                 'ext': determine_ext(href, 'vtt'),
             })
 
-        next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default='{}')
+        next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default={})
         video_data.update(traverse_obj(next_data, ('props', 'pageProps', ('title', 'episode')), expected_type=dict)[0] or {})
         title = traverse_obj(video_data, 'headerTitle', 'episodeTitle')
         info = self._og_extract(webpage, require_title=not title)
@@ -323,7 +316,7 @@ class ITVIE(ITVBaseIE):
 
 class ITVBTCCIE(ITVBaseIE):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/(?!(?:watch|hub)/)(?:[^/]+/)+(?P<id>[^/?#&]+)'
-    _IE_DESC = 'ITV articles: News, British Touring Car Championship'
+    IE_DESC = 'ITV articles: News, British Touring Car Championship'
     _TESTS = [{
         'note': 'British Touring Car Championship',
         'url': 'https://www.itv.com/btcc/articles/btcc-2018-all-the-action-from-brands-hatch',

youtube_dl/extractor/rumble.py

@@ -1,5 +1,6 @@
 # coding: utf-8
 from __future__ import unicode_literals
+import re
 
 from .common import InfoExtractor
 from ..compat import compat_str
@@ -8,31 +9,100 @@ from ..utils import (
     int_or_none,
     parse_iso8601,
     try_get,
+    ExtractorError,
 )
 
 
-class RumbleEmbedIE(InfoExtractor):
+class RumbleIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?rumble\.com/embed/(?:[0-9a-z]+\.)?(?P<id>[0-9a-z]+)'
-    _TESTS = [{
-        'url': 'https://rumble.com/embed/v5pv5f',
-        'md5': '36a18a049856720189f30977ccbb2c34',
-        'info_dict': {
-            'id': 'v5pv5f',
-            'ext': 'mp4',
-            'title': 'WMAR 2 News Latest Headlines | October 20, 6pm',
-            'timestamp': 1571611968,
-            'upload_date': '20191020',
-        }
-    }, {
-        'url': 'https://rumble.com/embed/ufe9n.v5pv5f',
-        'only_matching': True,
-    }]
 
-    def _real_extract(self, url):
+    RE_DICT = {
-        video_id = self._match_id(url)
+        'iframe_url': {
+            're': r'https?://(?:www\.)?rumble\.com/embed/(?:[0-9a-z]+\.)?(?P<id>[0-9a-z]+)',
+            'compiled': None},
+        'jscript_url': {
+            're': r'https?://rumble\.com/[a-zA-Z0-9-_.]*\.html',
+            'compiled': None},
+        'list_url': {
+            're': r'https?://rumble.com/(?:c|user)/(?P<id>[^/]+)',
+            'compiled': None},
+        'jscript_id': {
+            're': r'Rumble *\( *["\']play["\'], *\{[^}]*["\']video["\'] *: *["\'](?P<id>[^"\']+)',
+            'compiled': None}
+    }
+
+    _TESTS = [
+        {
+            'url': 'https://rumble.com/embed/v5pv5f',
+            'md5': '36a18a049856720189f30977ccbb2c34',
+            'info_dict': {
+                'id': 'v5pv5f',
+                'ext': 'mp4',
+                'title': 'WMAR 2 News Latest Headlines | October 20, 6pm',
+                'timestamp': 1571611968,
+                'upload_date': '20191020',
+            }
+        },
+        {
+            'url': 'https://rumble.com/v8c1bt-wmar-2-news-latest-headlines-october-20-6pm.html',
+            'md5': '36a18a049856720189f30977ccbb2c34',
+            'info_dict': {
+                'id': 'v5pv5f',
+                'ext': 'mp4',
+                'title': 'WMAR 2 News Latest Headlines | October 20, 6pm',
+                'timestamp': 1571611968,
+                'upload_date': '20191020',
+            }
+        },
+        {
+            'url': 'https://rumble.com/c/PeakProsperity',
+            'playlist_mincount': 25,
+            'info_dict': {
+                'id': 'PeakProsperity',
+            }
+        },
+        {
+            'url': 'https://rumble.com/embed/ufe9n.v5pv5f',
+            'only_matching': True,
+        }
+    ]
+
+    @classmethod
+    def get_re(cls, tag):
+        if cls.RE_DICT[tag]['compiled'] is None:
+            cls.RE_DICT[tag]['compiled'] = re.compile(cls.RE_DICT[tag]['re'])
+        return cls.RE_DICT[tag]['compiled']
+
+    @classmethod
+    def suitable(cls, url):
+        return (cls.get_re('jscript_url').match(url) is not None
+                or cls.get_re('list_url').match(url) is not None
+                or cls.get_re('iframe_url').match(url) is not None)
+
+    @staticmethod
+    def rumble_embedded_id(page_data):
+        '''For use by extractors of sites which use emedded Rumble videos. Given
+        a webpage as a string returns a list of url result dicts for each embedded
+        rumble video found. None is returned if no embeds were found. Duplicates
+        are not removed'''
+
+        embeds = []
+        # The JS embeds
+        for mobj in RumbleIE.get_re('jscript_id').finditer(page_data):
+            embeds.append(InfoExtractor.url_result('https://rumble.com/embed/' + mobj.group('id'), 'Rumble', mobj.group('id')))
+
+        # The iframes embeds
+        for mobj in RumbleIE.get_re('iframe_url').finditer(page_data):
+            embeds.append(InfoExtractor.url_result('https://rumble.com/embed/' + mobj.group('id'), 'Rumble', mobj.group('id')))
+
+        return embeds if embeds else None
+
+    def rumble_video_info(self, video_id):
         video = self._download_json(
             'https://rumble.com/embedJS/', video_id,
             query={'request': 'video', 'v': video_id})
+        if not video:
+            raise ExtractorError('Unable to locate video information.', expected=True)
+
         title = video['title']
 
         formats = []
@@ -65,3 +135,23 @@ class RumbleEmbedIE(InfoExtractor):
             'channel_url': author.get('url'),
             'duration': int_or_none(video.get('duration')),
         }
+
+    def _real_extract(self, url):
+        if self.get_re('jscript_url').match(url) is not None:
+            page = self._download_webpage(url, 'Rumble Page')
+            video_id = self._search_regex(self.get_re('jscript_id'), page, "id")
+            return self.rumble_video_info(video_id)
+
+        mobj = self.get_re('list_url').match(url)
+        if mobj is not None:
+            urls = []
+            id = mobj.group('id')
+            page = self._download_webpage(url, id)
+            for mobj in re.finditer(r'<a class=video-item--a href=\/(?P<href>[a-zA-Z0-9\-.]+)>', page):
+                urls.append('https://rumble.com/' + mobj.group('href'))
+
+            return self.playlist_from_matches(urls, id)
+
+        mobj = self.get_re('iframe_url').match(url)
+        if mobj is not None:
+            return self.rumble_video_info(mobj.group('id'))

youtube_dl/extractor/senateisvp.py

@@ -47,7 +47,7 @@ class SenateISVPIE(InfoExtractor):
         ['vetaff', '76462', 'http://vetaff-f.akamaihd.net'],
         ['arch', '', 'http://ussenate-f.akamaihd.net/']
     ]
-    _IE_NAME = 'senate.gov'
+    IE_NAME = 'senate.gov'
     _VALID_URL = r'https?://(?:www\.)?senate\.gov/isvp/?\?(?P<qs>.+)'
     _TESTS = [{
         'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',

youtube_dl/extractor/usawatchdog.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+import re
+
+from .common import InfoExtractor
+
+
+class UsaWatchdogIE(InfoExtractor):
+    _VALID_URL = r'^https?://(?:www\.)?usawatchdog\.com/?$'
+    _TEST = {
+        'url': 'https://usawatchdog.com/',
+        'playlist_mincount': 15,
+        'info_dict': {
+            'id': 'USA Watchdog',
+        }}
+
+    def _real_extract(self, url):
+        matches = []
+        for mobj in re.finditer(r'front-view-title[^<]+<a.+href=["\'](?P<href>https?:(?:www\.)?//usawatchdog.com/[^/]+\/?)[^>]+>(?P<title>[^<]+)',
+                                self._download_webpage(url, 'Site Root')):
+            matches.append(self.url_result(mobj.group('href'),
+                                           video_title=mobj.group('title').encode('utf8')))
+
+        return self.playlist_result(matches, 'USA Watchdog')

youtube_dl/jsinterp.py

@@ -686,6 +686,8 @@ class JSInterpreter(object):
             raise self.Exception('Cannot get index {idx!r:.100}'.format(**locals()), expr=repr(obj), cause=e)
 
     def _dump(self, obj, namespace):
+        if obj is JS_Undefined:
+            return 'undefined'
         try:
             return json.dumps(obj)
         except TypeError: