Merge ddbd5451010e00c2ce64e57ce1d03ddbcd8ab4ff into 4e714f9df1ed2cccd51df60d45ff5504abe827b7

* thx seproDev, yt-dlp/yt-dlp/pull/12694/commits/ae69e3c
* also add documenting comment in `InfoExtractor`

test/test_download.py

@@ -69,10 +69,10 @@ def _file_md5(fn):
         return hashlib.md5(f.read()).hexdigest()
 
 
-defs = gettestcases()
+ie_testcases = gettestcases()
 
 
-class TestDownload(unittest.TestCase):
+class BaseDownloadTCase(unittest.TestCase):
     # Parallel testing in nosetests. See
     # http://nose.readthedocs.org/en/latest/doc_tests/test_multiprocess/multiprocess.html
     _multiprocess_shared_ = True
@@ -91,8 +91,9 @@ class TestDownload(unittest.TestCase):
                                strclass(self.__class__),
                                ' [%s]' % add_ie if add_ie else '')
 
-    def setUp(self):
-        self.defs = defs
+
+TestDownload = unittest.TestSuite()
+
 
 # Dynamically generate tests
 
@@ -275,20 +276,42 @@ def generator(test_case, tname):
     return test_template
 
 
+cache = {}
+
 # And add them to TestDownload
-for n, test_case in enumerate(defs):
-    tname = 'test_' + str(test_case['name'])
-    i = 1
-    while hasattr(TestDownload, tname):
-        tname = 'test_%s_%d' % (test_case['name'], i)
+for test_case in ie_testcases:
+    # Get or create a sub-test for the extractor file
+    module_name = test_case.get('module_name', 'unknown').rsplit('.', 1)[-1]  # type: str
+    extractor_file_name = str('test_%s' % module_name)
+    extractor_file_suite = getattr(TestDownload, extractor_file_name, None)
+    if extractor_file_suite is None:
+        extractor_file_suite = type(extractor_file_name, (unittest.TestSuite,), {})()
+        setattr(TestDownload, extractor_file_name, extractor_file_suite)
+        TestDownload.addTest(extractor_file_suite)
+
+    # Get or create a sub-test for the info extractor
+    # This class contains the actual tests
+    extractor_class_name = str('test_%s' % test_case['name'])
+    ExtractorClass = getattr(extractor_file_suite, extractor_class_name, None)
+    if ExtractorClass is None:
+        ExtractorClass = type(extractor_class_name, (BaseDownloadTCase,), {})
+        setattr(extractor_file_suite, extractor_class_name, ExtractorClass)
+
+    i = 0
+    tname = 'test_%d' % i
+    while hasattr(ExtractorClass, tname):
+        tname = 'test_%d' % i
         i += 1
     test_method = generator(test_case, tname)
     test_method.__name__ = str(tname)
     ie_list = test_case.get('add_ie')
     test_method.add_ie = ie_list and ','.join(ie_list)
-    setattr(TestDownload, test_method.__name__, test_method)
-    del test_method
+
+    setattr(ExtractorClass, test_method.__name__, test_method)
+    extractor_file_suite.addTest(ExtractorClass(test_method.__name__))
+
+    del test_method, test_case
 
 
 if __name__ == '__main__':
-    unittest.main()
+    unittest.main(argv=['TestDownload'])

youtube_dl/extractor/bokecc.py

@@ -32,7 +32,7 @@ class BokeCCBaseIE(InfoExtractor):
 
 
 class BokeCCIE(BokeCCBaseIE):
-    _IE_DESC = 'CC视频'
+    IE_DESC = 'CC视频'
     _VALID_URL = r'https?://union\.bokecc\.com/playvideo\.bo\?(?P<query>.*)'
 
     _TESTS = [{

youtube_dl/extractor/cloudy.py

@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class CloudyIE(InfoExtractor):
-    _IE_DESC = 'cloudy.ec'
+    IE_DESC = 'cloudy.ec'
     _VALID_URL = r'https?://(?:www\.)?cloudy\.ec/(?:v/|embed\.php\?.*?\bid=)(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'https://www.cloudy.ec/v/af511e2527aac',

youtube_dl/extractor/common.py

@@ -422,6 +422,8 @@ class InfoExtractor(object):
     _GEO_COUNTRIES = None
     _GEO_IP_BLOCKS = None
     _WORKING = True
+    # supply this in public subclasses: used in supported sites list, etc
+    # IE_DESC = 'short description of IE'
 
     def __init__(self, downloader=None):
         """Constructor. Receives an optional downloader."""
@@ -3263,6 +3265,7 @@ class InfoExtractor(object):
             if not include_onlymatching and t.get('only_matching', False):
                 continue
             t['name'] = type(self).__name__[:-len('IE')]
+            t['module_name'] = type(self).__module__
             yield t
 
     def is_suitable(self, age_limit):

youtube_dl/extractor/itv.py

@@ -35,15 +35,6 @@ from ..utils import (
 
 class ITVBaseIE(InfoExtractor):
 
-    def _search_nextjs_data(self, webpage, video_id, **kw):
-        transform_source = kw.pop('transform_source', None)
-        fatal = kw.pop('fatal', True)
-        return self._parse_json(
-            self._search_regex(
-                r'''<script\b[^>]+\bid=('|")__NEXT_DATA__\1[^>]*>(?P<js>[^<]+)</script>''',
-                webpage, 'next.js data', group='js', fatal=fatal, **kw),
-            video_id, transform_source=transform_source, fatal=fatal)
-
     def __handle_request_webpage_error(self, err, video_id=None, errnote=None, fatal=True):
         if errnote is False:
             return False
@@ -109,7 +100,9 @@ class ITVBaseIE(InfoExtractor):
 
 class ITVIE(ITVBaseIE):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/(?:(?P<w>watch)|hub)/[^/]+/(?(w)[\w-]+/)(?P<id>\w+)'
-    _IE_DESC = 'ITVX'
+    IE_DESC = 'ITVX'
+    _WORKING = False
+
     _TESTS = [{
         'note': 'Hub URLs redirect to ITVX',
         'url': 'https://www.itv.com/hub/liar/2a4547a0012',
@@ -270,7 +263,7 @@ class ITVIE(ITVBaseIE):
                 'ext': determine_ext(href, 'vtt'),
             })
 
-        next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default='{}')
+        next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default={})
         video_data.update(traverse_obj(next_data, ('props', 'pageProps', ('title', 'episode')), expected_type=dict)[0] or {})
         title = traverse_obj(video_data, 'headerTitle', 'episodeTitle')
         info = self._og_extract(webpage, require_title=not title)
@@ -323,7 +316,7 @@ class ITVIE(ITVBaseIE):
 
 class ITVBTCCIE(ITVBaseIE):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/(?!(?:watch|hub)/)(?:[^/]+/)+(?P<id>[^/?#&]+)'
-    _IE_DESC = 'ITV articles: News, British Touring Car Championship'
+    IE_DESC = 'ITV articles: News, British Touring Car Championship'
     _TESTS = [{
         'note': 'British Touring Car Championship',
         'url': 'https://www.itv.com/btcc/articles/btcc-2018-all-the-action-from-brands-hatch',

youtube_dl/extractor/senateisvp.py

@@ -47,7 +47,7 @@ class SenateISVPIE(InfoExtractor):
         ['vetaff', '76462', 'http://vetaff-f.akamaihd.net'],
         ['arch', '', 'http://ussenate-f.akamaihd.net/']
     ]
-    _IE_NAME = 'senate.gov'
+    IE_NAME = 'senate.gov'
     _VALID_URL = r'https?://(?:www\.)?senate\.gov/isvp/?\?(?P<qs>.+)'
     _TESTS = [{
         'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',