Merge eaf29d53e4df3ff8be9d78e84c7c556b01bf7034 into 4e714f9df1ed2cccd51df60d45ff5504abe827b7

* thx seproDev, yt-dlp/yt-dlp/pull/12694/commits/ae69e3c
* also add documenting comment in `InfoExtractor`

README.md

@@ -404,6 +404,9 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     -2, --twofactor TWOFACTOR            Two-factor authentication code
     -n, --netrc                          Use .netrc authentication data
     --video-password PASSWORD            Video password (vimeo, youku)
+    --client-certificate                 Path to a single certificate file in 
+                                         PEM format, used to authenticate to the
+                                         site (including private key)
 
 ## Adobe Pass Options:
     --ap-mso MSO                         Adobe Pass multiple-system operator (TV

test/test_clientcert.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# coding: utf-8
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import http_server_port
+from youtube_dl import YoutubeDL
+from youtube_dl.compat import compat_http_server
+import ssl
+import threading
+
+from test.test_http import HTTPTestRequestHandler, FakeLogger
+
+
+# See https://gist.github.com/dergachev/7028596
+# and http://www.piware.de/2011/01/creating-an-https-server-in-python/
+# and https://blog.devolutions.net/2020/07/tutorial-how-to-generate-secure-self-signed-server-and-client-certificates-with-openssl
+
+
+TEST_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+class TestClientCert(unittest.TestCase):
+    def setUp(self):
+        certfn = os.path.join(TEST_DIR, 'testcert.pem')
+        cacertfn = os.path.join(TEST_DIR, 'testdata', 'clientcert', 'ca.crt')
+        self.httpd = compat_http_server.HTTPServer(('127.0.0.1', 0), HTTPTestRequestHandler)
+        self.httpd.socket = ssl.wrap_socket(
+            self.httpd.socket, cert_reqs=ssl.CERT_REQUIRED, ca_certs=cacertfn, certfile=certfn, server_side=True)
+        self.port = http_server_port(self.httpd)
+        self.server_thread = threading.Thread(target=self.httpd.serve_forever)
+        self.server_thread.daemon = True
+        self.server_thread.start()
+
+    def test_check_clientcertificate(self):
+        clientcertfn = os.path.join(TEST_DIR, 'testdata', 'clientcert', 'client.crt')
+        ydl = YoutubeDL({'logger': FakeLogger(), 'clientcertificate': clientcertfn,
+            # Disable client-side validation of unacceptable self-signed testcert.pem
+            # The test is of a check on the server side, so unaffected
+            'nocheckcertificate': True,
+            })
+        r = ydl.extract_info('https://127.0.0.1:%d/video.html' % self.port)
+        self.assertEqual(r['entries'][0]['url'], 'https://127.0.0.1:%d/vid.mp4' % self.port)
+
+
+if __name__ == '__main__':
+    unittest.main()

test/testdata/clientcert/ca.crt

@@ -0,0 +1,11 @@
+-----BEGIN CERTIFICATE-----
+MIIBnTCCAUOgAwIBAgIUN4jSR5qgSLKJs4lWdBUQPiOyUPEwCgYIKoZIzj0EAwIw
+JDETMBEGA1UECgwKWW91dHViZS1ETDENMAsGA1UEAwwEVGVzdDAeFw0yMTA3MTkx
+NTE1MzJaFw0zODAxMTgxNTE1MzJaMCQxEzARBgNVBAoMCllvdXR1YmUtREwxDTAL
+BgNVBAMMBFRlc3QwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQ8DNZrIDTIQ4mN
+IofBxPIDF2nZUKKAUPAMyn6ntXh99WhLRO5caGKTPWip+LspF5j2uyd2DAcAKMZ5
+170qIbSCo1MwUTAdBgNVHQ4EFgQUj+rfMTfIDHufM94pYwjvI8pZKlYwHwYDVR0j
+BBgwFoAUj+rfMTfIDHufM94pYwjvI8pZKlYwDwYDVR0TAQH/BAUwAwEB/zAKBggq
+hkjOPQQDAgNIADBFAiAwhl8mpPiZoAXWfPHSZaxiLPjy2m4pZK70O0BHnxmSJQIh
+AOeQgZh0j0SkZW0kXHBPWguCgvVm5tqQPQJCevgNDKWP
+-----END CERTIFICATE-----

test/testdata/clientcert/client.crt

@@ -0,0 +1,14 @@
+-----BEGIN CERTIFICATE-----
+MIIBSTCB8AIUE2DY1KuqtYWIi0KYeSYvta9sV+swCgYIKoZIzj0EAwIwJDETMBEG
+A1UECgwKWW91dHViZS1ETDENMAsGA1UEAwwEVGVzdDAeFw0yMTA3MTkxNTE2MjZa
+Fw0zODAxMTgxNTE2MjZaMCsxEzARBgNVBAoMCllvdXR1YmUtREwxFDASBgNVBAMM
+C1Rlc3QgQ2xpZW50MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEc0ldxFETUFCS
+CsMq01OUEYp9zkPbXZ9IkTUu1RQhliuPYCsc4Q+UZ8z+Ttcyqa76jAMcmQWh+n2P
+4i7uCDvZ8zAKBggqhkjOPQQDAgNIADBFAiEAiuQWNv6F7EO+bZGhDDxhUkGdhWOy
+36YbZa+BZ8CYae0CIBVfdEnrG5M9tc6PZjXiXgoUMUrnPnRXs76ihQ55hHPW
+-----END CERTIFICATE-----
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIBVDCR/z/PuVFzGKFCOt9GYGpwQ8vJTXAj59jPwP4OFVoAoGCCqGSM49
+AwEHoUQDQgAEc0ldxFETUFCSCsMq01OUEYp9zkPbXZ9IkTUu1RQhliuPYCsc4Q+U
+Z8z+Ttcyqa76jAMcmQWh+n2P4i7uCDvZ8w==
+-----END EC PRIVATE KEY-----

test/testdata/clientcert/instructions.txt

@@ -0,0 +1,11 @@
+#https://blog.devolutions.net/2020/07/tutorial-how-to-generate-secure-self-signed-server-and-client-certificates-with-openssl
+# Adapt the commands below
+# 6027 days from the time of signing to the day before Y2038
+# Recalculate or use -preserve_dates if re-signing, until
+# 32-bit time_t is not an issue 
+#openssl ecparam -name prime256v1 -genkey -noout -out ca.key
+#openssl req -new -x509 -sha256 -days 6027 -key ca.key -out ca.crt
+#openssl ecparam -name prime256v1 -genkey -noout -out client.key
+#openssl req -new -sha256 -key client.key -out client.csr
+#openssl x509 -req -in client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out client.crt -days 6027 -sha256
+#cat client.key >> client.crt

youtube_dl/__init__.py

@@ -322,6 +322,7 @@ def _real_main(argv=None):
         'password': opts.password,
         'twofactor': opts.twofactor,
         'videopassword': opts.videopassword,
+        'clientcertificate': opts.clientcertificate,
         'ap_mso': opts.ap_mso,
         'ap_username': opts.ap_username,
         'ap_password': opts.ap_password,

youtube_dl/extractor/bokecc.py

@@ -32,7 +32,7 @@ class BokeCCBaseIE(InfoExtractor):
 
 
 class BokeCCIE(BokeCCBaseIE):
-    _IE_DESC = 'CC视频'
+    IE_DESC = 'CC视频'
     _VALID_URL = r'https?://union\.bokecc\.com/playvideo\.bo\?(?P<query>.*)'
 
     _TESTS = [{

youtube_dl/extractor/cloudy.py

@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class CloudyIE(InfoExtractor):
-    _IE_DESC = 'cloudy.ec'
+    IE_DESC = 'cloudy.ec'
     _VALID_URL = r'https?://(?:www\.)?cloudy\.ec/(?:v/|embed\.php\?.*?\bid=)(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'https://www.cloudy.ec/v/af511e2527aac',

youtube_dl/extractor/common.py

@@ -422,6 +422,8 @@ class InfoExtractor(object):
     _GEO_COUNTRIES = None
     _GEO_IP_BLOCKS = None
     _WORKING = True
+    # supply this in public subclasses: used in supported sites list, etc
+    # IE_DESC = 'short description of IE'
 
     def __init__(self, downloader=None):
         """Constructor. Receives an optional downloader."""

youtube_dl/extractor/itv.py

@@ -35,15 +35,6 @@ from ..utils import (
 
 class ITVBaseIE(InfoExtractor):
 
-    def _search_nextjs_data(self, webpage, video_id, **kw):
-        transform_source = kw.pop('transform_source', None)
-        fatal = kw.pop('fatal', True)
-        return self._parse_json(
-            self._search_regex(
-                r'''<script\b[^>]+\bid=('|")__NEXT_DATA__\1[^>]*>(?P<js>[^<]+)</script>''',
-                webpage, 'next.js data', group='js', fatal=fatal, **kw),
-            video_id, transform_source=transform_source, fatal=fatal)
-
     def __handle_request_webpage_error(self, err, video_id=None, errnote=None, fatal=True):
         if errnote is False:
             return False
@@ -109,7 +100,9 @@ class ITVBaseIE(InfoExtractor):
 
 class ITVIE(ITVBaseIE):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/(?:(?P<w>watch)|hub)/[^/]+/(?(w)[\w-]+/)(?P<id>\w+)'
-    _IE_DESC = 'ITVX'
+    IE_DESC = 'ITVX'
+    _WORKING = False
+
     _TESTS = [{
         'note': 'Hub URLs redirect to ITVX',
         'url': 'https://www.itv.com/hub/liar/2a4547a0012',
@@ -270,7 +263,7 @@ class ITVIE(ITVBaseIE):
                 'ext': determine_ext(href, 'vtt'),
             })
 
-        next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default='{}')
+        next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default={})
         video_data.update(traverse_obj(next_data, ('props', 'pageProps', ('title', 'episode')), expected_type=dict)[0] or {})
         title = traverse_obj(video_data, 'headerTitle', 'episodeTitle')
         info = self._og_extract(webpage, require_title=not title)
@@ -323,7 +316,7 @@ class ITVIE(ITVBaseIE):
 
 class ITVBTCCIE(ITVBaseIE):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/(?!(?:watch|hub)/)(?:[^/]+/)+(?P<id>[^/?#&]+)'
-    _IE_DESC = 'ITV articles: News, British Touring Car Championship'
+    IE_DESC = 'ITV articles: News, British Touring Car Championship'
     _TESTS = [{
         'note': 'British Touring Car Championship',
         'url': 'https://www.itv.com/btcc/articles/btcc-2018-all-the-action-from-brands-hatch',

youtube_dl/extractor/senateisvp.py

@@ -47,7 +47,7 @@ class SenateISVPIE(InfoExtractor):
         ['vetaff', '76462', 'http://vetaff-f.akamaihd.net'],
         ['arch', '', 'http://ussenate-f.akamaihd.net/']
     ]
-    _IE_NAME = 'senate.gov'
+    IE_NAME = 'senate.gov'
     _VALID_URL = r'https?://(?:www\.)?senate\.gov/isvp/?\?(?P<qs>.+)'
     _TESTS = [{
         'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',

youtube_dl/options.py

@@ -368,6 +368,10 @@ def parseOpts(overrideArguments=None):
         '--video-password',
         dest='videopassword', metavar='PASSWORD',
         help='Video password (vimeo, youku)')
+    authentication.add_option(
+        '--client-certificate',
+        dest='clientcertificate', metavar='PATH',
+        help='Path to a single certificate file in PEM format, used to authenticate to the site (including private key)')
 
     adobe_pass = optparse.OptionGroup(parser, 'Adobe Pass Options')
     adobe_pass.add_option(

youtube_dl/utils.py

@@ -2529,6 +2529,10 @@ def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
     # https://github.com/ytdl-org/youtube-dl/issues/6727)
     if sys.version_info < (3, 0):
         kwargs['strict'] = True
+    if is_https:
+        client_cert_path = ydl_handler._params.get('clientcertificate')
+        if client_cert_path:
+            kwargs['cert_file'] = client_cert_path
     hc = http_class(*args, **compat_kwargs(kwargs))
     source_address = ydl_handler._params.get('source_address')