From a67dafe3aa93b6114f177dc3c4bf86177451cda9 Mon Sep 17 00:00:00 2001 From: df Date: Wed, 14 Jul 2021 16:51:24 +0100 Subject: [PATCH 1/4] Add and implement --client-certificate option Use a PEM certificate to authenticate HTTPS access to site --- youtube_dl/__init__.py | 1 + youtube_dl/options.py | 4 ++++ youtube_dl/utils.py | 4 ++++ 3 files changed, 9 insertions(+) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index e1bd67919..d43155993 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -320,6 +320,7 @@ def _real_main(argv=None): 'password': opts.password, 'twofactor': opts.twofactor, 'videopassword': opts.videopassword, + 'clientcertificate': opts.clientcertificate, 'ap_mso': opts.ap_mso, 'ap_username': opts.ap_username, 'ap_password': opts.ap_password, diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 0a0641bd4..399160c2f 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -370,6 +370,10 @@ def parseOpts(overrideArguments=None): '--video-password', dest='videopassword', metavar='PASSWORD', help='Video password (vimeo, youku)') + authentication.add_option( + '--client-certificate', + dest='clientcertificate', metavar='PATH', + help='Path to a single certificate file in PEM format, used to authenticate to the site') adobe_pass = optparse.OptionGroup(parser, 'Adobe Pass Options') adobe_pass.add_option( diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index e722eed58..ff31d570f 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -2477,6 +2477,10 @@ def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs): # https://github.com/ytdl-org/youtube-dl/issues/6727) if sys.version_info < (3, 0): kwargs['strict'] = True + if is_https: + client_cert_path = ydl_handler._params.get('clientcertificate') + if client_cert_path: + kwargs['cert_file'] = client_cert_path hc = http_class(*args, **compat_kwargs(kwargs)) source_address = ydl_handler._params.get('source_address') From 8f2341c5310e72913c0bcb8a892369919b1fa6fe Mon Sep 17 00:00:00 2001 From: df Date: Mon, 19 Jul 2021 16:32:12 +0100 Subject: [PATCH 2/4] Added test routine for clientcertificate option --- test/test_clientcert.py | 48 +++++++++++++++++++++++ test/testdata/clientcert/ca.crt | 11 ++++++ test/testdata/clientcert/client.crt | 14 +++++++ test/testdata/clientcert/instructions.txt | 11 ++++++ 4 files changed, 84 insertions(+) create mode 100644 test/test_clientcert.py create mode 100644 test/testdata/clientcert/ca.crt create mode 100644 test/testdata/clientcert/client.crt create mode 100644 test/testdata/clientcert/instructions.txt diff --git a/test/test_clientcert.py b/test/test_clientcert.py new file mode 100644 index 000000000..e054e8129 --- /dev/null +++ b/test/test_clientcert.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# coding: utf-8 +from __future__ import unicode_literals + +# Allow direct execution +import os +import sys +import unittest +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from test.helper import http_server_port +from youtube_dl import YoutubeDL +from youtube_dl.compat import compat_http_server +import ssl +import threading + +from test.test_http import HTTPTestRequestHandler, FakeLogger + + +# See https://gist.github.com/dergachev/7028596 +# and http://www.piware.de/2011/01/creating-an-https-server-in-python/ +# and https://blog.devolutions.net/2020/07/tutorial-how-to-generate-secure-self-signed-server-and-client-certificates-with-openssl + + +TEST_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class TestClientCert(unittest.TestCase): + def setUp(self): + certfn = os.path.join(TEST_DIR, 'testcert.pem') + cacertfn = os.path.join(TEST_DIR, 'testdata', 'clientcert', 'ca.crt') + self.httpd = compat_http_server.HTTPServer(('127.0.0.1', 0), HTTPTestRequestHandler) + self.httpd.socket = ssl.wrap_socket( + self.httpd.socket, cert_reqs=ssl.CERT_REQUIRED, ca_certs=cacertfn, certfile=certfn, server_side=True) + self.port = http_server_port(self.httpd) + self.server_thread = threading.Thread(target=self.httpd.serve_forever) + self.server_thread.daemon = True + self.server_thread.start() + + def test_check_clientcertificate(self): + clientcertfn = os.path.join(TEST_DIR, 'testdata', 'clientcert', 'client.crt') + ydl = YoutubeDL({'logger': FakeLogger(), 'clientcertificate': clientcertfn}) + r = ydl.extract_info('https://127.0.0.1:%d/video.html' % self.port) + self.assertEqual(r['entries'][0]['url'], 'https://127.0.0.1:%d/vid.mp4' % self.port) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/testdata/clientcert/ca.crt b/test/testdata/clientcert/ca.crt new file mode 100644 index 000000000..9f7160254 --- /dev/null +++ b/test/testdata/clientcert/ca.crt @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBnTCCAUOgAwIBAgIUN4jSR5qgSLKJs4lWdBUQPiOyUPEwCgYIKoZIzj0EAwIw +JDETMBEGA1UECgwKWW91dHViZS1ETDENMAsGA1UEAwwEVGVzdDAeFw0yMTA3MTkx +NTE1MzJaFw0zODAxMTgxNTE1MzJaMCQxEzARBgNVBAoMCllvdXR1YmUtREwxDTAL +BgNVBAMMBFRlc3QwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQ8DNZrIDTIQ4mN +IofBxPIDF2nZUKKAUPAMyn6ntXh99WhLRO5caGKTPWip+LspF5j2uyd2DAcAKMZ5 +170qIbSCo1MwUTAdBgNVHQ4EFgQUj+rfMTfIDHufM94pYwjvI8pZKlYwHwYDVR0j +BBgwFoAUj+rfMTfIDHufM94pYwjvI8pZKlYwDwYDVR0TAQH/BAUwAwEB/zAKBggq +hkjOPQQDAgNIADBFAiAwhl8mpPiZoAXWfPHSZaxiLPjy2m4pZK70O0BHnxmSJQIh +AOeQgZh0j0SkZW0kXHBPWguCgvVm5tqQPQJCevgNDKWP +-----END CERTIFICATE----- diff --git a/test/testdata/clientcert/client.crt b/test/testdata/clientcert/client.crt new file mode 100644 index 000000000..c3cd01bab --- /dev/null +++ b/test/testdata/clientcert/client.crt @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIIBSTCB8AIUE2DY1KuqtYWIi0KYeSYvta9sV+swCgYIKoZIzj0EAwIwJDETMBEG +A1UECgwKWW91dHViZS1ETDENMAsGA1UEAwwEVGVzdDAeFw0yMTA3MTkxNTE2MjZa +Fw0zODAxMTgxNTE2MjZaMCsxEzARBgNVBAoMCllvdXR1YmUtREwxFDASBgNVBAMM +C1Rlc3QgQ2xpZW50MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEc0ldxFETUFCS +CsMq01OUEYp9zkPbXZ9IkTUu1RQhliuPYCsc4Q+UZ8z+Ttcyqa76jAMcmQWh+n2P +4i7uCDvZ8zAKBggqhkjOPQQDAgNIADBFAiEAiuQWNv6F7EO+bZGhDDxhUkGdhWOy +36YbZa+BZ8CYae0CIBVfdEnrG5M9tc6PZjXiXgoUMUrnPnRXs76ihQ55hHPW +-----END CERTIFICATE----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIBVDCR/z/PuVFzGKFCOt9GYGpwQ8vJTXAj59jPwP4OFVoAoGCCqGSM49 +AwEHoUQDQgAEc0ldxFETUFCSCsMq01OUEYp9zkPbXZ9IkTUu1RQhliuPYCsc4Q+U +Z8z+Ttcyqa76jAMcmQWh+n2P4i7uCDvZ8w== +-----END EC PRIVATE KEY----- diff --git a/test/testdata/clientcert/instructions.txt b/test/testdata/clientcert/instructions.txt new file mode 100644 index 000000000..fbca92980 --- /dev/null +++ b/test/testdata/clientcert/instructions.txt @@ -0,0 +1,11 @@ +#https://blog.devolutions.net/2020/07/tutorial-how-to-generate-secure-self-signed-server-and-client-certificates-with-openssl +# Adapt the commands below +# 6027 days from the time of signing to the day before Y2038 +# Recalculate or use -preserve_dates if re-signing, until +# 32-bit time_t is not an issue +#openssl ecparam -name prime256v1 -genkey -noout -out ca.key +#openssl req -new -x509 -sha256 -days 6027 -key ca.key -out ca.crt +#openssl ecparam -name prime256v1 -genkey -noout -out client.key +#openssl req -new -sha256 -key client.key -out client.csr +#openssl x509 -req -in client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out client.crt -days 6027 -sha256 +#cat client.key >> client.crt From f2c3bef77b9038a4bc31b94b547e7b97227d372e Mon Sep 17 00:00:00 2001 From: df Date: Mon, 19 Jul 2021 16:44:13 +0100 Subject: [PATCH 3/4] Update --client-certificate option help --- README.md | 3 +++ youtube_dl/options.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2841ed68f..ccca4df19 100644 --- a/README.md +++ b/README.md @@ -404,6 +404,9 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo -2, --twofactor TWOFACTOR Two-factor authentication code -n, --netrc Use .netrc authentication data --video-password PASSWORD Video password (vimeo, youku) + --client-certificate Path to a single certificate file in + PEM format, used to authenticate to the + site (including private key) ## Adobe Pass Options: --ap-mso MSO Adobe Pass multiple-system operator (TV diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 399160c2f..2898f1a77 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -373,7 +373,7 @@ def parseOpts(overrideArguments=None): authentication.add_option( '--client-certificate', dest='clientcertificate', metavar='PATH', - help='Path to a single certificate file in PEM format, used to authenticate to the site') + help='Path to a single certificate file in PEM format, used to authenticate to the site (including private key)') adobe_pass = optparse.OptionGroup(parser, 'Adobe Pass Options') adobe_pass.add_option( From eaf29d53e4df3ff8be9d78e84c7c556b01bf7034 Mon Sep 17 00:00:00 2001 From: df Date: Tue, 20 Jul 2021 19:17:48 +0100 Subject: [PATCH 4/4] Disable hopeless server certificate validation by test client --- test/test_clientcert.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/test_clientcert.py b/test/test_clientcert.py index e054e8129..db1f8af91 100644 --- a/test/test_clientcert.py +++ b/test/test_clientcert.py @@ -39,7 +39,11 @@ class TestClientCert(unittest.TestCase): def test_check_clientcertificate(self): clientcertfn = os.path.join(TEST_DIR, 'testdata', 'clientcert', 'client.crt') - ydl = YoutubeDL({'logger': FakeLogger(), 'clientcertificate': clientcertfn}) + ydl = YoutubeDL({'logger': FakeLogger(), 'clientcertificate': clientcertfn, + # Disable client-side validation of unacceptable self-signed testcert.pem + # The test is of a check on the server side, so unaffected + 'nocheckcertificate': True, + }) r = ydl.extract_info('https://127.0.0.1:%d/video.html' % self.port) self.assertEqual(r['entries'][0]['url'], 'https://127.0.0.1:%d/vid.mp4' % self.port)