From ddacd2d6cf40327470d97e9e3fd1f6e9555876a7 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Wed, 2 Jun 2021 21:49:27 +0100 Subject: [PATCH 01/25] added --cookies-from-browser --- test/test_cookies.py | 42 ++++ youtube_dl/YoutubeDL.py | 11 +- youtube_dl/__init__.py | 1 + youtube_dl/cookies.py | 475 ++++++++++++++++++++++++++++++++++++++++ youtube_dl/options.py | 5 + 5 files changed, 526 insertions(+), 8 deletions(-) create mode 100644 test/test_cookies.py create mode 100644 youtube_dl/cookies.py diff --git a/test/test_cookies.py b/test/test_cookies.py new file mode 100644 index 000000000..c30683ef0 --- /dev/null +++ b/test/test_cookies.py @@ -0,0 +1,42 @@ +import unittest +from unittest import mock + +from youtube_dl.cookies import LinuxChromeCookieDecryptor, WindowsChromeCookieDecryptor, CRYPTO_AVAILABLE, \ + MacChromeCookieDecryptor + + +@unittest.skipIf(not CRYPTO_AVAILABLE, 'cryptography library not available') +class TestCookies(unittest.TestCase): + @mock.patch('youtube_dl.cookies._get_linux_keyring_password') + def test_chrome_cookie_decryptor_linux_v10(self, mock_get_keyring_password): + mock_get_keyring_password.return_value = '' + encrypted_value = b'v10\xccW%\xcd\xe6\xe6\x9fM" \xa7\xb0\xca\xe4\x07\xd6' + value = 'USD' + decryptor = LinuxChromeCookieDecryptor('Chrome') + assert decryptor.decrypt(encrypted_value) == value + + @mock.patch('youtube_dl.cookies._get_linux_keyring_password') + def test_chrome_cookie_decryptor_linux_v11(self, mock_get_keyring_password): + mock_get_keyring_password.return_value = '' + encrypted_value = b'v11#\x81\x10>`w\x8f)\xc0\xb2\xc1\r\xf4\x1al\xdd\x93\xfd\xf8\xf8N\xf2\xa9\x83\xf1\xe9o\x0elVQd' + value = 'tz=Europe.London' + decryptor = LinuxChromeCookieDecryptor('Chrome') + assert decryptor.decrypt(encrypted_value) == value + + @mock.patch('youtube_dl.cookies._get_windows_v10_password') + def test_chrome_cookie_decryptor_windows_v10(self, mock_get_windows_v10_password): + mock_get_windows_v10_password.return_value = b'Y\xef\xad\xad\xeerp\xf0Y\xe6\x9b\x12\xc2 0: + failed_message = ' ({} could not be decrypted)'.format(failed_cookies) + else: + failed_message = '' + print('extracted {} cookies from {}{}'.format(len(jar), browser_name, failed_message)) + return jar + finally: + if cursor is not None: + cursor.connection.close() + + +class ChromeCookieDecryptor(ABC): + """ + Overview: + + Linux: + - cookies are either v10 or v11 + - v10: AES-CBC encrypted with a fixed key + - v11: AES-CBC encrypted with an OS protected key (keyring) + - v11 keys can be stored in various places depending on the activate desktop environment [2] + + Mac: + - cookies are either v10 or not v10 + - v10: AES-CBC encrypted (with more iterations than Linux) with an OS protected key (keyring) + - not v10: 'old data' stored as plaintext + + Windows: + - cookies are either v10 or not v10 + - v10: AES-GCM encrypted with a key which is encrypted with DPAPI + - not v10: encrypted with DPAPI + + Sources: + - [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/ + - [2] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_linux.cc + - KeyStorageLinux::CreateService + """ + + @abstractmethod + def decrypt(self, encrypted_value): + raise NotImplementedError + + +def get_cookie_decryptor(browser_root, browser_keyring_name): + if sys.platform in ('linux', 'linux2'): + return LinuxChromeCookieDecryptor(browser_keyring_name) + elif sys.platform == 'darwin': + return MacChromeCookieDecryptor(browser_keyring_name) + elif sys.platform == 'win32': + return WindowsChromeCookieDecryptor(browser_root) + else: + raise NotImplementedError('Chrome cookie decryption is not supported ' + 'on this platform: {}'.format(sys.platform)) + + +class LinuxChromeCookieDecryptor(ChromeCookieDecryptor): + def __init__(self, browser_keyring_name): + self._v10_key = None + self._v11_key = None + if CRYPTO_AVAILABLE: + # values from + # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_linux.cc + self._v10_key = PBKDF2HMAC(algorithm=SHA1(), length=16, + salt=b'saltysalt', iterations=1).derive(b'peanuts') + + if KEYRING_AVAILABLE: + password = _get_linux_keyring_password(browser_keyring_name) + self._v11_key = PBKDF2HMAC(algorithm=SHA1(), length=16, + salt=b'saltysalt', iterations=1).derive(password.encode('utf-8')) + + def decrypt(self, encrypted_value): + version = encrypted_value[:3] + ciphertext = encrypted_value[3:] + + if version == b'v10': + if self._v10_key is None: + warnings.warn('cannot decrypt cookie as the cryptography module is not installed') + return None + return _decrypt_aes_cbc(ciphertext, self._v10_key) + + elif version == b'v11': + if self._v11_key is None: + warnings.warn('cannot decrypt cookie as the cryptography or keyring modules are not installed') + return None + return _decrypt_aes_cbc(ciphertext, self._v11_key) + + else: + return None + + +class MacChromeCookieDecryptor(ChromeCookieDecryptor): + def __init__(self, browser_keyring_name): + self._v10_key = None + if CRYPTO_AVAILABLE: + # values from + # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm + password = _get_mac_keyring_password(browser_keyring_name) + self._v10_key = PBKDF2HMAC(algorithm=SHA1(), length=16, + salt=b'saltysalt', iterations=1003).derive(password.encode('utf-8')) + + def decrypt(self, encrypted_value): + version = encrypted_value[:3] + ciphertext = encrypted_value[3:] + + if version == b'v10': + if self._v10_key is None: + warnings.warn('cannot decrypt cookie as the cryptography module is not installed') + return None + return _decrypt_aes_cbc(ciphertext, self._v10_key) + + else: + # other prefixes are considered 'old data' which were stored as plaintext + # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm + return encrypted_value + + +class WindowsChromeCookieDecryptor(ChromeCookieDecryptor): + def __init__(self, browser_root): + self._v10_key = _get_windows_v10_password(browser_root) + + def decrypt(self, encrypted_value): + version = encrypted_value[:3] + ciphertext = encrypted_value[3:] + + if version == b'v10': + if self._v10_key is None: + warnings.warn('cannot decrypt cookie') + return None + + # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc + # kNonceLength + nonce_length = 96 // 8 + # boringssl + # EVP_AEAD_AES_GCM_TAG_LEN + authentication_tag_length = 16 + + raw_ciphertext = ciphertext + nonce = raw_ciphertext[:nonce_length] + ciphertext = raw_ciphertext[nonce_length:-authentication_tag_length] + authentication_tag = raw_ciphertext[-authentication_tag_length:] + + return _decrypt_aes_gcm(ciphertext, self._v10_key, nonce, authentication_tag) + + else: + # any other prefix means the data is DPAPI encrypted + # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc + return _decrypt_windows_dpapi(encrypted_value) + + +def _get_linux_keyring_password(browser_keyring_name): + if KEYRING_AVAILABLE: + password = keyring.get_password('{} Keys'.format(browser_keyring_name), + '{} Safe Storage'.format(browser_keyring_name)) + if password is None: + # this sometimes occurs in KDE because chrome does not check hasEntry and instead + # just tries to read the value (which kwallet returns "") whereas keyring checks hasEntry + # to verify this: + # dbus-monitor "interface='org.kde.KWallet'" "type=method_return" + # while starting chrome. + # this may be a bug as the intended behaviour is to generate a random password and store + # it, but that doesn't matter here. + password = '' + return password + + +def _get_mac_keyring_password(browser_keyring_name): + if KEYRING_AVAILABLE: + return keyring.get_password('{} Safe Storage'.format(browser_keyring_name), browser_keyring_name) + else: + proc = subprocess.Popen(['security', 'find-generic-password', + '-w', # write password to stdout + '-a', browser_keyring_name, # match 'account' + '-s', '{} Safe Storage'.format(browser_keyring_name)], # match 'service' + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL) + proc.wait() + if proc.returncode == 0: + return proc.stdout.read().decode('utf-8').strip() + else: + return None + + +def _get_windows_v10_password(browser_root): + path = _find_most_recently_used_file(browser_root, 'Local State') + if path is None: + print('could not find local state file') + return None + with open(path, 'r') as f: + data = json.load(f) + try: + base64_password = data['os_crypt']['encrypted_key'] + except KeyError: + return None + encrypted_password = base64.b64decode(base64_password) + prefix = b'DPAPI' + if not encrypted_password.startswith(prefix): + return None + return _decrypt_windows_dpapi(encrypted_password[len(prefix):]) + + +def _decrypt_aes_cbc(ciphertext, key): + cipher = Cipher(algorithm=AES(key), mode=CBC(initialization_vector=b' ' * 16)) + decryptor = cipher.decryptor() + plaintext = decryptor.update(ciphertext) + decryptor.finalize() + padding_length = plaintext[-1] + try: + return plaintext[:-padding_length].decode('utf-8') + except UnicodeDecodeError: + warnings.warn('failed to decrypt cookie because UTF-8 decoding failed. Possibly the key is wrong?') + return None + + +def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag): + cipher = Cipher(algorithm=AES(key), mode=GCM(nonce, tag=authentication_tag)) + decryptor = cipher.decryptor() + plaintext = decryptor.update(ciphertext) + decryptor.finalize() + try: + return plaintext.decode('utf-8') + except UnicodeDecodeError: + warnings.warn('failed to decrypt cookie because UTF-8 decoding failed. Possibly the key is wrong?') + return None + + +def _decrypt_windows_dpapi(ciphertext): + """ + References: + - https://docs.microsoft.com/en-us/windows/win32/api/dpapi/nf-dpapi-cryptunprotectdata + """ + class DATA_BLOB(ctypes.Structure): + _fields_ = [('cbData', DWORD), + ('pbData', ctypes.POINTER(ctypes.c_char))] + + buffer = ctypes.create_string_buffer(ciphertext) + blob_in = DATA_BLOB(ctypes.sizeof(buffer), buffer) + blob_out = DATA_BLOB() + ret = ctypes.windll.crypt32.CryptUnprotectData( + ctypes.byref(blob_in), # pDataIn + None, # ppszDataDescr: human readable description of pDataIn + None, # pOptionalEntropy: salt? + None, # pvReserved: must be NULL + None, # pPromptStruct: information about prompts to display + 0, # dwFlags + ctypes.byref(blob_out) # pDataOut + ) + if not ret: + print('failed to decrypt with DPAPI') + return None + + result = ctypes.string_at(blob_out.pbData, blob_out.cbData) + ctypes.windll.kernel32.LocalFree(blob_out.pbData) + return result + + +def _config_home(): + return os.environ.get('XDG_CONFIG_HOME', os.path.expanduser('~/.config')) + + +def _open_database_copy(database_path, tmpdir): + # cannot open sqlite databases if they are already in use (e.g. by the browser) + database_copy_path = os.path.join(tmpdir, 'temporary.sqlite') + shutil.copy(database_path, database_copy_path) + conn = sqlite3.connect(database_copy_path) + return conn.cursor() + + +def _find_most_recently_used_file(root, filename): + # if there are multiple browser profiles, take the most recently used one + paths = glob.iglob(os.path.join(root, '**', filename), recursive=True) + paths = [path for path in paths if os.path.isfile(path)] + return None if not paths else max(paths, key=lambda path: os.lstat(path).st_mtime) + + +def _merge_cookie_jars(jars): + output_jar = YoutubeDLCookieJar() + for jar in jars: + for cookie in jar: + output_jar.set_cookie(cookie) + if jar.filename is not None: + output_jar.filename = jar.filename + return output_jar + + +def load_cookies(cookie_file, browser): + cookie_jars = [] + if browser is not None: + cookie_jars.append(extract_cookies_from_browser(browser)) + + if cookie_file is not None: + cookie_file = expand_path(cookie_file) + jar = YoutubeDLCookieJar(cookie_file) + if os.access(cookie_file, os.R_OK): + jar.load(ignore_discard=True, ignore_expires=True) + cookie_jars.append(jar) + + return _merge_cookie_jars(cookie_jars) diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 0a0641bd4..457a041e2 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -17,6 +17,7 @@ from .utils import ( preferredencoding, write_string, ) +from .cookies import SUPPORTED_BROWSERS from .version import __version__ @@ -757,6 +758,10 @@ def parseOpts(overrideArguments=None): '--cookies', dest='cookiefile', metavar='FILE', help='File to read cookies from and dump cookie jar in') + filesystem.add_option( + '--cookies-from-browser', + dest='cookiesfrombrowser', metavar='BROWSER', + help='Browser to load cookies from: {}'.format(', '.join(SUPPORTED_BROWSERS))) filesystem.add_option( '--cache-dir', dest='cachedir', default=None, metavar='DIR', help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.') From 0dd9df2e77b116982eeb1806901c4384cf79a9de Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Wed, 2 Jun 2021 22:18:08 +0100 Subject: [PATCH 02/25] fixed python2 compatibility --- youtube_dl/compat.py | 20 ++++++++++++++++++++ youtube_dl/cookies.py | 29 ++++++++++++++--------------- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index 9e45c454b..27b0ead30 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -19,6 +19,7 @@ import socket import struct import subprocess import sys +import tempfile import xml.etree.ElementTree @@ -2923,6 +2924,24 @@ except TypeError: # Python 2.6 yield n n += step + +try: + from tempfile import TemporaryDirectory as Compat_TemporaryDirectory +except ImportError: + class Compat_TemporaryDirectory: + def __init__(self, prefix=None): + self._prefix = prefix + self._path = None + + def __enter__(self): + self._path = tempfile.mkdtemp(prefix=self._prefix) + return self._path + + def __exit__(self, exc_type, exc_val, exc_tb): + if self._path is not None: + shutil.rmtree(self._path) + + if sys.version_info >= (3, 0): from tokenize import tokenize as compat_tokenize_tokenize else: @@ -3040,6 +3059,7 @@ __all__ = [ 'compat_struct_pack', 'compat_struct_unpack', 'compat_subprocess_get_DEVNULL', + 'compat_TemporaryDirectory', 'compat_tokenize_tokenize', 'compat_urllib_error', 'compat_urllib_parse', diff --git a/youtube_dl/cookies.py b/youtube_dl/cookies.py index 248e0f93a..9e35201fd 100644 --- a/youtube_dl/cookies.py +++ b/youtube_dl/cookies.py @@ -1,6 +1,4 @@ -import base64 import ctypes -import glob import json import os import shutil @@ -8,11 +6,8 @@ import sqlite3 import subprocess import sys import warnings -from abc import ABC, abstractmethod -from ctypes.wintypes import DWORD -from tempfile import TemporaryDirectory -from youtube_dl.compat import compat_cookiejar_Cookie +from youtube_dl.compat import compat_cookiejar_Cookie, compat_b64decode, Compat_TemporaryDirectory, compat_ord from youtube_dl.utils import YoutubeDLCookieJar, expand_path try: @@ -36,7 +31,7 @@ SUPPORTED_BROWSERS = ['firefox', 'chrome', 'chrome_beta', 'chrome_dev', 'chromiu 'edge', 'edge_beta'] -def extract_cookies_from_browser(browser_name: str): +def extract_cookies_from_browser(browser_name): if browser_name == 'firefox': return _extract_firefox_cookies() elif browser_name in ('chrome', 'chrome_beta', 'chrome_dev', 'chromium', @@ -62,7 +57,7 @@ def _extract_firefox_cookies(): if cookie_database_path is None: raise FileNotFoundError('could not find firefox cookies database') - with TemporaryDirectory(prefix='youtube_dl') as tmpdir: + with Compat_TemporaryDirectory(prefix='youtube_dl') as tmpdir: cursor = None try: cursor = _open_database_copy(cookie_database_path, tmpdir) @@ -153,7 +148,7 @@ def _extract_chrome_cookies(browser_name): decryptor = get_cookie_decryptor(config['browser_dir'], config['keyring_name']) - with TemporaryDirectory(prefix='youtube_dl') as tmpdir: + with Compat_TemporaryDirectory(prefix='youtube_dl') as tmpdir: cursor = None try: cursor = _open_database_copy(cookie_database_path, tmpdir) @@ -190,7 +185,7 @@ def _extract_chrome_cookies(browser_name): cursor.connection.close() -class ChromeCookieDecryptor(ABC): +class ChromeCookieDecryptor: """ Overview: @@ -216,7 +211,6 @@ class ChromeCookieDecryptor(ABC): - KeyStorageLinux::CreateService """ - @abstractmethod def decrypt(self, encrypted_value): raise NotImplementedError @@ -371,7 +365,7 @@ def _get_windows_v10_password(browser_root): base64_password = data['os_crypt']['encrypted_key'] except KeyError: return None - encrypted_password = base64.b64decode(base64_password) + encrypted_password = compat_b64decode(base64_password) prefix = b'DPAPI' if not encrypted_password.startswith(prefix): return None @@ -382,7 +376,7 @@ def _decrypt_aes_cbc(ciphertext, key): cipher = Cipher(algorithm=AES(key), mode=CBC(initialization_vector=b' ' * 16)) decryptor = cipher.decryptor() plaintext = decryptor.update(ciphertext) + decryptor.finalize() - padding_length = plaintext[-1] + padding_length = compat_ord(plaintext[-1]) try: return plaintext[:-padding_length].decode('utf-8') except UnicodeDecodeError: @@ -406,6 +400,8 @@ def _decrypt_windows_dpapi(ciphertext): References: - https://docs.microsoft.com/en-us/windows/win32/api/dpapi/nf-dpapi-cryptunprotectdata """ + from ctypes.wintypes import DWORD + class DATA_BLOB(ctypes.Structure): _fields_ = [('cbData', DWORD), ('pbData', ctypes.POINTER(ctypes.c_char))] @@ -445,8 +441,11 @@ def _open_database_copy(database_path, tmpdir): def _find_most_recently_used_file(root, filename): # if there are multiple browser profiles, take the most recently used one - paths = glob.iglob(os.path.join(root, '**', filename), recursive=True) - paths = [path for path in paths if os.path.isfile(path)] + paths = [] + for root, dirs, files in os.walk(root): + for file in files: + if file == filename: + paths.append(os.path.join(root, file)) return None if not paths else max(paths, key=lambda path: os.lstat(path).st_mtime) From 82918442019cb3f4d2dd6dcf53aa6416909e95c8 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Wed, 2 Jun 2021 22:48:21 +0100 Subject: [PATCH 03/25] added python2 support to tests --- test/test_cookies.py | 70 ++++++++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/test/test_cookies.py b/test/test_cookies.py index c30683ef0..67a13b938 100644 --- a/test/test_cookies.py +++ b/test/test_cookies.py @@ -1,42 +1,54 @@ import unittest -from unittest import mock +from youtube_dl import cookies from youtube_dl.cookies import LinuxChromeCookieDecryptor, WindowsChromeCookieDecryptor, CRYPTO_AVAILABLE, \ MacChromeCookieDecryptor +class MonkeyPatch: + def __init__(self, module, name, temp_value): + self._module = module + self._name = name + self._temp_value = temp_value + self._backup_value = None + + def __enter__(self): + self._backup_value = getattr(self._module, self._name) + setattr(self._module, self._name, self._temp_value) + + def __exit__(self, exc_type, exc_val, exc_tb): + setattr(self._module, self._name, self._backup_value) + + @unittest.skipIf(not CRYPTO_AVAILABLE, 'cryptography library not available') class TestCookies(unittest.TestCase): - @mock.patch('youtube_dl.cookies._get_linux_keyring_password') - def test_chrome_cookie_decryptor_linux_v10(self, mock_get_keyring_password): - mock_get_keyring_password.return_value = '' - encrypted_value = b'v10\xccW%\xcd\xe6\xe6\x9fM" \xa7\xb0\xca\xe4\x07\xd6' - value = 'USD' - decryptor = LinuxChromeCookieDecryptor('Chrome') - assert decryptor.decrypt(encrypted_value) == value + def test_chrome_cookie_decryptor_linux_v10(self): + with MonkeyPatch(cookies, '_get_linux_keyring_password', lambda *args, **kwargs: ''): + encrypted_value = b'v10\xccW%\xcd\xe6\xe6\x9fM" \xa7\xb0\xca\xe4\x07\xd6' + value = 'USD' + decryptor = LinuxChromeCookieDecryptor('Chrome') + assert decryptor.decrypt(encrypted_value) == value - @mock.patch('youtube_dl.cookies._get_linux_keyring_password') - def test_chrome_cookie_decryptor_linux_v11(self, mock_get_keyring_password): - mock_get_keyring_password.return_value = '' - encrypted_value = b'v11#\x81\x10>`w\x8f)\xc0\xb2\xc1\r\xf4\x1al\xdd\x93\xfd\xf8\xf8N\xf2\xa9\x83\xf1\xe9o\x0elVQd' - value = 'tz=Europe.London' - decryptor = LinuxChromeCookieDecryptor('Chrome') - assert decryptor.decrypt(encrypted_value) == value + def test_chrome_cookie_decryptor_linux_v11(self): + with MonkeyPatch(cookies, '_get_linux_keyring_password', lambda *args, **kwargs: ''): + encrypted_value = b'v11#\x81\x10>`w\x8f)\xc0\xb2\xc1\r\xf4\x1al\xdd\x93\xfd\xf8\xf8N\xf2\xa9\x83\xf1\xe9o\x0elVQd' + value = 'tz=Europe.London' + decryptor = LinuxChromeCookieDecryptor('Chrome') + assert decryptor.decrypt(encrypted_value) == value - @mock.patch('youtube_dl.cookies._get_windows_v10_password') - def test_chrome_cookie_decryptor_windows_v10(self, mock_get_windows_v10_password): - mock_get_windows_v10_password.return_value = b'Y\xef\xad\xad\xeerp\xf0Y\xe6\x9b\x12\xc2 Date: Thu, 3 Jun 2021 20:08:45 +0100 Subject: [PATCH 04/25] switched to using pycryptodome as it is already an optional dependency of youtube_dl --- test/test_cookies.py | 8 ++++++ youtube_dl/cookies.py | 61 +++++++++++++++++++++++-------------------- 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/test/test_cookies.py b/test/test_cookies.py index 67a13b938..d217e46ef 100644 --- a/test/test_cookies.py +++ b/test/test_cookies.py @@ -22,6 +22,14 @@ class MonkeyPatch: @unittest.skipIf(not CRYPTO_AVAILABLE, 'cryptography library not available') class TestCookies(unittest.TestCase): + def test_chrome_cookie_decryptor_linux_derive_key(self): + key = LinuxChromeCookieDecryptor.derive_key('abc') + assert key == b'7\xa1\xec\xd4m\xfcA\xc7\xb19Z\xd0\x19\xdcM\x17' + + def test_chrome_cookie_decryptor_mac_derive_key(self): + key = MacChromeCookieDecryptor.derive_key('abc') + assert key == b'Y\xe2\xc0\xd0P\xf6\xf4\xe1l\xc1\x8cQ\xcb|\xcdY' + def test_chrome_cookie_decryptor_linux_v10(self): with MonkeyPatch(cookies, '_get_linux_keyring_password', lambda *args, **kwargs: ''): encrypted_value = b'v10\xccW%\xcd\xe6\xe6\x9fM" \xa7\xb0\xca\xe4\x07\xd6' diff --git a/youtube_dl/cookies.py b/youtube_dl/cookies.py index 9e35201fd..cefd55403 100644 --- a/youtube_dl/cookies.py +++ b/youtube_dl/cookies.py @@ -11,11 +11,9 @@ from youtube_dl.compat import compat_cookiejar_Cookie, compat_b64decode, Compat_ from youtube_dl.utils import YoutubeDLCookieJar, expand_path try: - from cryptography.hazmat.primitives.ciphers import Cipher - from cryptography.hazmat.primitives.ciphers.algorithms import AES - from cryptography.hazmat.primitives.ciphers.modes import CBC, GCM - from cryptography.hazmat.primitives.hashes import SHA1 - from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC + from Crypto.Cipher import AES + from Crypto.Protocol.KDF import PBKDF2 + from Crypto.Hash import SHA1 CRYPTO_AVAILABLE = True except ImportError: CRYPTO_AVAILABLE = False @@ -197,7 +195,7 @@ class ChromeCookieDecryptor: Mac: - cookies are either v10 or not v10 - - v10: AES-CBC encrypted (with more iterations than Linux) with an OS protected key (keyring) + - v10: AES-CBC encrypted with an OS protected key (keyring) and more key derivation iterations than linux - not v10: 'old data' stored as plaintext Windows: @@ -232,15 +230,15 @@ class LinuxChromeCookieDecryptor(ChromeCookieDecryptor): self._v10_key = None self._v11_key = None if CRYPTO_AVAILABLE: - # values from - # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_linux.cc - self._v10_key = PBKDF2HMAC(algorithm=SHA1(), length=16, - salt=b'saltysalt', iterations=1).derive(b'peanuts') - + self._v10_key = self.derive_key('peanuts') if KEYRING_AVAILABLE: - password = _get_linux_keyring_password(browser_keyring_name) - self._v11_key = PBKDF2HMAC(algorithm=SHA1(), length=16, - salt=b'saltysalt', iterations=1).derive(password.encode('utf-8')) + self._v11_key = self.derive_key(_get_linux_keyring_password(browser_keyring_name)) + + @staticmethod + def derive_key(password): + # values from + # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_linux.cc + return PBKDF2(password, salt=b'saltysalt', dkLen=16, count=1, hmac_hash_module=SHA1) def decrypt(self, encrypted_value): version = encrypted_value[:3] @@ -248,13 +246,13 @@ class LinuxChromeCookieDecryptor(ChromeCookieDecryptor): if version == b'v10': if self._v10_key is None: - warnings.warn('cannot decrypt cookie as the cryptography module is not installed') + warnings.warn('cannot decrypt cookie as the module `pycryptodome` is not installed') return None return _decrypt_aes_cbc(ciphertext, self._v10_key) elif version == b'v11': if self._v11_key is None: - warnings.warn('cannot decrypt cookie as the cryptography or keyring modules are not installed') + warnings.warn('cannot decrypt cookie as the `pycryptodome` or `keyring` modules are not installed') return None return _decrypt_aes_cbc(ciphertext, self._v11_key) @@ -266,11 +264,13 @@ class MacChromeCookieDecryptor(ChromeCookieDecryptor): def __init__(self, browser_keyring_name): self._v10_key = None if CRYPTO_AVAILABLE: - # values from - # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm - password = _get_mac_keyring_password(browser_keyring_name) - self._v10_key = PBKDF2HMAC(algorithm=SHA1(), length=16, - salt=b'saltysalt', iterations=1003).derive(password.encode('utf-8')) + self._v10_key = self.derive_key(_get_mac_keyring_password(browser_keyring_name)) + + @staticmethod + def derive_key(password): + # values from + # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm + return PBKDF2(password, salt=b'saltysalt', dkLen=16, count=1003, hmac_hash_module=SHA1) def decrypt(self, encrypted_value): version = encrypted_value[:3] @@ -278,7 +278,7 @@ class MacChromeCookieDecryptor(ChromeCookieDecryptor): if version == b'v10': if self._v10_key is None: - warnings.warn('cannot decrypt cookie as the cryptography module is not installed') + warnings.warn('cannot decrypt cookie as the `pycryptodome` module is not installed') return None return _decrypt_aes_cbc(ciphertext, self._v10_key) @@ -372,10 +372,9 @@ def _get_windows_v10_password(browser_root): return _decrypt_windows_dpapi(encrypted_password[len(prefix):]) -def _decrypt_aes_cbc(ciphertext, key): - cipher = Cipher(algorithm=AES(key), mode=CBC(initialization_vector=b' ' * 16)) - decryptor = cipher.decryptor() - plaintext = decryptor.update(ciphertext) + decryptor.finalize() +def _decrypt_aes_cbc(ciphertext, key, initialization_vector=b' ' * 16): + cipher = AES.new(key, AES.MODE_CBC, iv=initialization_vector) + plaintext = cipher.decrypt(ciphertext) padding_length = compat_ord(plaintext[-1]) try: return plaintext[:-padding_length].decode('utf-8') @@ -385,9 +384,13 @@ def _decrypt_aes_cbc(ciphertext, key): def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag): - cipher = Cipher(algorithm=AES(key), mode=GCM(nonce, tag=authentication_tag)) - decryptor = cipher.decryptor() - plaintext = decryptor.update(ciphertext) + decryptor.finalize() + cipher = AES.new(key, AES.MODE_GCM, nonce) + try: + plaintext = cipher.decrypt_and_verify(ciphertext, authentication_tag) + except ValueError: + warnings.warn('failed to decrypt cookie because the MAC check failed. Possibly the key is wrong?') + return None + try: return plaintext.decode('utf-8') except UnicodeDecodeError: From 97977004d674da2f00916e54107a3882a737f64c Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 3 Jun 2021 20:22:13 +0100 Subject: [PATCH 05/25] improved logging --- youtube_dl/YoutubeDL.py | 2 +- youtube_dl/cookies.py | 70 +++++++++++++++++++++++++++-------------- 2 files changed, 47 insertions(+), 25 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index bd886ef1b..d8d8c1435 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -2371,7 +2371,7 @@ class YoutubeDL(object): opts_cookiefile = self.params.get('cookiefile') opts_proxy = self.params.get('proxy') - self.cookiejar = load_cookies(opts_cookiefile, opts_cookiesfrombrowser) + self.cookiejar = load_cookies(opts_cookiefile, opts_cookiesfrombrowser, self) cookie_processor = YoutubeDLCookieProcessor(self.cookiejar) if opts_proxy is not None: diff --git a/youtube_dl/cookies.py b/youtube_dl/cookies.py index cefd55403..897e3c4ee 100644 --- a/youtube_dl/cookies.py +++ b/youtube_dl/cookies.py @@ -29,18 +29,26 @@ SUPPORTED_BROWSERS = ['firefox', 'chrome', 'chrome_beta', 'chrome_dev', 'chromiu 'edge', 'edge_beta'] -def extract_cookies_from_browser(browser_name): +class Logger: + def info(self, message): + print(message) + + def error(self, message): + print(message, file=sys.stderr) + + +def extract_cookies_from_browser(browser_name, logger=Logger()): if browser_name == 'firefox': - return _extract_firefox_cookies() + return _extract_firefox_cookies(logger) elif browser_name in ('chrome', 'chrome_beta', 'chrome_dev', 'chromium', 'brave', 'opera', 'edge', 'edge_beta'): - return _extract_chrome_cookies(browser_name) + return _extract_chrome_cookies(browser_name, logger) else: raise ValueError('unknown browser: {}'.format(browser_name)) -def _extract_firefox_cookies(): - print('extracting cookies from firefox') +def _extract_firefox_cookies(logger): + logger.info('extracting cookies from firefox') if sys.platform in ('linux', 'linux2'): root = os.path.expanduser('~/.mozilla/firefox') @@ -68,14 +76,14 @@ def _extract_firefox_cookies(): path=path, path_specified=bool(path), secure=is_secure, expires=expiry, discard=False, comment=None, comment_url=None, rest={}) jar.set_cookie(cookie) - print('extracted {} cookies from firefox'.format(len(jar))) + logger.info('extracted {} cookies from firefox'.format(len(jar))) return jar finally: if cursor is not None: cursor.connection.close() -def _get_browser_settings(browser_name): +def _get_chrome_like_browser_settings(browser_name): # https://chromium.googlesource.com/chromium/src/+/HEAD/docs/user_data_dir.md if sys.platform in ('linux', 'linux2'): config = _config_home() @@ -136,15 +144,15 @@ def _get_browser_settings(browser_name): } -def _extract_chrome_cookies(browser_name): - print('extracting cookies from {}'.format(browser_name)) - config = _get_browser_settings(browser_name) +def _extract_chrome_cookies(browser_name, logger): + logger.info('extracting cookies from {}'.format(browser_name)) + config = _get_chrome_like_browser_settings(browser_name) cookie_database_path = _find_most_recently_used_file(config['browser_dir'], 'Cookies') if cookie_database_path is None: raise FileNotFoundError('could not find {} cookies database'.format(browser_name)) - decryptor = get_cookie_decryptor(config['browser_dir'], config['keyring_name']) + decryptor = get_cookie_decryptor(config['browser_dir'], config['keyring_name'], logger) with Compat_TemporaryDirectory(prefix='youtube_dl') as tmpdir: cursor = None @@ -176,7 +184,7 @@ def _extract_chrome_cookies(browser_name): failed_message = ' ({} could not be decrypted)'.format(failed_cookies) else: failed_message = '' - print('extracted {} cookies from {}{}'.format(len(jar), browser_name, failed_message)) + logger.info('extracted {} cookies from {}{}'.format(len(jar), browser_name, failed_message)) return jar finally: if cursor is not None: @@ -213,13 +221,13 @@ class ChromeCookieDecryptor: raise NotImplementedError -def get_cookie_decryptor(browser_root, browser_keyring_name): +def get_cookie_decryptor(browser_root, browser_keyring_name, logger): if sys.platform in ('linux', 'linux2'): return LinuxChromeCookieDecryptor(browser_keyring_name) elif sys.platform == 'darwin': return MacChromeCookieDecryptor(browser_keyring_name) elif sys.platform == 'win32': - return WindowsChromeCookieDecryptor(browser_root) + return WindowsChromeCookieDecryptor(browser_root, logger) else: raise NotImplementedError('Chrome cookie decryption is not supported ' 'on this platform: {}'.format(sys.platform)) @@ -289,8 +297,9 @@ class MacChromeCookieDecryptor(ChromeCookieDecryptor): class WindowsChromeCookieDecryptor(ChromeCookieDecryptor): - def __init__(self, browser_root): - self._v10_key = _get_windows_v10_password(browser_root) + def __init__(self, browser_root, logger): + self._logger = logger + self._v10_key = _get_windows_v10_password(browser_root, logger) def decrypt(self, encrypted_value): version = encrypted_value[:3] @@ -318,7 +327,7 @@ class WindowsChromeCookieDecryptor(ChromeCookieDecryptor): else: # any other prefix means the data is DPAPI encrypted # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc - return _decrypt_windows_dpapi(encrypted_value) + return _decrypt_windows_dpapi(encrypted_value, self._logger) def _get_linux_keyring_password(browser_keyring_name): @@ -354,22 +363,24 @@ def _get_mac_keyring_password(browser_keyring_name): return None -def _get_windows_v10_password(browser_root): +def _get_windows_v10_password(browser_root, logger): path = _find_most_recently_used_file(browser_root, 'Local State') if path is None: - print('could not find local state file') + logger.error('could not find local state file') return None with open(path, 'r') as f: data = json.load(f) try: base64_password = data['os_crypt']['encrypted_key'] except KeyError: + logger.error('no encrypted key in Local State') return None encrypted_password = compat_b64decode(base64_password) prefix = b'DPAPI' if not encrypted_password.startswith(prefix): + logger.error('invalid key') return None - return _decrypt_windows_dpapi(encrypted_password[len(prefix):]) + return _decrypt_windows_dpapi(encrypted_password[len(prefix):], logger) def _decrypt_aes_cbc(ciphertext, key, initialization_vector=b' ' * 16): @@ -398,7 +409,7 @@ def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag): return None -def _decrypt_windows_dpapi(ciphertext): +def _decrypt_windows_dpapi(ciphertext, logger): """ References: - https://docs.microsoft.com/en-us/windows/win32/api/dpapi/nf-dpapi-cryptunprotectdata @@ -422,7 +433,7 @@ def _decrypt_windows_dpapi(ciphertext): ctypes.byref(blob_out) # pDataOut ) if not ret: - print('failed to decrypt with DPAPI') + logger.info('failed to decrypt with DPAPI') return None result = ctypes.string_at(blob_out.pbData, blob_out.cbData) @@ -462,10 +473,21 @@ def _merge_cookie_jars(jars): return output_jar -def load_cookies(cookie_file, browser): +class YDLLogger(Logger): + def __init__(self, ydl): + self._ydl = ydl + + def info(self, message): + self._ydl.to_screen(message) + + def error(self, message): + self._ydl.to_stderr(message) + + +def load_cookies(cookie_file, browser, ydl): cookie_jars = [] if browser is not None: - cookie_jars.append(extract_cookies_from_browser(browser)) + cookie_jars.append(extract_cookies_from_browser(browser, YDLLogger(ydl))) if cookie_file is not None: cookie_file = expand_path(cookie_file) From 1e3eb70ac4bcd1ca85dd2d36221888be14e664f9 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 3 Jun 2021 21:17:23 +0100 Subject: [PATCH 06/25] added option to specify profile name or path --- youtube_dl/cookies.py | 80 +++++++++++++++++++++++++++++++++---------- youtube_dl/options.py | 2 +- 2 files changed, 62 insertions(+), 20 deletions(-) diff --git a/youtube_dl/cookies.py b/youtube_dl/cookies.py index 897e3c4ee..8d178c267 100644 --- a/youtube_dl/cookies.py +++ b/youtube_dl/cookies.py @@ -30,6 +30,9 @@ SUPPORTED_BROWSERS = ['firefox', 'chrome', 'chrome_beta', 'chrome_dev', 'chromiu class Logger: + def debug(self, message): + print(message) + def info(self, message): print(message) @@ -37,31 +40,34 @@ class Logger: print(message, file=sys.stderr) -def extract_cookies_from_browser(browser_name, logger=Logger()): +def _is_path(value): + return os.path.sep in value + + +def extract_cookies_from_browser(browser_name, profile=None, logger=Logger()): if browser_name == 'firefox': - return _extract_firefox_cookies(logger) + return _extract_firefox_cookies(profile, logger) elif browser_name in ('chrome', 'chrome_beta', 'chrome_dev', 'chromium', 'brave', 'opera', 'edge', 'edge_beta'): - return _extract_chrome_cookies(browser_name, logger) + return _extract_chrome_cookies(browser_name, profile, logger) else: raise ValueError('unknown browser: {}'.format(browser_name)) -def _extract_firefox_cookies(logger): +def _extract_firefox_cookies(profile, logger): logger.info('extracting cookies from firefox') - if sys.platform in ('linux', 'linux2'): - root = os.path.expanduser('~/.mozilla/firefox') - elif sys.platform == 'win32': - root = os.path.expandvars(r'%APPDATA%\Mozilla\Firefox') - elif sys.platform == 'darwin': - root = os.path.expanduser('~/Library/Application Support/Firefox') + if profile is None: + search_root = _firefox_browser_dir() + elif _is_path(profile): + search_root = profile else: - raise ValueError('unsupported platform: {}'.format(sys.platform)) + search_root = os.path.join(_firefox_browser_dir(), profile) - cookie_database_path = _find_most_recently_used_file(root, 'cookies.sqlite') + cookie_database_path = _find_most_recently_used_file(search_root, 'cookies.sqlite') if cookie_database_path is None: - raise FileNotFoundError('could not find firefox cookies database') + raise FileNotFoundError('could not find firefox cookies database in {}'.format(search_root)) + logger.debug('extracting from: "{}"'.format(cookie_database_path)) with Compat_TemporaryDirectory(prefix='youtube_dl') as tmpdir: cursor = None @@ -83,6 +89,17 @@ def _extract_firefox_cookies(logger): cursor.connection.close() +def _firefox_browser_dir(): + if sys.platform in ('linux', 'linux2'): + return os.path.expanduser('~/.mozilla/firefox') + elif sys.platform == 'win32': + return os.path.expandvars(r'%APPDATA%\Mozilla\Firefox') + elif sys.platform == 'darwin': + return os.path.expanduser('~/Library/Application Support/Firefox') + else: + raise ValueError('unsupported platform: {}'.format(sys.platform)) + + def _get_chrome_like_browser_settings(browser_name): # https://chromium.googlesource.com/chromium/src/+/HEAD/docs/user_data_dir.md if sys.platform in ('linux', 'linux2'): @@ -144,13 +161,22 @@ def _get_chrome_like_browser_settings(browser_name): } -def _extract_chrome_cookies(browser_name, logger): +def _extract_chrome_cookies(browser_name, profile, logger): logger.info('extracting cookies from {}'.format(browser_name)) config = _get_chrome_like_browser_settings(browser_name) - cookie_database_path = _find_most_recently_used_file(config['browser_dir'], 'Cookies') + if profile is None: + search_root = config['browser_dir'] + elif _is_path(profile): + search_root = profile + config['browser_dir'] = os.path.dirname(profile) + else: + search_root = os.path.join(config['browser_dir'], profile) + + cookie_database_path = _find_most_recently_used_file(search_root, 'Cookies') if cookie_database_path is None: - raise FileNotFoundError('could not find {} cookies database'.format(browser_name)) + raise FileNotFoundError('could not find {} cookies database in "{}"'.format(browser_name, search_root)) + logger.debug('extracting from: "{}"'.format(cookie_database_path)) decryptor = get_cookie_decryptor(config['browser_dir'], config['keyring_name'], logger) @@ -477,6 +503,10 @@ class YDLLogger(Logger): def __init__(self, ydl): self._ydl = ydl + def debug(self, message): + if self._ydl.params.get('verbose'): + self._ydl.to_screen('[debug] ' + message) + def info(self, message): self._ydl.to_screen(message) @@ -484,10 +514,22 @@ class YDLLogger(Logger): self._ydl.to_stderr(message) -def load_cookies(cookie_file, browser, ydl): +def parse_browser_specification(browser_specification): + parts = browser_specification.split(':') + while len(parts) < 2: + parts.append('') + parts = [p.strip() or None for p in parts] + if not parts[0] or len(parts) > 2: + raise ValueError('invalid browser specification: "{}"'.format(browser_specification)) + browser_name, profile = parts + return browser_name, profile + + +def load_cookies(cookie_file, browser_specification, ydl): cookie_jars = [] - if browser is not None: - cookie_jars.append(extract_cookies_from_browser(browser, YDLLogger(ydl))) + if browser_specification is not None: + browser_name, profile = parse_browser_specification(browser_specification) + cookie_jars.append(extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl))) if cookie_file is not None: cookie_file = expand_path(cookie_file) diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 457a041e2..694bb4136 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -761,7 +761,7 @@ def parseOpts(overrideArguments=None): filesystem.add_option( '--cookies-from-browser', dest='cookiesfrombrowser', metavar='BROWSER', - help='Browser to load cookies from: {}'.format(', '.join(SUPPORTED_BROWSERS))) + help='Load cookies from a user profile of the given web browser: {}. You can specify an alternative user profile name or directory using "BROWSER:PROFILE_NAME" or "BROWSER:PROFILE_PATH"'.format(', '.join(SUPPORTED_BROWSERS))) filesystem.add_option( '--cache-dir', dest='cachedir', default=None, metavar='DIR', help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.') From e9bb3d5e21b9212e9a8fca743f672131a9727ac3 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 3 Jun 2021 21:21:52 +0100 Subject: [PATCH 07/25] added note --- youtube_dl/cookies.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/youtube_dl/cookies.py b/youtube_dl/cookies.py index 8d178c267..1dbe92f14 100644 --- a/youtube_dl/cookies.py +++ b/youtube_dl/cookies.py @@ -144,6 +144,8 @@ def _get_chrome_like_browser_settings(browser_name): else: raise ValueError('unsupported platform: {}'.format(sys.platform)) + # Linux keyring names can be determined by snooping on dbus while opening the browser in KDE: + # dbus-monitor "interface='org.kde.KWallet'" "type=method_return" keyring_name = { 'chrome': 'Chrome', 'chrome_beta': 'Chrome', From 5f323b1a94b0bc7294a25ec7f446ecf9b3c6b292 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 3 Jun 2021 21:27:12 +0100 Subject: [PATCH 08/25] removed alternative browser channels in favour of passing a profile path --- youtube_dl/cookies.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/youtube_dl/cookies.py b/youtube_dl/cookies.py index 1dbe92f14..4740009b1 100644 --- a/youtube_dl/cookies.py +++ b/youtube_dl/cookies.py @@ -25,8 +25,7 @@ except ImportError: KEYRING_AVAILABLE = False -SUPPORTED_BROWSERS = ['firefox', 'chrome', 'chrome_beta', 'chrome_dev', 'chromium', 'brave', 'opera', - 'edge', 'edge_beta'] +SUPPORTED_BROWSERS = ['firefox', 'chrome', 'chromium', 'brave', 'opera', 'edge'] class Logger: @@ -47,8 +46,7 @@ def _is_path(value): def extract_cookies_from_browser(browser_name, profile=None, logger=Logger()): if browser_name == 'firefox': return _extract_firefox_cookies(profile, logger) - elif browser_name in ('chrome', 'chrome_beta', 'chrome_dev', 'chromium', - 'brave', 'opera', 'edge', 'edge_beta'): + elif browser_name in ('chrome', 'chromium', 'brave', 'opera', 'edge'): return _extract_chrome_cookies(browser_name, profile, logger) else: raise ValueError('unknown browser: {}'.format(browser_name)) @@ -106,13 +104,10 @@ def _get_chrome_like_browser_settings(browser_name): config = _config_home() browser_dir = { 'chrome': os.path.join(config, 'google-chrome'), - 'chrome_beta': os.path.join(config, 'google-chrome-beta'), - 'chrome_dev': os.path.join(config, 'google-chrome-unstable'), 'chromium': os.path.join(config, 'chromium'), 'brave': os.path.join(config, 'BraveSoftware/Brave-Browser'), 'opera': os.path.join(config, 'opera'), 'edge': os.path.join(config, 'microsoft-edge'), - 'edge_beta': os.path.join(config, 'microsoft-edge-beta'), }[browser_name] elif sys.platform == 'win32': @@ -120,21 +115,16 @@ def _get_chrome_like_browser_settings(browser_name): appdata_roaming = os.path.expandvars('%APPDATA%') browser_dir = { 'chrome': os.path.join(appdata_local, r'Google\Chrome'), - 'chrome_beta': os.path.join(appdata_local, r'Google\Chrome Beta'), - 'chrome_dev': os.path.join(appdata_local, r'Google\Chrome SxS'), 'chromium': os.path.join(appdata_local, r'Google\Chromium'), 'brave': os.path.join(appdata_local, r'BraveSoftware\Brave-Browser'), 'opera': os.path.join(appdata_roaming, r'Opera Software\Opera Stable'), 'edge': os.path.join(appdata_local, r'Microsoft\Edge'), - 'edge_beta': os.path.join(appdata_local, r'Microsoft\Edge Beta'), }[browser_name] elif sys.platform == 'darwin': appdata = os.path.expanduser('~/Library/Application Support') browser_dir = { 'chrome': os.path.join(appdata, 'Google/Chrome'), - 'chrome_beta': os.path.join(appdata, 'Google/Chrome Beta'), - 'chrome_dev': os.path.join(appdata, 'Google/Chrome Canary'), 'chromium': os.path.join(appdata, 'Google/Chromium'), 'brave': os.path.join(appdata, 'BraveSoftware/Brave-Browser'), 'opera': os.path.join(appdata, 'com.operasoftware.Opera'), @@ -148,13 +138,10 @@ def _get_chrome_like_browser_settings(browser_name): # dbus-monitor "interface='org.kde.KWallet'" "type=method_return" keyring_name = { 'chrome': 'Chrome', - 'chrome_beta': 'Chrome', - 'chrome_dev': 'Chrome', 'chromium': 'Chromium', 'brave': 'Brave', 'opera': 'Opera' if sys.platform == 'darwin' else 'Chromium', 'edge': 'Mirosoft Edge' if sys.platform == 'darwin' else 'Chromium', - 'edge_beta': 'Mirosoft Edge' if sys.platform == 'darwin' else 'Chromium', }[browser_name] return { @@ -524,6 +511,8 @@ def parse_browser_specification(browser_specification): if not parts[0] or len(parts) > 2: raise ValueError('invalid browser specification: "{}"'.format(browser_specification)) browser_name, profile = parts + if _is_path(profile): + profile = os.path.expanduser(profile) return browser_name, profile From 64c425364ebb81f12107d0cb4a8ea6c79b65eac2 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 3 Jun 2021 21:59:32 +0100 Subject: [PATCH 09/25] using hashlib for PBKDF2 --- test/test_cookies.py | 14 +++++++------- youtube_dl/cookies.py | 39 ++++++++++++++++++++------------------- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/test/test_cookies.py b/test/test_cookies.py index d217e46ef..dbdb2f912 100644 --- a/test/test_cookies.py +++ b/test/test_cookies.py @@ -2,7 +2,7 @@ import unittest from youtube_dl import cookies from youtube_dl.cookies import LinuxChromeCookieDecryptor, WindowsChromeCookieDecryptor, CRYPTO_AVAILABLE, \ - MacChromeCookieDecryptor + MacChromeCookieDecryptor, Logger class MonkeyPatch: @@ -23,22 +23,22 @@ class MonkeyPatch: @unittest.skipIf(not CRYPTO_AVAILABLE, 'cryptography library not available') class TestCookies(unittest.TestCase): def test_chrome_cookie_decryptor_linux_derive_key(self): - key = LinuxChromeCookieDecryptor.derive_key('abc') + key = LinuxChromeCookieDecryptor.derive_key(b'abc') assert key == b'7\xa1\xec\xd4m\xfcA\xc7\xb19Z\xd0\x19\xdcM\x17' def test_chrome_cookie_decryptor_mac_derive_key(self): - key = MacChromeCookieDecryptor.derive_key('abc') + key = MacChromeCookieDecryptor.derive_key(b'abc') assert key == b'Y\xe2\xc0\xd0P\xf6\xf4\xe1l\xc1\x8cQ\xcb|\xcdY' def test_chrome_cookie_decryptor_linux_v10(self): - with MonkeyPatch(cookies, '_get_linux_keyring_password', lambda *args, **kwargs: ''): + with MonkeyPatch(cookies, '_get_linux_keyring_password', lambda *args, **kwargs: b''): encrypted_value = b'v10\xccW%\xcd\xe6\xe6\x9fM" \xa7\xb0\xca\xe4\x07\xd6' value = 'USD' decryptor = LinuxChromeCookieDecryptor('Chrome') assert decryptor.decrypt(encrypted_value) == value def test_chrome_cookie_decryptor_linux_v11(self): - with MonkeyPatch(cookies, '_get_linux_keyring_password', lambda *args, **kwargs: ''): + with MonkeyPatch(cookies, '_get_linux_keyring_password', lambda *args, **kwargs: b''): encrypted_value = b'v11#\x81\x10>`w\x8f)\xc0\xb2\xc1\r\xf4\x1al\xdd\x93\xfd\xf8\xf8N\xf2\xa9\x83\xf1\xe9o\x0elVQd' value = 'tz=Europe.London' decryptor = LinuxChromeCookieDecryptor('Chrome') @@ -49,11 +49,11 @@ class TestCookies(unittest.TestCase): lambda *args, **kwargs: b'Y\xef\xad\xad\xeerp\xf0Y\xe6\x9b\x12\xc2 2: raise ValueError('invalid browser specification: "{}"'.format(browser_specification)) browser_name, profile = parts - if _is_path(profile): + if profile is not None and _is_path(profile): profile = os.path.expanduser(profile) return browser_name, profile From cec4807c3f2af8b68e17ba3e01be95880891f538 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 3 Jun 2021 22:25:43 +0100 Subject: [PATCH 10/25] added support for vivaldi --- youtube_dl/cookies.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/youtube_dl/cookies.py b/youtube_dl/cookies.py index d489725d6..361b04184 100644 --- a/youtube_dl/cookies.py +++ b/youtube_dl/cookies.py @@ -24,8 +24,8 @@ except ImportError: KEYRING_AVAILABLE = False -SUPPORTED_BROWSERS = ['brave', 'chrome', 'chromium', 'edge' 'firefox', 'opera'] -CHROME_LIKE_BROWSERS = {'brave', 'chrome', 'chromium', 'edge' 'opera'} +SUPPORTED_BROWSERS = ['brave', 'chrome', 'chromium', 'edge' 'firefox', 'opera', 'vivaldi'] +CHROME_LIKE_BROWSERS = {'brave', 'chrome', 'chromium', 'edge' 'opera', 'vivaldi'} class Logger: @@ -108,17 +108,19 @@ def _get_chrome_like_browser_settings(browser_name): 'chromium': os.path.join(config, 'chromium'), 'edge': os.path.join(config, 'microsoft-edge'), 'opera': os.path.join(config, 'opera'), + 'vivaldi': os.path.join(config, 'vivaldi'), }[browser_name] elif sys.platform == 'win32': appdata_local = os.path.expandvars('%LOCALAPPDATA%') appdata_roaming = os.path.expandvars('%APPDATA%') browser_dir = { - 'brave': os.path.join(appdata_local, r'BraveSoftware\Brave-Browser'), - 'chrome': os.path.join(appdata_local, r'Google\Chrome'), - 'chromium': os.path.join(appdata_local, r'Google\Chromium'), - 'edge': os.path.join(appdata_local, r'Microsoft\Edge'), + 'brave': os.path.join(appdata_local, r'BraveSoftware\Brave-Browser\User Data'), + 'chrome': os.path.join(appdata_local, r'Google\Chrome\User Data'), + 'chromium': os.path.join(appdata_local, r'Google\Chromium\User Data'), + 'edge': os.path.join(appdata_local, r'Microsoft\Edge\User Data'), 'opera': os.path.join(appdata_roaming, r'Opera Software\Opera Stable'), + 'vivaldi': os.path.join(appdata_local, r'Vivaldi\User Data'), }[browser_name] elif sys.platform == 'darwin': @@ -126,9 +128,10 @@ def _get_chrome_like_browser_settings(browser_name): browser_dir = { 'brave': os.path.join(appdata, 'BraveSoftware/Brave-Browser'), 'chrome': os.path.join(appdata, 'Google/Chrome'), - 'chromium': os.path.join(appdata, 'Google/Chromium'), + 'chromium': os.path.join(appdata, 'Chromium'), 'edge': os.path.join(appdata, 'Microsoft Edge'), 'opera': os.path.join(appdata, 'com.operasoftware.Opera'), + 'vivaldi': os.path.join(appdata, 'Vivaldi'), }[browser_name] else: @@ -142,6 +145,7 @@ def _get_chrome_like_browser_settings(browser_name): 'chromium': 'Chromium', 'edge': 'Mirosoft Edge' if sys.platform == 'darwin' else 'Chromium', 'opera': 'Opera' if sys.platform == 'darwin' else 'Chromium', + 'vivaldi': 'Vivaldi' if sys.platform == 'darwin' else 'Chrome', }[browser_name] return { From 21458358561d77a4662cb9285587c1b2aa2a7553 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 3 Jun 2021 22:33:25 +0100 Subject: [PATCH 11/25] using youtube_dl implementation of AES-CBC --- youtube_dl/cookies.py | 61 ++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/youtube_dl/cookies.py b/youtube_dl/cookies.py index 361b04184..4770425cc 100644 --- a/youtube_dl/cookies.py +++ b/youtube_dl/cookies.py @@ -8,8 +8,9 @@ import subprocess import sys import warnings +from youtube_dl.aes import aes_cbc_decrypt from youtube_dl.compat import compat_cookiejar_Cookie, compat_b64decode, Compat_TemporaryDirectory, compat_ord -from youtube_dl.utils import YoutubeDLCookieJar, expand_path +from youtube_dl.utils import YoutubeDLCookieJar, expand_path, bytes_to_intlist, intlist_to_bytes try: from Crypto.Cipher import AES @@ -254,12 +255,11 @@ def get_cookie_decryptor(browser_root, browser_keyring_name, logger): class LinuxChromeCookieDecryptor(ChromeCookieDecryptor): def __init__(self, browser_keyring_name): - self._v10_key = None - self._v11_key = None - if CRYPTO_AVAILABLE: - self._v10_key = self.derive_key(b'peanuts') - if KEYRING_AVAILABLE: - self._v11_key = self.derive_key(_get_linux_keyring_password(browser_keyring_name)) + self._v10_key = self.derive_key(b'peanuts') + if KEYRING_AVAILABLE: + self._v11_key = self.derive_key(_get_linux_keyring_password(browser_keyring_name)) + else: + self._v11_key = None @staticmethod def derive_key(password): @@ -272,14 +272,11 @@ class LinuxChromeCookieDecryptor(ChromeCookieDecryptor): ciphertext = encrypted_value[3:] if version == b'v10': - if self._v10_key is None: - warnings.warn('cannot decrypt cookie as the module `pycryptodome` is not installed') - return None return _decrypt_aes_cbc(ciphertext, self._v10_key) elif version == b'v11': if self._v11_key is None: - warnings.warn('cannot decrypt cookie as the `pycryptodome` or `keyring` modules are not installed') + warnings.warn('cannot decrypt cookie as the `keyring` modules is not installed') return None return _decrypt_aes_cbc(ciphertext, self._v11_key) @@ -289,9 +286,7 @@ class LinuxChromeCookieDecryptor(ChromeCookieDecryptor): class MacChromeCookieDecryptor(ChromeCookieDecryptor): def __init__(self, browser_keyring_name): - self._v10_key = None - if CRYPTO_AVAILABLE: - self._v10_key = self.derive_key(_get_mac_keyring_password(browser_keyring_name)) + self._v10_key = self.derive_key(_get_mac_keyring_password(browser_keyring_name)) @staticmethod def derive_key(password): @@ -304,9 +299,6 @@ class MacChromeCookieDecryptor(ChromeCookieDecryptor): ciphertext = encrypted_value[3:] if version == b'v10': - if self._v10_key is None: - warnings.warn('cannot decrypt cookie as the `pycryptodome` module is not installed') - return None return _decrypt_aes_cbc(ciphertext, self._v10_key) else: @@ -328,6 +320,9 @@ class WindowsChromeCookieDecryptor(ChromeCookieDecryptor): if self._v10_key is None: warnings.warn('cannot decrypt cookie') return None + elif not CRYPTO_AVAILABLE: + warnings.warn('cannot decrypt cookie as the `pycryptodome` module is not installed') + return None # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc # kNonceLength @@ -350,19 +345,18 @@ class WindowsChromeCookieDecryptor(ChromeCookieDecryptor): def _get_linux_keyring_password(browser_keyring_name): - if KEYRING_AVAILABLE: - password = keyring.get_password('{} Keys'.format(browser_keyring_name), - '{} Safe Storage'.format(browser_keyring_name)) - if password is None: - # this sometimes occurs in KDE because chrome does not check hasEntry and instead - # just tries to read the value (which kwallet returns "") whereas keyring checks hasEntry - # to verify this: - # dbus-monitor "interface='org.kde.KWallet'" "type=method_return" - # while starting chrome. - # this may be a bug as the intended behaviour is to generate a random password and store - # it, but that doesn't matter here. - password = '' - return password.encode('utf-8') + password = keyring.get_password('{} Keys'.format(browser_keyring_name), + '{} Safe Storage'.format(browser_keyring_name)) + if password is None: + # this sometimes occurs in KDE because chrome does not check hasEntry and instead + # just tries to read the value (which kwallet returns "") whereas keyring checks hasEntry + # to verify this: + # dbus-monitor "interface='org.kde.KWallet'" "type=method_return" + # while starting chrome. + # this may be a bug as the intended behaviour is to generate a random password and store + # it, but that doesn't matter here. + password = '' + return password.encode('utf-8') def _get_mac_keyring_password(browser_keyring_name): @@ -404,11 +398,12 @@ def _get_windows_v10_password(browser_root, logger): def _decrypt_aes_cbc(ciphertext, key, initialization_vector=b' ' * 16): - cipher = AES.new(key, AES.MODE_CBC, iv=initialization_vector) - plaintext = cipher.decrypt(ciphertext) + plaintext = aes_cbc_decrypt(bytes_to_intlist(ciphertext), + bytes_to_intlist(key), + bytes_to_intlist(initialization_vector)) padding_length = compat_ord(plaintext[-1]) try: - return plaintext[:-padding_length].decode('utf-8') + return intlist_to_bytes(plaintext[:-padding_length]).decode('utf-8') except UnicodeDecodeError: warnings.warn('failed to decrypt cookie because UTF-8 decoding failed. Possibly the key is wrong?') return None From 1f442fd1fb77fffaadf1b9fa680ae1fd69c7ef6b Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 3 Jun 2021 22:51:56 +0100 Subject: [PATCH 12/25] small fixes and tidying --- youtube_dl/cookies.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/youtube_dl/cookies.py b/youtube_dl/cookies.py index 4770425cc..92cc73e15 100644 --- a/youtube_dl/cookies.py +++ b/youtube_dl/cookies.py @@ -40,8 +40,20 @@ class Logger: print(message, file=sys.stderr) -def _is_path(value): - return os.path.sep in value +def load_cookies(cookie_file, browser_specification, ydl): + cookie_jars = [] + if browser_specification is not None: + browser_name, profile = _parse_browser_specification(browser_specification) + cookie_jars.append(extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl))) + + if cookie_file is not None: + cookie_file = expand_path(cookie_file) + jar = YoutubeDLCookieJar(cookie_file) + if os.access(cookie_file, os.R_OK): + jar.load(ignore_discard=True, ignore_expires=True) + cookie_jars.append(jar) + + return _merge_cookie_jars(cookie_jars) def extract_cookies_from_browser(browser_name, profile=None, logger=Logger()): @@ -276,7 +288,7 @@ class LinuxChromeCookieDecryptor(ChromeCookieDecryptor): elif version == b'v11': if self._v11_key is None: - warnings.warn('cannot decrypt cookie as the `keyring` modules is not installed') + warnings.warn('cannot decrypt cookie as the `keyring` module is not installed') return None return _decrypt_aes_cbc(ciphertext, self._v11_key) @@ -401,7 +413,7 @@ def _decrypt_aes_cbc(ciphertext, key, initialization_vector=b' ' * 16): plaintext = aes_cbc_decrypt(bytes_to_intlist(ciphertext), bytes_to_intlist(key), bytes_to_intlist(initialization_vector)) - padding_length = compat_ord(plaintext[-1]) + padding_length = plaintext[-1] try: return intlist_to_bytes(plaintext[:-padding_length]).decode('utf-8') except UnicodeDecodeError: @@ -503,7 +515,11 @@ class YDLLogger(Logger): self._ydl.to_stderr(message) -def parse_browser_specification(browser_specification): +def _is_path(value): + return os.path.sep in value + + +def _parse_browser_specification(browser_specification): parts = browser_specification.split(':') while len(parts) < 2: parts.append('') @@ -514,19 +530,3 @@ def parse_browser_specification(browser_specification): if profile is not None and _is_path(profile): profile = os.path.expanduser(profile) return browser_name, profile - - -def load_cookies(cookie_file, browser_specification, ydl): - cookie_jars = [] - if browser_specification is not None: - browser_name, profile = parse_browser_specification(browser_specification) - cookie_jars.append(extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl))) - - if cookie_file is not None: - cookie_file = expand_path(cookie_file) - jar = YoutubeDLCookieJar(cookie_file) - if os.access(cookie_file, os.R_OK): - jar.load(ignore_discard=True, ignore_expires=True) - cookie_jars.append(jar) - - return _merge_cookie_jars(cookie_jars) From 3096927ce0fce397c71954cc2d9bd4081db4d2dd Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 3 Jun 2021 23:00:58 +0100 Subject: [PATCH 13/25] removed unused import --- youtube_dl/cookies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/cookies.py b/youtube_dl/cookies.py index 92cc73e15..6b8c1400b 100644 --- a/youtube_dl/cookies.py +++ b/youtube_dl/cookies.py @@ -9,7 +9,7 @@ import sys import warnings from youtube_dl.aes import aes_cbc_decrypt -from youtube_dl.compat import compat_cookiejar_Cookie, compat_b64decode, Compat_TemporaryDirectory, compat_ord +from youtube_dl.compat import compat_cookiejar_Cookie, compat_b64decode, Compat_TemporaryDirectory from youtube_dl.utils import YoutubeDLCookieJar, expand_path, bytes_to_intlist, intlist_to_bytes try: From 2ccfd900f1d1f8049aff9d697c8a28899db0f414 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 3 Jun 2021 23:10:28 +0100 Subject: [PATCH 14/25] fix firefox path on windows to allow profiles to be specified by name --- youtube_dl/cookies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/cookies.py b/youtube_dl/cookies.py index 6b8c1400b..c766340de 100644 --- a/youtube_dl/cookies.py +++ b/youtube_dl/cookies.py @@ -104,7 +104,7 @@ def _firefox_browser_dir(): if sys.platform in ('linux', 'linux2'): return os.path.expanduser('~/.mozilla/firefox') elif sys.platform == 'win32': - return os.path.expandvars(r'%APPDATA%\Mozilla\Firefox') + return os.path.expandvars(r'%APPDATA%\Mozilla\Firefox\Profiles') elif sys.platform == 'darwin': return os.path.expanduser('~/Library/Application Support/Firefox') else: From a1aa3e695716fc9e57fb62facb25ed4fdd8efc5e Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sat, 5 Jun 2021 11:09:13 +0100 Subject: [PATCH 15/25] renamed password to key to keep naming consistent --- test/test_cookies.py | 2 +- youtube_dl/cookies.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_cookies.py b/test/test_cookies.py index dbdb2f912..d4db9c669 100644 --- a/test/test_cookies.py +++ b/test/test_cookies.py @@ -45,7 +45,7 @@ class TestCookies(unittest.TestCase): assert decryptor.decrypt(encrypted_value) == value def test_chrome_cookie_decryptor_windows_v10(self): - with MonkeyPatch(cookies, '_get_windows_v10_password', + with MonkeyPatch(cookies, '_get_windows_v10_key', lambda *args, **kwargs: b'Y\xef\xad\xad\xeerp\xf0Y\xe6\x9b\x12\xc2 Date: Fri, 11 Jun 2021 21:51:16 +0100 Subject: [PATCH 16/25] added special handling of browsers without profiles so alternative data paths can be specified --- youtube_dl/cookies.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/youtube_dl/cookies.py b/youtube_dl/cookies.py index 2a89c2392..e0530a9fd 100644 --- a/youtube_dl/cookies.py +++ b/youtube_dl/cookies.py @@ -26,7 +26,7 @@ except ImportError: SUPPORTED_BROWSERS = ['brave', 'chrome', 'chromium', 'edge' 'firefox', 'opera', 'vivaldi'] -CHROME_LIKE_BROWSERS = {'brave', 'chrome', 'chromium', 'edge' 'opera', 'vivaldi'} +CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge' 'opera', 'vivaldi'} class Logger: @@ -59,7 +59,7 @@ def load_cookies(cookie_file, browser_specification, ydl): def extract_cookies_from_browser(browser_name, profile=None, logger=Logger()): if browser_name == 'firefox': return _extract_firefox_cookies(profile, logger) - elif browser_name in CHROME_LIKE_BROWSERS: + elif browser_name in CHROMIUM_BASED_BROWSERS: return _extract_chrome_cookies(browser_name, profile, logger) else: raise ValueError('unknown browser: {}'.format(browser_name)) @@ -111,7 +111,7 @@ def _firefox_browser_dir(): raise ValueError('unsupported platform: {}'.format(sys.platform)) -def _get_chrome_like_browser_settings(browser_name): +def _get_chromium_based_browser_settings(browser_name): # https://chromium.googlesource.com/chromium/src/+/HEAD/docs/user_data_dir.md if sys.platform in ('linux', 'linux2'): config = _config_home() @@ -161,23 +161,30 @@ def _get_chrome_like_browser_settings(browser_name): 'vivaldi': 'Vivaldi' if sys.platform == 'darwin' else 'Chrome', }[browser_name] + browsers_without_profiles = {'opera'} + return { 'browser_dir': browser_dir, - 'keyring_name': keyring_name + 'keyring_name': keyring_name, + 'supports_profiles': browser_name not in browsers_without_profiles } def _extract_chrome_cookies(browser_name, profile, logger): logger.info('extracting cookies from {}'.format(browser_name)) - config = _get_chrome_like_browser_settings(browser_name) + config = _get_chromium_based_browser_settings(browser_name) if profile is None: search_root = config['browser_dir'] elif _is_path(profile): search_root = profile - config['browser_dir'] = os.path.dirname(profile) + config['browser_dir'] = os.path.dirname(profile) if config['supports_profiles'] else profile else: - search_root = os.path.join(config['browser_dir'], profile) + if config['supports_profiles']: + search_root = os.path.join(config['browser_dir'], profile) + else: + logger.error('{} does not support profiles'.format(browser_name)) + search_root = config['browser_dir'] cookie_database_path = _find_most_recently_used_file(search_root, 'Cookies') if cookie_database_path is None: From c9d38e47e1f670f11e12fb0922a82d2ae8ebaa6f Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sat, 12 Jun 2021 17:01:55 +0100 Subject: [PATCH 17/25] a few fixes to add Chromium support on Windows --- youtube_dl/cookies.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/youtube_dl/cookies.py b/youtube_dl/cookies.py index e0530a9fd..04e30dc7c 100644 --- a/youtube_dl/cookies.py +++ b/youtube_dl/cookies.py @@ -130,7 +130,7 @@ def _get_chromium_based_browser_settings(browser_name): browser_dir = { 'brave': os.path.join(appdata_local, r'BraveSoftware\Brave-Browser\User Data'), 'chrome': os.path.join(appdata_local, r'Google\Chrome\User Data'), - 'chromium': os.path.join(appdata_local, r'Google\Chromium\User Data'), + 'chromium': os.path.join(appdata_local, r'Chromium\User Data'), 'edge': os.path.join(appdata_local, r'Microsoft\Edge\User Data'), 'opera': os.path.join(appdata_roaming, r'Opera Software\Opera Stable'), 'vivaldi': os.path.join(appdata_local, r'Vivaldi\User Data'), @@ -198,7 +198,10 @@ def _extract_chrome_cookies(browser_name, profile, logger): try: cursor = _open_database_copy(cookie_database_path, tmpdir) cursor.connection.text_factory = bytes - cursor.execute('SELECT host_key, name, value, encrypted_value, path, expires_utc, is_secure FROM cookies') + column_names = _get_column_names(cursor, 'cookies') + secure_column = 'is_secure' if 'is_secure' in column_names else 'secure' + cursor.execute('SELECT host_key, name, value, encrypted_value, path, ' + 'expires_utc, {} FROM cookies'.format(secure_column)) jar = YoutubeDLCookieJar() failed_cookies = 0 for host_key, name, value, encrypted_value, path, expires_utc, is_secure in cursor.fetchall(): @@ -360,7 +363,7 @@ class WindowsChromeCookieDecryptor(ChromeCookieDecryptor): else: # any other prefix means the data is DPAPI encrypted # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc - return _decrypt_windows_dpapi(encrypted_value, self._logger) + return _decrypt_windows_dpapi(encrypted_value, self._logger).decode('utf-8') def _get_linux_keyring_password(browser_keyring_name): @@ -487,6 +490,11 @@ def _open_database_copy(database_path, tmpdir): return conn.cursor() +def _get_column_names(cursor, table_name): + table_info = cursor.execute('PRAGMA table_info({})'.format(table_name)).fetchall() + return [row[1].decode('utf-8') for row in table_info] + + def _find_most_recently_used_file(root, filename): # if there are multiple browser profiles, take the most recently used one paths = [] From 50aa5601ebb085540ba9018182fe2db2dcc167ae Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sat, 12 Jun 2021 17:05:45 +0100 Subject: [PATCH 18/25] added Safari support --- test/test_cookies.py | 24 +++++- youtube_dl/cookies.py | 176 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 197 insertions(+), 3 deletions(-) diff --git a/test/test_cookies.py b/test/test_cookies.py index d4db9c669..091326012 100644 --- a/test/test_cookies.py +++ b/test/test_cookies.py @@ -1,8 +1,9 @@ import unittest +from datetime import datetime, timezone from youtube_dl import cookies from youtube_dl.cookies import LinuxChromeCookieDecryptor, WindowsChromeCookieDecryptor, CRYPTO_AVAILABLE, \ - MacChromeCookieDecryptor, Logger + MacChromeCookieDecryptor, Logger, parse_safari_cookies class MonkeyPatch: @@ -20,7 +21,6 @@ class MonkeyPatch: setattr(self._module, self._name, self._backup_value) -@unittest.skipIf(not CRYPTO_AVAILABLE, 'cryptography library not available') class TestCookies(unittest.TestCase): def test_chrome_cookie_decryptor_linux_derive_key(self): key = LinuxChromeCookieDecryptor.derive_key(b'abc') @@ -44,6 +44,7 @@ class TestCookies(unittest.TestCase): decryptor = LinuxChromeCookieDecryptor('Chrome') assert decryptor.decrypt(encrypted_value) == value + @unittest.skipIf(not CRYPTO_AVAILABLE, 'cryptography library not available') def test_chrome_cookie_decryptor_windows_v10(self): with MonkeyPatch(cookies, '_get_windows_v10_key', lambda *args, **kwargs: b'Y\xef\xad\xad\xeerp\xf0Y\xe6\x9b\x12\xc2\xc3A\x00\x00\x80\xc3\x07:\xc3A' \ + b'localhost\x00foo\x00/\x00test%20%3Bcookie\x00\x00\x00\x054\x07\x17 \x05\x00\x00\x00Kbplist00\xd1\x01' \ + b'\x02_\x10\x18NSHTTPCookieAcceptPolicy\x10\x02\x08\x0b&\x00\x00\x00\x00\x00\x00\x01\x01\x00\x00\x00' \ + b'\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00(' + jar = parse_safari_cookies(cookies) + assert len(jar) == 1 + cookie = list(jar)[0] + assert cookie.domain == 'localhost' + assert cookie.port is None + assert cookie.path == '/' + assert cookie.name == 'foo' + assert cookie.value == 'test%20%3Bcookie' + assert not cookie.secure + expected_expiration = datetime(2021, 6, 18, 20, 39, 19, tzinfo=timezone.utc) + assert cookie.expires == expected_expiration.timestamp() diff --git a/youtube_dl/cookies.py b/youtube_dl/cookies.py index 04e30dc7c..5e9ca0563 100644 --- a/youtube_dl/cookies.py +++ b/youtube_dl/cookies.py @@ -4,9 +4,11 @@ import json import os import shutil import sqlite3 +import struct import subprocess import sys import warnings +from datetime import datetime, timedelta from youtube_dl.aes import aes_cbc_decrypt from youtube_dl.compat import compat_cookiejar_Cookie, compat_b64decode, Compat_TemporaryDirectory @@ -25,7 +27,7 @@ except ImportError: KEYRING_AVAILABLE = False -SUPPORTED_BROWSERS = ['brave', 'chrome', 'chromium', 'edge' 'firefox', 'opera', 'vivaldi'] +SUPPORTED_BROWSERS = ['brave', 'chrome', 'chromium', 'edge' 'firefox', 'opera', 'safari', 'vivaldi'] CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge' 'opera', 'vivaldi'} @@ -36,6 +38,9 @@ class Logger: def info(self, message): print(message) + def warning(self, message): + print(message, file=sys.stderr) + def error(self, message): print(message, file=sys.stderr) @@ -59,6 +64,8 @@ def load_cookies(cookie_file, browser_specification, ydl): def extract_cookies_from_browser(browser_name, profile=None, logger=Logger()): if browser_name == 'firefox': return _extract_firefox_cookies(profile, logger) + elif browser_name == 'safari': + return _extract_safari_cookies(profile, logger) elif browser_name in CHROMIUM_BASED_BROWSERS: return _extract_chrome_cookies(browser_name, profile, logger) else: @@ -366,6 +373,170 @@ class WindowsChromeCookieDecryptor(ChromeCookieDecryptor): return _decrypt_windows_dpapi(encrypted_value, self._logger).decode('utf-8') +def _extract_safari_cookies(profile, logger): + if profile is not None: + logger.error('safari does not support profiles') + if sys.platform != 'darwin': + raise ValueError('unsupported platform: {}'.format(sys.platform)) + + cookies_path = os.path.expanduser('~/Library/Cookies/Cookies.binarycookies') + + if not os.path.isfile(cookies_path): + raise FileNotFoundError('could not find safari cookies database') + + with open(cookies_path, 'rb') as f: + cookies_data = f.read() + + jar = parse_safari_cookies(cookies_data, logger=logger) + logger.info('extracted {} cookies from safari'.format(len(jar))) + return jar + + +class ParserError(Exception): + pass + + +class DataParser: + def __init__(self, data, logger): + self._data = data + self.cursor = 0 + self._logger = logger + + def read_bytes(self, num_bytes): + if num_bytes < 0: + raise ParserError('invalid read of {} bytes'.format(num_bytes)) + end = self.cursor + num_bytes + if end > len(self._data): + raise ParserError('reached end of input') + data = self._data[self.cursor:end] + self.cursor = end + return data + + def expect_bytes(self, expected_value, message): + value = self.read_bytes(len(expected_value)) + if value != expected_value: + raise ParserError('unexpected value: {} != {} ({})'.format(value, expected_value, message)) + + def read_uint(self, big_endian=False): + data_format = '>I' if big_endian else ' 0: + self._logger.debug('skipping {} bytes ({}): {}'.format( + num_bytes, description, self.read_bytes(num_bytes))) + elif num_bytes < 0: + raise ParserError('invalid skip of {} bytes'.format(num_bytes)) + + def skip_to(self, offset, description='unknown'): + self.skip(offset - self.cursor, description) + + def skip_to_end(self, description='unknown'): + self.skip_to(len(self._data), description) + + +def _mac_absolute_time_to_posix(timestamp): + return (datetime(2001, 1, 1, 0, 0) + timedelta(seconds=timestamp)).timestamp() + + +def _parse_safari_cookies_header(data, logger): + p = DataParser(data, logger) + p.expect_bytes(b'cook', 'database signature') + number_of_pages = p.read_uint(big_endian=True) + page_sizes = [p.read_uint(big_endian=True) for _ in range(number_of_pages)] + return page_sizes, p.cursor + + +def _parse_safari_cookies_page(data, jar, logger): + p = DataParser(data, logger) + p.expect_bytes(b'\x00\x00\x01\x00', 'page signature') + number_of_cookies = p.read_uint() + record_offsets = [p.read_uint() for _ in range(number_of_cookies)] + if number_of_cookies == 0: + logger.debug('a cookies page of size {} has no cookies'.format(len(data))) + return + + p.skip_to(record_offsets[0], 'unknown page header field') + + for record_offset in record_offsets: + p.skip_to(record_offset, 'space between records') + record_length = _parse_safari_cookies_record(data[record_offset:], jar, logger) + p.read_bytes(record_length) + p.skip_to_end('space in between pages') + + +def _parse_safari_cookies_record(data, jar, logger): + p = DataParser(data, logger) + record_size = p.read_uint() + p.skip(4, 'unknown record field 1') + flags = p.read_uint() + is_secure = bool(flags & 0x0001) + p.skip(4, 'unknown record field 2') + domain_offset = p.read_uint() + name_offset = p.read_uint() + path_offset = p.read_uint() + value_offset = p.read_uint() + p.skip(8, 'unknown record field 3') + expiration_date = _mac_absolute_time_to_posix(p.read_double()) + _creation_date = _mac_absolute_time_to_posix(p.read_double()) + + try: + p.skip_to(domain_offset) + domain = p.read_cstring() + + p.skip_to(name_offset) + name = p.read_cstring() + + p.skip_to(path_offset) + path = p.read_cstring() + + p.skip_to(value_offset) + value = p.read_cstring() + except UnicodeDecodeError: + warnings.warn('failed to parse cookie because UTF-8 decoding failed') + return record_size + + p.skip_to(record_size, 'space at the end of the record') + + cookie = compat_cookiejar_Cookie( + version=0, name=name, value=value, port=None, port_specified=False, + domain=domain, domain_specified=bool(domain), domain_initial_dot=domain.startswith('.'), + path=path, path_specified=bool(path), secure=is_secure, expires=expiration_date, discard=False, + comment=None, comment_url=None, rest={}) + jar.set_cookie(cookie) + return record_size + + +def parse_safari_cookies(data, jar=None, logger=Logger()): + """ + References: + - https://github.com/libyal/dtformats/blob/main/documentation/Safari%20Cookies.asciidoc + - this data appears to be out of date but the important parts of the database structure is the same + - there are a few bytes here and there which are skipped during parsing + """ + if jar is None: + jar = YoutubeDLCookieJar() + page_sizes, body_start = _parse_safari_cookies_header(data, logger) + p = DataParser(data[body_start:], logger) + for page_size in page_sizes: + _parse_safari_cookies_page(p.read_bytes(page_size), jar, logger) + p.skip_to_end('footer') + return jar + + def _get_linux_keyring_password(browser_keyring_name): password = keyring.get_password('{} Keys'.format(browser_keyring_name), '{} Safe Storage'.format(browser_keyring_name)) @@ -526,6 +697,9 @@ class YDLLogger(Logger): def info(self, message): self._ydl.to_screen(message) + def warning(self, message): + self._ydl.to_stderr(message) + def error(self, message): self._ydl.to_stderr(message) From 613fe995f575954dea82fa5e60cf32d321209750 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 17 Jun 2021 19:56:05 +0100 Subject: [PATCH 19/25] changes for linter --- youtube_dl/YoutubeDL.py | 1 - youtube_dl/compat.py | 4 ++-- youtube_dl/cookies.py | 8 ++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index d8d8c1435..4c1004973 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -89,7 +89,6 @@ from .utils import ( version_tuple, write_json_file, write_string, - YoutubeDLCookieJar, YoutubeDLCookieProcessor, YoutubeDLHandler, YoutubeDLRedirectHandler, diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index 27b0ead30..7a4f832f3 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -2926,9 +2926,9 @@ except TypeError: # Python 2.6 try: - from tempfile import TemporaryDirectory as Compat_TemporaryDirectory + from tempfile import TemporaryDirectory as compat_TemporaryDirectory except ImportError: - class Compat_TemporaryDirectory: + class compat_TemporaryDirectory: def __init__(self, prefix=None): self._prefix = prefix self._path = None diff --git a/youtube_dl/cookies.py b/youtube_dl/cookies.py index 5e9ca0563..362b12233 100644 --- a/youtube_dl/cookies.py +++ b/youtube_dl/cookies.py @@ -11,7 +11,7 @@ import warnings from datetime import datetime, timedelta from youtube_dl.aes import aes_cbc_decrypt -from youtube_dl.compat import compat_cookiejar_Cookie, compat_b64decode, Compat_TemporaryDirectory +from youtube_dl.compat import compat_cookiejar_Cookie, compat_b64decode, compat_TemporaryDirectory from youtube_dl.utils import YoutubeDLCookieJar, expand_path, bytes_to_intlist, intlist_to_bytes try: @@ -87,7 +87,7 @@ def _extract_firefox_cookies(profile, logger): raise FileNotFoundError('could not find firefox cookies database in {}'.format(search_root)) logger.debug('extracting from: "{}"'.format(cookie_database_path)) - with Compat_TemporaryDirectory(prefix='youtube_dl') as tmpdir: + with compat_TemporaryDirectory(prefix='youtube_dl') as tmpdir: cursor = None try: cursor = _open_database_copy(cookie_database_path, tmpdir) @@ -200,7 +200,7 @@ def _extract_chrome_cookies(browser_name, profile, logger): decryptor = get_cookie_decryptor(config['browser_dir'], config['keyring_name'], logger) - with Compat_TemporaryDirectory(prefix='youtube_dl') as tmpdir: + with compat_TemporaryDirectory(prefix='youtube_dl') as tmpdir: cursor = None try: cursor = _open_database_copy(cookie_database_path, tmpdir) @@ -491,7 +491,7 @@ def _parse_safari_cookies_record(data, jar, logger): value_offset = p.read_uint() p.skip(8, 'unknown record field 3') expiration_date = _mac_absolute_time_to_posix(p.read_double()) - _creation_date = _mac_absolute_time_to_posix(p.read_double()) + _creation_date = _mac_absolute_time_to_posix(p.read_double()) # noqa: F841 try: p.skip_to(domain_offset) From 104af51a208e13398f605e4cc8b849f02e84a2d2 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 17 Jun 2021 20:32:43 +0100 Subject: [PATCH 20/25] added compatability wrapper for pbkdf2_sha1 --- test/test_cookies.py | 41 ++++++++++++++++++++++++++++------------- youtube_dl/cookies.py | 23 ++++++++++++++++++++--- 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/test/test_cookies.py b/test/test_cookies.py index 091326012..f1f32923d 100644 --- a/test/test_cookies.py +++ b/test/test_cookies.py @@ -1,44 +1,51 @@ +import sys import unittest from datetime import datetime, timezone from youtube_dl import cookies from youtube_dl.cookies import LinuxChromeCookieDecryptor, WindowsChromeCookieDecryptor, CRYPTO_AVAILABLE, \ - MacChromeCookieDecryptor, Logger, parse_safari_cookies + MacChromeCookieDecryptor, Logger, parse_safari_cookies, pbkdf2_sha1, PBKDF2_AVAILABLE class MonkeyPatch: - def __init__(self, module, name, temp_value): + def __init__(self, module, temporary_values): self._module = module - self._name = name - self._temp_value = temp_value - self._backup_value = None + self._temporary_values = temporary_values + self._backup_values = {} def __enter__(self): - self._backup_value = getattr(self._module, self._name) - setattr(self._module, self._name, self._temp_value) + for name, temp_value in self._temporary_values.items(): + self._backup_values[name] = getattr(self._module, name) + setattr(self._module, name, temp_value) def __exit__(self, exc_type, exc_val, exc_tb): - setattr(self._module, self._name, self._backup_value) + for name, backup_value in self._backup_values.items(): + setattr(self._module, name, backup_value) class TestCookies(unittest.TestCase): + @unittest.skipIf(not PBKDF2_AVAILABLE, 'PBKDF2 not available') def test_chrome_cookie_decryptor_linux_derive_key(self): key = LinuxChromeCookieDecryptor.derive_key(b'abc') assert key == b'7\xa1\xec\xd4m\xfcA\xc7\xb19Z\xd0\x19\xdcM\x17' + @unittest.skipIf(not PBKDF2_AVAILABLE, 'PBKDF2 not available') def test_chrome_cookie_decryptor_mac_derive_key(self): key = MacChromeCookieDecryptor.derive_key(b'abc') assert key == b'Y\xe2\xc0\xd0P\xf6\xf4\xe1l\xc1\x8cQ\xcb|\xcdY' + @unittest.skipIf(not PBKDF2_AVAILABLE, 'PBKDF2 not available') def test_chrome_cookie_decryptor_linux_v10(self): - with MonkeyPatch(cookies, '_get_linux_keyring_password', lambda *args, **kwargs: b''): + with MonkeyPatch(cookies, {'_get_linux_keyring_password': lambda *args, **kwargs: b''}): encrypted_value = b'v10\xccW%\xcd\xe6\xe6\x9fM" \xa7\xb0\xca\xe4\x07\xd6' value = 'USD' decryptor = LinuxChromeCookieDecryptor('Chrome') assert decryptor.decrypt(encrypted_value) == value + @unittest.skipIf(not PBKDF2_AVAILABLE, 'PBKDF2 not available') def test_chrome_cookie_decryptor_linux_v11(self): - with MonkeyPatch(cookies, '_get_linux_keyring_password', lambda *args, **kwargs: b''): + with MonkeyPatch(cookies, {'_get_linux_keyring_password': lambda *args, **kwargs: b'', + 'KEYRING_AVAILABLE': True}): encrypted_value = b'v11#\x81\x10>`w\x8f)\xc0\xb2\xc1\r\xf4\x1al\xdd\x93\xfd\xf8\xf8N\xf2\xa9\x83\xf1\xe9o\x0elVQd' value = 'tz=Europe.London' decryptor = LinuxChromeCookieDecryptor('Chrome') @@ -46,15 +53,16 @@ class TestCookies(unittest.TestCase): @unittest.skipIf(not CRYPTO_AVAILABLE, 'cryptography library not available') def test_chrome_cookie_decryptor_windows_v10(self): - with MonkeyPatch(cookies, '_get_windows_v10_key', - lambda *args, **kwargs: b'Y\xef\xad\xad\xeerp\xf0Y\xe6\x9b\x12\xc2= (3, 4) or CRYPTO_AVAILABLE + + +def pbkdf2_sha1(password, salt, iterations, key_length): + try: + from hashlib import pbkdf2_hmac + return pbkdf2_hmac('sha1', password, salt, iterations, key_length) + except ImportError: + try: + from Crypto.Protocol.KDF import PBKDF2 + from Crypto.Hash import SHA1 + return PBKDF2(password, salt, key_length, iterations, hmac_hash_module=SHA1) + except ImportError: + warnings.warn('PBKDF2 is not available. You must either upgrade to ' + 'python >= 3.4 or install the pycryptodome package') + return None + + def _decrypt_aes_cbc(ciphertext, key, initialization_vector=b' ' * 16): plaintext = aes_cbc_decrypt(bytes_to_intlist(ciphertext), bytes_to_intlist(key), From 135aab91b29b133f66d2cd2e9c48459f1482b580 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Thu, 17 Jun 2021 20:43:42 +0100 Subject: [PATCH 21/25] skip more tests --- test/test_cookies.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_cookies.py b/test/test_cookies.py index f1f32923d..452293a8f 100644 --- a/test/test_cookies.py +++ b/test/test_cookies.py @@ -51,6 +51,7 @@ class TestCookies(unittest.TestCase): decryptor = LinuxChromeCookieDecryptor('Chrome') assert decryptor.decrypt(encrypted_value) == value + @unittest.skipIf(not PBKDF2_AVAILABLE, 'PBKDF2 not available') @unittest.skipIf(not CRYPTO_AVAILABLE, 'cryptography library not available') def test_chrome_cookie_decryptor_windows_v10(self): with MonkeyPatch(cookies, { @@ -61,6 +62,7 @@ class TestCookies(unittest.TestCase): decryptor = WindowsChromeCookieDecryptor('', Logger()) assert decryptor.decrypt(encrypted_value) == value + @unittest.skipIf(not PBKDF2_AVAILABLE, 'PBKDF2 not available') def test_chrome_cookie_decryptor_mac_v10(self): with MonkeyPatch(cookies, {'_get_mac_keyring_password': lambda *args, **kwargs: b'6eIDUdtKAacvlHwBVwvg/Q=='}): encrypted_value = b'v10\xb3\xbe\xad\xa1[\x9fC\xa1\x98\xe0\x9a\x01\xd9\xcf\xbfc' From 0b87e6e7e33e2fec3b94eba5b0bab7a1bff44491 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 11 Jul 2021 16:34:20 +0100 Subject: [PATCH 22/25] updated README --- README.md | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2841ed68f..eb9b33d67 100644 --- a/README.md +++ b/README.md @@ -276,6 +276,13 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo info-json" option) --cookies FILE File to read cookies from and dump cookie jar in + --cookies-from-browser BROWSER Load cookies from a user profile of the + given web browser: brave, chrome, + chromium, edgefirefox, opera, safari, + vivaldi. You can specify an alternative + user profile name or directory using + "BROWSER:PROFILE_NAME" or + "BROWSER:PROFILE_PATH" --cache-dir DIR Location in the filesystem where youtube-dl can store some downloaded information permanently. By default @@ -916,14 +923,19 @@ Either prepend `https://www.youtube.com/watch?v=` or separate the ID from the op ### How do I pass cookies to youtube-dl? -Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. - -In order to extract cookies from browser use any conforming browser extension for exporting cookies. For example, [Get cookies.txt](https://chrome.google.com/webstore/detail/get-cookiestxt/bgaddhkoddajcdgocldbbfleckgcbcid/) (for Chrome) or [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/) (for Firefox). - -Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows and `LF` (`\n`) for Unix and Unix-like systems (Linux, macOS, etc.). `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format. +If you have access to a web browser, simply log into youtube in the browser and pass `--cookies-from-browser ` to youtube-dl (see `youtube-dl --help` for supported browsers). Passing cookies to youtube-dl is a good way to workaround login when a particular extractor does not implement it explicitly. Another use case is working around [CAPTCHA](https://en.wikipedia.org/wiki/CAPTCHA) some websites require you to solve in particular cases in order to get access (e.g. YouTube, CloudFlare). +#### Passing cookies without a web browser +Warning: be careful with your cookies file as it can be used to gain access to any accounts you are logged into! + +If you do not have access to a web browser (e.g. if running headless) then manually extract cookies to a file from a web browser on another machine. There are browser extensions that can do this such as [Get cookies.txt](https://chrome.google.com/webstore/detail/get-cookiestxt/bgaddhkoddajcdgocldbbfleckgcbcid/) (for Chrome) or [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/) (for Firefox). +Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows and `LF` (`\n`) for Unix and Unix-like systems (Linux, macOS, etc.). `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format. + +After you have extracted a cookies file, pass it to youtube-dl using the option: `--cookies ` + + ### How do I stream directly to media player? You will first need to tell youtube-dl to stream media to stdout with `-o -`, and also tell your media player to read from stdin (it must be capable of this for streaming) and then pipe former to latter. For example, streaming to [vlc](https://www.videolan.org/) can be achieved with: From 6d49f66de82152dff3e0c12926c3e35bb66bfaa6 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 11 Jul 2021 21:45:35 +0100 Subject: [PATCH 23/25] moved class to fix compat ordering --- youtube_dl/compat.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index 7a4f832f3..28ae98a55 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -2925,23 +2925,6 @@ except TypeError: # Python 2.6 n += step -try: - from tempfile import TemporaryDirectory as compat_TemporaryDirectory -except ImportError: - class compat_TemporaryDirectory: - def __init__(self, prefix=None): - self._prefix = prefix - self._path = None - - def __enter__(self): - self._path = tempfile.mkdtemp(prefix=self._prefix) - return self._path - - def __exit__(self, exc_type, exc_val, exc_tb): - if self._path is not None: - shutil.rmtree(self._path) - - if sys.version_info >= (3, 0): from tokenize import tokenize as compat_tokenize_tokenize else: @@ -2981,6 +2964,23 @@ else: compat_Struct = struct.Struct +try: + from tempfile import TemporaryDirectory as compat_TemporaryDirectory +except ImportError: + class compat_TemporaryDirectory: + def __init__(self, prefix=None): + self._prefix = prefix + self._path = None + + def __enter__(self): + self._path = tempfile.mkdtemp(prefix=self._prefix) + return self._path + + def __exit__(self, exc_type, exc_val, exc_tb): + if self._path is not None: + shutil.rmtree(self._path) + + try: from future_builtins import zip as compat_zip except ImportError: # not 2.6+ or is 3.x @@ -3022,6 +3022,7 @@ __all__ = [ 'compat_HTMLParser', 'compat_HTTPError', 'compat_Struct', + 'compat_TemporaryDirectory', 'compat_b64decode', 'compat_basestring', 'compat_chr', @@ -3059,7 +3060,6 @@ __all__ = [ 'compat_struct_pack', 'compat_struct_unpack', 'compat_subprocess_get_DEVNULL', - 'compat_TemporaryDirectory', 'compat_tokenize_tokenize', 'compat_urllib_error', 'compat_urllib_parse', From 8f4536edee9e6e014efd556e1c9b03d776db695c Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sat, 31 Jul 2021 19:31:13 +0100 Subject: [PATCH 24/25] incoporated improvements made during pull request to yt-dlp --- test/test_cookies.py | 59 +++++----- youtube_dl/YoutubeDL.py | 3 + youtube_dl/__init__.py | 8 +- youtube_dl/cookies.py | 192 +++++++++++++++++++------------- youtube_dl/extractor/youtube.py | 5 +- youtube_dl/options.py | 12 +- 6 files changed, 171 insertions(+), 108 deletions(-) diff --git a/test/test_cookies.py b/test/test_cookies.py index 452293a8f..c3365bbcc 100644 --- a/test/test_cookies.py +++ b/test/test_cookies.py @@ -1,10 +1,17 @@ -import sys import unittest from datetime import datetime, timezone from youtube_dl import cookies -from youtube_dl.cookies import LinuxChromeCookieDecryptor, WindowsChromeCookieDecryptor, CRYPTO_AVAILABLE, \ - MacChromeCookieDecryptor, Logger, parse_safari_cookies, pbkdf2_sha1, PBKDF2_AVAILABLE +from youtube_dl.cookies import ( + LinuxChromeCookieDecryptor, + WindowsChromeCookieDecryptor, + CRYPTO_AVAILABLE, + MacChromeCookieDecryptor, + parse_safari_cookies, + pbkdf2_sha1, + PBKDF2_AVAILABLE, + YDLLogger, +) class MonkeyPatch: @@ -26,21 +33,21 @@ class MonkeyPatch: class TestCookies(unittest.TestCase): @unittest.skipIf(not PBKDF2_AVAILABLE, 'PBKDF2 not available') def test_chrome_cookie_decryptor_linux_derive_key(self): - key = LinuxChromeCookieDecryptor.derive_key(b'abc') - assert key == b'7\xa1\xec\xd4m\xfcA\xc7\xb19Z\xd0\x19\xdcM\x17' + key = LinuxChromeCookieDecryptor(None, YDLLogger()).derive_key(b'abc') + self.assertEqual(key, b'7\xa1\xec\xd4m\xfcA\xc7\xb19Z\xd0\x19\xdcM\x17') @unittest.skipIf(not PBKDF2_AVAILABLE, 'PBKDF2 not available') def test_chrome_cookie_decryptor_mac_derive_key(self): - key = MacChromeCookieDecryptor.derive_key(b'abc') - assert key == b'Y\xe2\xc0\xd0P\xf6\xf4\xe1l\xc1\x8cQ\xcb|\xcdY' + key = MacChromeCookieDecryptor(None, YDLLogger()).derive_key(b'abc') + self.assertEqual(key, b'Y\xe2\xc0\xd0P\xf6\xf4\xe1l\xc1\x8cQ\xcb|\xcdY') @unittest.skipIf(not PBKDF2_AVAILABLE, 'PBKDF2 not available') def test_chrome_cookie_decryptor_linux_v10(self): with MonkeyPatch(cookies, {'_get_linux_keyring_password': lambda *args, **kwargs: b''}): encrypted_value = b'v10\xccW%\xcd\xe6\xe6\x9fM" \xa7\xb0\xca\xe4\x07\xd6' value = 'USD' - decryptor = LinuxChromeCookieDecryptor('Chrome') - assert decryptor.decrypt(encrypted_value) == value + decryptor = LinuxChromeCookieDecryptor('Chrome', YDLLogger()) + self.assertEqual(decryptor.decrypt(encrypted_value), value) @unittest.skipIf(not PBKDF2_AVAILABLE, 'PBKDF2 not available') def test_chrome_cookie_decryptor_linux_v11(self): @@ -48,8 +55,8 @@ class TestCookies(unittest.TestCase): 'KEYRING_AVAILABLE': True}): encrypted_value = b'v11#\x81\x10>`w\x8f)\xc0\xb2\xc1\r\xf4\x1al\xdd\x93\xfd\xf8\xf8N\xf2\xa9\x83\xf1\xe9o\x0elVQd' value = 'tz=Europe.London' - decryptor = LinuxChromeCookieDecryptor('Chrome') - assert decryptor.decrypt(encrypted_value) == value + decryptor = LinuxChromeCookieDecryptor('Chrome', YDLLogger()) + self.assertEqual(decryptor.decrypt(encrypted_value), value) @unittest.skipIf(not PBKDF2_AVAILABLE, 'PBKDF2 not available') @unittest.skipIf(not CRYPTO_AVAILABLE, 'cryptography library not available') @@ -59,16 +66,16 @@ class TestCookies(unittest.TestCase): }): encrypted_value = b'v10T\xb8\xf3\xb8\x01\xa7TtcV\xfc\x88\xb8\xb8\xef\x05\xb5\xfd\x18\xc90\x009\xab\xb1\x893\x85)\x87\xe1\xa9-\xa3\xad=' value = '32101439' - decryptor = WindowsChromeCookieDecryptor('', Logger()) - assert decryptor.decrypt(encrypted_value) == value + decryptor = WindowsChromeCookieDecryptor('', YDLLogger()) + self.assertEqual(decryptor.decrypt(encrypted_value), value) @unittest.skipIf(not PBKDF2_AVAILABLE, 'PBKDF2 not available') def test_chrome_cookie_decryptor_mac_v10(self): with MonkeyPatch(cookies, {'_get_mac_keyring_password': lambda *args, **kwargs: b'6eIDUdtKAacvlHwBVwvg/Q=='}): encrypted_value = b'v10\xb3\xbe\xad\xa1[\x9fC\xa1\x98\xe0\x9a\x01\xd9\xcf\xbfc' value = '2021-06-01-22' - decryptor = MacChromeCookieDecryptor('') - assert decryptor.decrypt(encrypted_value) == value + decryptor = MacChromeCookieDecryptor('', YDLLogger()) + self.assertEqual(decryptor.decrypt(encrypted_value), value) def test_safari_cookie_parsing(self): cookies = \ @@ -80,20 +87,20 @@ class TestCookies(unittest.TestCase): b'\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00(' jar = parse_safari_cookies(cookies) - assert len(jar) == 1 + self.assertEqual(len(jar), 1) cookie = list(jar)[0] - assert cookie.domain == 'localhost' - assert cookie.port is None - assert cookie.path == '/' - assert cookie.name == 'foo' - assert cookie.value == 'test%20%3Bcookie' - assert not cookie.secure - expected_expiration = datetime(2021, 6, 18, 20, 39, 19, tzinfo=timezone.utc) - assert cookie.expires == expected_expiration.timestamp() + self.assertEqual(cookie.domain, 'localhost') + self.assertEqual(cookie.port, None) + self.assertEqual(cookie.path, '/') + self.assertEqual(cookie.name, 'foo') + self.assertEqual(cookie.value, 'test%20%3Bcookie') + self.assertFalse(cookie.secure) + expected_expiration = datetime(2021, 6, 18, 21, 39, 19, tzinfo=timezone.utc) + self.assertEqual(cookie.expires, int(expected_expiration.timestamp())) def test_pbkdf2_sha1(self): key = pbkdf2_sha1(b'peanuts', b' ' * 16, 1, 16) if PBKDF2_AVAILABLE: - assert key == b'g\xe1\x8e\x0fQ\x1c\x9b\xf3\xc9`!\xaa\x90\xd9\xd34' + self.assertEqual(key, b'g\xe1\x8e\x0fQ\x1c\x9b\xf3\xc9`!\xaa\x90\xd9\xd34') else: - assert key is None + self.assertIsNone(key) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 4c1004973..7bc36e4a1 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -209,6 +209,9 @@ class YoutubeDL(object): Videos already present in the file are not downloaded again. cookiefile: File name where cookies should be read from and dumped to. + cookiesfrombrowser:A tuple containing the name of the browser and the profile + name/path from where cookies are loaded. + Eg: ('chrome', ) or (vivaldi, 'default') nocheckcertificate:Do not verify SSL certificates prefer_insecure: Use HTTP instead of HTTPS to retrieve information. At the moment, this is only supported by YouTube. diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index a29481b28..aa15d1171 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -11,7 +11,7 @@ import os import random import sys - +from .cookies import SUPPORTED_BROWSERS from .options import ( parseOpts, ) @@ -216,6 +216,12 @@ def _real_main(argv=None): if opts.convertsubtitles not in ['srt', 'vtt', 'ass', 'lrc']: parser.error('invalid subtitle format specified') + if opts.cookiesfrombrowser is not None: + opts.cookiesfrombrowser = [ + part.strip() or None for part in opts.cookiesfrombrowser.split(':', 1)] + if opts.cookiesfrombrowser[0] not in SUPPORTED_BROWSERS: + parser.error('unsupported browser specified for cookies') + if opts.date is not None: date = DateRange.day(opts.date) else: diff --git a/youtube_dl/cookies.py b/youtube_dl/cookies.py index 52010a234..595eaede2 100644 --- a/youtube_dl/cookies.py +++ b/youtube_dl/cookies.py @@ -2,16 +2,33 @@ import ctypes import json import os import shutil -import sqlite3 import struct import subprocess import sys -import warnings -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from youtube_dl.aes import aes_cbc_decrypt -from youtube_dl.compat import compat_cookiejar_Cookie, compat_b64decode, compat_TemporaryDirectory -from youtube_dl.utils import YoutubeDLCookieJar, expand_path, bytes_to_intlist, intlist_to_bytes +from youtube_dl.compat import ( + compat_cookiejar_Cookie, + compat_b64decode, + compat_TemporaryDirectory, +) +from youtube_dl.utils import ( + YoutubeDLCookieJar, + expand_path, + bytes_to_intlist, + intlist_to_bytes, + bug_reports_message, +) + +try: + import sqlite3 + SQLITE_AVAILABLE = True +except ImportError: + # although sqlite3 is part of the standard library, it is possible to compile python without + # sqlite support. See: https://github.com/yt-dlp/yt-dlp/issues/544 + SQLITE_AVAILABLE = False + try: from Crypto.Cipher import AES @@ -22,32 +39,54 @@ except ImportError: try: import keyring KEYRING_AVAILABLE = True + KEYRING_UNAVAILABLE_REASON = 'due to unknown reasons ' + bug_reports_message() except ImportError: KEYRING_AVAILABLE = False + KEYRING_UNAVAILABLE_REASON = ( + 'as the `keyring` module is not installed. ' + 'Please install by running `python -m pip install keyring`. ' + 'Depending on your platform, additional packages may be required ' + 'to access the keyring; see https://pypi.org/project/keyring') +except Exception as _err: + KEYRING_AVAILABLE = False + KEYRING_UNAVAILABLE_REASON = 'as the `keyring` module could not be initialized: {}'.format(_err) -SUPPORTED_BROWSERS = ['brave', 'chrome', 'chromium', 'edge' 'firefox', 'opera', 'safari', 'vivaldi'] -CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge' 'opera', 'vivaldi'} +CHROMIUM_BASED_BROWSERS = {'brave', 'chrome', 'chromium', 'edge', 'opera', 'vivaldi'} +SUPPORTED_BROWSERS = CHROMIUM_BASED_BROWSERS | {'firefox', 'safari'} -class Logger: +class YDLLogger: + def __init__(self, ydl=None): + self._ydl = ydl + self._reported_warnings = set() + def debug(self, message): - print(message) + if self._ydl: + self._ydl.to_screen('[debug] ' + message) def info(self, message): - print(message) + if self._ydl: + self._ydl.to_screen('[Cookies] ' + message) - def warning(self, message): - print(message, file=sys.stderr) + def warning(self, message, only_once=False): + if self._ydl: + if only_once: + if message in self._reported_warnings: + return + else: + self._reported_warnings.add(message) + self._ydl.to_stderr(message) def error(self, message): - print(message, file=sys.stderr) + if self._ydl: + self._ydl.to_stderr(message) def load_cookies(cookie_file, browser_specification, ydl): cookie_jars = [] if browser_specification is not None: - browser_name, profile = _parse_browser_specification(browser_specification) + browser_name, profile = _parse_browser_specification(*browser_specification) cookie_jars.append(extract_cookies_from_browser(browser_name, profile, YDLLogger(ydl))) if cookie_file is not None: @@ -60,7 +99,7 @@ def load_cookies(cookie_file, browser_specification, ydl): return _merge_cookie_jars(cookie_jars) -def extract_cookies_from_browser(browser_name, profile=None, logger=Logger()): +def extract_cookies_from_browser(browser_name, profile=None, logger=YDLLogger()): if browser_name == 'firefox': return _extract_firefox_cookies(profile, logger) elif browser_name == 'safari': @@ -72,7 +111,11 @@ def extract_cookies_from_browser(browser_name, profile=None, logger=Logger()): def _extract_firefox_cookies(profile, logger): - logger.info('extracting cookies from firefox') + logger.info('Extracting cookies from firefox') + if not SQLITE_AVAILABLE: + logger.warning('Cannot extract cookies from firefox without sqlite3 support. ' + 'Please use a python interpreter compiled with sqlite3 support') + return YoutubeDLCookieJar() if profile is None: search_root = _firefox_browser_dir() @@ -99,7 +142,7 @@ def _extract_firefox_cookies(profile, logger): path=path, path_specified=bool(path), secure=is_secure, expires=expiry, discard=False, comment=None, comment_url=None, rest={}) jar.set_cookie(cookie) - logger.info('extracted {} cookies from firefox'.format(len(jar))) + logger.info('Extracted {} cookies from firefox'.format(len(jar))) return jar finally: if cursor is not None: @@ -162,7 +205,7 @@ def _get_chromium_based_browser_settings(browser_name): 'brave': 'Brave', 'chrome': 'Chrome', 'chromium': 'Chromium', - 'edge': 'Mirosoft Edge' if sys.platform == 'darwin' else 'Chromium', + 'edge': 'Microsoft Edge' if sys.platform == 'darwin' else 'Chromium', 'opera': 'Opera' if sys.platform == 'darwin' else 'Chromium', 'vivaldi': 'Vivaldi' if sys.platform == 'darwin' else 'Chrome', }[browser_name] @@ -177,7 +220,13 @@ def _get_chromium_based_browser_settings(browser_name): def _extract_chrome_cookies(browser_name, profile, logger): - logger.info('extracting cookies from {}'.format(browser_name)) + logger.info('Extracting cookies from {}'.format(browser_name)) + + if not SQLITE_AVAILABLE: + logger.warning(('Cannot extract cookies from {} without sqlite3 support. ' + 'Please use a python interpreter compiled with sqlite3 support').format(browser_name)) + return YoutubeDLCookieJar() + config = _get_chromium_based_browser_settings(browser_name) if profile is None: @@ -232,7 +281,7 @@ def _extract_chrome_cookies(browser_name, profile, logger): failed_message = ' ({} could not be decrypted)'.format(failed_cookies) else: failed_message = '' - logger.info('extracted {} cookies from {}{}'.format(len(jar), browser_name, failed_message)) + logger.info('Extracted {} cookies from {}{}'.format(len(jar), browser_name, failed_message)) return jar finally: if cursor is not None: @@ -271,9 +320,9 @@ class ChromeCookieDecryptor: def get_cookie_decryptor(browser_root, browser_keyring_name, logger): if sys.platform in ('linux', 'linux2'): - return LinuxChromeCookieDecryptor(browser_keyring_name) + return LinuxChromeCookieDecryptor(browser_keyring_name, logger) elif sys.platform == 'darwin': - return MacChromeCookieDecryptor(browser_keyring_name) + return MacChromeCookieDecryptor(browser_keyring_name, logger) elif sys.platform == 'win32': return WindowsChromeCookieDecryptor(browser_root, logger) else: @@ -282,52 +331,60 @@ def get_cookie_decryptor(browser_root, browser_keyring_name, logger): class LinuxChromeCookieDecryptor(ChromeCookieDecryptor): - def __init__(self, browser_keyring_name): + def __init__(self, browser_keyring_name, logger): + self._logger = logger self._v10_key = self.derive_key(b'peanuts') - if KEYRING_AVAILABLE: + if KEYRING_AVAILABLE and browser_keyring_name is not None: self._v11_key = self.derive_key(_get_linux_keyring_password(browser_keyring_name)) else: self._v11_key = None - @staticmethod - def derive_key(password): + def derive_key(self, password): # values from # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_linux.cc - return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1, key_length=16) + return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1, key_length=16, logger=self._logger) def decrypt(self, encrypted_value): version = encrypted_value[:3] ciphertext = encrypted_value[3:] if version == b'v10': - return _decrypt_aes_cbc(ciphertext, self._v10_key) + return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger) elif version == b'v11': if self._v11_key is None: - warnings.warn('cannot decrypt cookie as the `keyring` module is not installed') + self._logger.warning('cannot decrypt cookie {}'.format(KEYRING_UNAVAILABLE_REASON), only_once=True) return None - return _decrypt_aes_cbc(ciphertext, self._v11_key) + return _decrypt_aes_cbc(ciphertext, self._v11_key, self._logger) else: return None class MacChromeCookieDecryptor(ChromeCookieDecryptor): - def __init__(self, browser_keyring_name): - self._v10_key = self.derive_key(_get_mac_keyring_password(browser_keyring_name)) + def __init__(self, browser_keyring_name, logger): + self._logger = logger + if browser_keyring_name is not None: + password = _get_mac_keyring_password(browser_keyring_name) + self._v10_key = None if password is None else self.derive_key(password) + else: + self._v10_key = None - @staticmethod - def derive_key(password): + def derive_key(self, password): # values from # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm - return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1003, key_length=16) + return pbkdf2_sha1(password, salt=b'saltysalt', iterations=1003, key_length=16, logger=self._logger) def decrypt(self, encrypted_value): version = encrypted_value[:3] ciphertext = encrypted_value[3:] if version == b'v10': - return _decrypt_aes_cbc(ciphertext, self._v10_key) + if self._v10_key is None: + self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True) + return None + + return _decrypt_aes_cbc(ciphertext, self._v10_key, self._logger) else: # other prefixes are considered 'old data' which were stored as plaintext @@ -346,10 +403,12 @@ class WindowsChromeCookieDecryptor(ChromeCookieDecryptor): if version == b'v10': if self._v10_key is None: - warnings.warn('cannot decrypt cookie') + self._logger.warning('cannot decrypt v10 cookies: no key found', only_once=True) return None elif not CRYPTO_AVAILABLE: - warnings.warn('cannot decrypt cookie as the `pycryptodome` module is not installed') + self._logger.warning('cannot decrypt cookie as the `pycryptodome` module is not installed. ' + 'Please install by running `python -m pip install pycryptodome`', + only_once=True) return None # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc @@ -364,7 +423,7 @@ class WindowsChromeCookieDecryptor(ChromeCookieDecryptor): ciphertext = raw_ciphertext[nonce_length:-authentication_tag_length] authentication_tag = raw_ciphertext[-authentication_tag_length:] - return _decrypt_aes_gcm(ciphertext, self._v10_key, nonce, authentication_tag) + return _decrypt_aes_gcm(ciphertext, self._v10_key, nonce, authentication_tag, self._logger) else: # any other prefix means the data is DPAPI encrypted @@ -387,7 +446,7 @@ def _extract_safari_cookies(profile, logger): cookies_data = f.read() jar = parse_safari_cookies(cookies_data, logger=logger) - logger.info('extracted {} cookies from safari'.format(len(jar))) + logger.info('Extracted {} cookies from safari'.format(len(jar))) return jar @@ -448,7 +507,7 @@ class DataParser: def _mac_absolute_time_to_posix(timestamp): - return (datetime(2001, 1, 1, 0, 0) + timedelta(seconds=timestamp)).timestamp() + return int((datetime(2001, 1, 1, 0, 0, tzinfo=timezone.utc) + timedelta(seconds=timestamp)).timestamp()) def _parse_safari_cookies_header(data, logger): @@ -505,7 +564,7 @@ def _parse_safari_cookies_record(data, jar, logger): p.skip_to(value_offset) value = p.read_cstring() except UnicodeDecodeError: - warnings.warn('failed to parse cookie because UTF-8 decoding failed') + logger.warning('failed to parse cookie because UTF-8 decoding failed') return record_size p.skip_to(record_size, 'space at the end of the record') @@ -519,7 +578,7 @@ def _parse_safari_cookies_record(data, jar, logger): return record_size -def parse_safari_cookies(data, jar=None, logger=Logger()): +def parse_safari_cookies(data, jar=None, logger=YDLLogger()): """ References: - https://github.com/libyal/dtformats/blob/main/documentation/Safari%20Cookies.asciidoc @@ -592,7 +651,7 @@ def _get_windows_v10_key(browser_root, logger): PBKDF2_AVAILABLE = sys.version_info[:2] >= (3, 4) or CRYPTO_AVAILABLE -def pbkdf2_sha1(password, salt, iterations, key_length): +def pbkdf2_sha1(password, salt, iterations, key_length, logger=YDLLogger()): try: from hashlib import pbkdf2_hmac return pbkdf2_hmac('sha1', password, salt, iterations, key_length) @@ -602,12 +661,12 @@ def pbkdf2_sha1(password, salt, iterations, key_length): from Crypto.Hash import SHA1 return PBKDF2(password, salt, key_length, iterations, hmac_hash_module=SHA1) except ImportError: - warnings.warn('PBKDF2 is not available. You must either upgrade to ' - 'python >= 3.4 or install the pycryptodome package') + logger.warning('PBKDF2 is not available. You must either upgrade to ' + 'python >= 3.4 or install the pycryptodome package', only_once=True) return None -def _decrypt_aes_cbc(ciphertext, key, initialization_vector=b' ' * 16): +def _decrypt_aes_cbc(ciphertext, key, logger, initialization_vector=b' ' * 16): plaintext = aes_cbc_decrypt(bytes_to_intlist(ciphertext), bytes_to_intlist(key), bytes_to_intlist(initialization_vector)) @@ -615,22 +674,22 @@ def _decrypt_aes_cbc(ciphertext, key, initialization_vector=b' ' * 16): try: return intlist_to_bytes(plaintext[:-padding_length]).decode('utf-8') except UnicodeDecodeError: - warnings.warn('failed to decrypt cookie because UTF-8 decoding failed. Possibly the key is wrong?') + logger.warning('failed to decrypt cookie because UTF-8 decoding failed. Possibly the key is wrong?') return None -def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag): +def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag, logger): cipher = AES.new(key, AES.MODE_GCM, nonce) try: plaintext = cipher.decrypt_and_verify(ciphertext, authentication_tag) except ValueError: - warnings.warn('failed to decrypt cookie because the MAC check failed. Possibly the key is wrong?') + logger.warning('failed to decrypt cookie because the MAC check failed. Possibly the key is wrong?') return None try: return plaintext.decode('utf-8') except UnicodeDecodeError: - warnings.warn('failed to decrypt cookie because UTF-8 decoding failed. Possibly the key is wrong?') + logger.warning('failed to decrypt cookie because UTF-8 decoding failed. Possibly the key is wrong?') return None @@ -658,7 +717,7 @@ def _decrypt_windows_dpapi(ciphertext, logger): ctypes.byref(blob_out) # pDataOut ) if not ret: - logger.info('failed to decrypt with DPAPI') + logger.warning('failed to decrypt with DPAPI') return None result = ctypes.string_at(blob_out.pbData, blob_out.cbData) @@ -703,36 +762,13 @@ def _merge_cookie_jars(jars): return output_jar -class YDLLogger(Logger): - def __init__(self, ydl): - self._ydl = ydl - - def debug(self, message): - if self._ydl.params.get('verbose'): - self._ydl.to_screen('[debug] ' + message) - - def info(self, message): - self._ydl.to_screen(message) - - def warning(self, message): - self._ydl.to_stderr(message) - - def error(self, message): - self._ydl.to_stderr(message) - - def _is_path(value): return os.path.sep in value -def _parse_browser_specification(browser_specification): - parts = browser_specification.split(':') - while len(parts) < 2: - parts.append('') - parts = [p.strip() or None for p in parts] - if not parts[0] or len(parts) > 2: - raise ValueError('invalid browser specification: "{}"'.format(browser_specification)) - browser_name, profile = parts +def _parse_browser_specification(browser_name, profile=None): + if browser_name not in SUPPORTED_BROWSERS: + raise ValueError('unsupported browser: "{}"'.format(browser_name)) if profile is not None and _is_path(profile): profile = os.path.expanduser(profile) return browser_name, profile diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index dc4bd4a77..6b3e70139 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -76,7 +76,10 @@ class YoutubeBaseInfoExtractor(InfoExtractor): username, password = self._get_login_info() # No authentication to be performed if username is None: - if self._LOGIN_REQUIRED and self._downloader.params.get('cookiefile') is None: + + if (self._LOGIN_REQUIRED + and self._downloader.params.get('cookiefile') is None + and self._downloader.params.get('cookiesfrombrowser') is None): raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True) return True diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 694bb4136..5eae7b707 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -760,8 +760,16 @@ def parseOpts(overrideArguments=None): help='File to read cookies from and dump cookie jar in') filesystem.add_option( '--cookies-from-browser', - dest='cookiesfrombrowser', metavar='BROWSER', - help='Load cookies from a user profile of the given web browser: {}. You can specify an alternative user profile name or directory using "BROWSER:PROFILE_NAME" or "BROWSER:PROFILE_PATH"'.format(', '.join(SUPPORTED_BROWSERS))) + dest='cookiesfrombrowser', metavar='BROWSER[:PROFILE]', + help=( + 'Load cookies from a user profile of the given web browser. ' + 'Currently supported browsers are: {}. ' + 'You can specify the user profile name or directory using ' + '"BROWSER:PROFILE_NAME" or "BROWSER:PROFILE_PATH". ' + 'If no profile is given, the most recently accessed one is used'.format( + '|'.join(sorted(SUPPORTED_BROWSERS)))) + ) + filesystem.add_option( '--cache-dir', dest='cachedir', default=None, metavar='DIR', help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.') From 0aa4970c908a8a640fa9f737b00e6de309badc0d Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sat, 25 Sep 2021 14:08:15 +0100 Subject: [PATCH 25/25] more selective about stripping only a single newline from the end of the MacOS keyring password --- youtube_dl/cookies.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/youtube_dl/cookies.py b/youtube_dl/cookies.py index 595eaede2..22819e7c6 100644 --- a/youtube_dl/cookies.py +++ b/youtube_dl/cookies.py @@ -365,7 +365,7 @@ class MacChromeCookieDecryptor(ChromeCookieDecryptor): def __init__(self, browser_keyring_name, logger): self._logger = logger if browser_keyring_name is not None: - password = _get_mac_keyring_password(browser_keyring_name) + password = _get_mac_keyring_password(browser_keyring_name, logger) self._v10_key = None if password is None else self.derive_key(password) else: self._v10_key = None @@ -610,11 +610,13 @@ def _get_linux_keyring_password(browser_keyring_name): return password.encode('utf-8') -def _get_mac_keyring_password(browser_keyring_name): +def _get_mac_keyring_password(browser_keyring_name, logger): if KEYRING_AVAILABLE: + logger.debug('using keyring to obtain password') password = keyring.get_password('{} Safe Storage'.format(browser_keyring_name), browser_keyring_name) return password.encode('utf-8') else: + logger.debug('using find-generic-password to obtain password') proc = subprocess.Popen(['security', 'find-generic-password', '-w', # write password to stdout '-a', browser_keyring_name, # match 'account' @@ -623,7 +625,10 @@ def _get_mac_keyring_password(browser_keyring_name): stderr=subprocess.DEVNULL) proc.wait() if proc.returncode == 0: - return proc.stdout.read().strip() + stdout = proc.stdout.read() + if stdout[-1:] == b'\n': + stdout = stdout[:-1] + return stdout else: return None