From ddacd2d6cf40327470d97e9e3fd1f6e9555876a7 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Wed, 2 Jun 2021 21:49:27 +0100 Subject: [PATCH] added --cookies-from-browser --- test/test_cookies.py | 42 ++++ youtube_dl/YoutubeDL.py | 11 +- youtube_dl/__init__.py | 1 + youtube_dl/cookies.py | 475 ++++++++++++++++++++++++++++++++++++++++ youtube_dl/options.py | 5 + 5 files changed, 526 insertions(+), 8 deletions(-) create mode 100644 test/test_cookies.py create mode 100644 youtube_dl/cookies.py diff --git a/test/test_cookies.py b/test/test_cookies.py new file mode 100644 index 000000000..c30683ef0 --- /dev/null +++ b/test/test_cookies.py @@ -0,0 +1,42 @@ +import unittest +from unittest import mock + +from youtube_dl.cookies import LinuxChromeCookieDecryptor, WindowsChromeCookieDecryptor, CRYPTO_AVAILABLE, \ + MacChromeCookieDecryptor + + +@unittest.skipIf(not CRYPTO_AVAILABLE, 'cryptography library not available') +class TestCookies(unittest.TestCase): + @mock.patch('youtube_dl.cookies._get_linux_keyring_password') + def test_chrome_cookie_decryptor_linux_v10(self, mock_get_keyring_password): + mock_get_keyring_password.return_value = '' + encrypted_value = b'v10\xccW%\xcd\xe6\xe6\x9fM" \xa7\xb0\xca\xe4\x07\xd6' + value = 'USD' + decryptor = LinuxChromeCookieDecryptor('Chrome') + assert decryptor.decrypt(encrypted_value) == value + + @mock.patch('youtube_dl.cookies._get_linux_keyring_password') + def test_chrome_cookie_decryptor_linux_v11(self, mock_get_keyring_password): + mock_get_keyring_password.return_value = '' + encrypted_value = b'v11#\x81\x10>`w\x8f)\xc0\xb2\xc1\r\xf4\x1al\xdd\x93\xfd\xf8\xf8N\xf2\xa9\x83\xf1\xe9o\x0elVQd' + value = 'tz=Europe.London' + decryptor = LinuxChromeCookieDecryptor('Chrome') + assert decryptor.decrypt(encrypted_value) == value + + @mock.patch('youtube_dl.cookies._get_windows_v10_password') + def test_chrome_cookie_decryptor_windows_v10(self, mock_get_windows_v10_password): + mock_get_windows_v10_password.return_value = b'Y\xef\xad\xad\xeerp\xf0Y\xe6\x9b\x12\xc2 0: + failed_message = ' ({} could not be decrypted)'.format(failed_cookies) + else: + failed_message = '' + print('extracted {} cookies from {}{}'.format(len(jar), browser_name, failed_message)) + return jar + finally: + if cursor is not None: + cursor.connection.close() + + +class ChromeCookieDecryptor(ABC): + """ + Overview: + + Linux: + - cookies are either v10 or v11 + - v10: AES-CBC encrypted with a fixed key + - v11: AES-CBC encrypted with an OS protected key (keyring) + - v11 keys can be stored in various places depending on the activate desktop environment [2] + + Mac: + - cookies are either v10 or not v10 + - v10: AES-CBC encrypted (with more iterations than Linux) with an OS protected key (keyring) + - not v10: 'old data' stored as plaintext + + Windows: + - cookies are either v10 or not v10 + - v10: AES-GCM encrypted with a key which is encrypted with DPAPI + - not v10: encrypted with DPAPI + + Sources: + - [1] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/ + - [2] https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/key_storage_linux.cc + - KeyStorageLinux::CreateService + """ + + @abstractmethod + def decrypt(self, encrypted_value): + raise NotImplementedError + + +def get_cookie_decryptor(browser_root, browser_keyring_name): + if sys.platform in ('linux', 'linux2'): + return LinuxChromeCookieDecryptor(browser_keyring_name) + elif sys.platform == 'darwin': + return MacChromeCookieDecryptor(browser_keyring_name) + elif sys.platform == 'win32': + return WindowsChromeCookieDecryptor(browser_root) + else: + raise NotImplementedError('Chrome cookie decryption is not supported ' + 'on this platform: {}'.format(sys.platform)) + + +class LinuxChromeCookieDecryptor(ChromeCookieDecryptor): + def __init__(self, browser_keyring_name): + self._v10_key = None + self._v11_key = None + if CRYPTO_AVAILABLE: + # values from + # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_linux.cc + self._v10_key = PBKDF2HMAC(algorithm=SHA1(), length=16, + salt=b'saltysalt', iterations=1).derive(b'peanuts') + + if KEYRING_AVAILABLE: + password = _get_linux_keyring_password(browser_keyring_name) + self._v11_key = PBKDF2HMAC(algorithm=SHA1(), length=16, + salt=b'saltysalt', iterations=1).derive(password.encode('utf-8')) + + def decrypt(self, encrypted_value): + version = encrypted_value[:3] + ciphertext = encrypted_value[3:] + + if version == b'v10': + if self._v10_key is None: + warnings.warn('cannot decrypt cookie as the cryptography module is not installed') + return None + return _decrypt_aes_cbc(ciphertext, self._v10_key) + + elif version == b'v11': + if self._v11_key is None: + warnings.warn('cannot decrypt cookie as the cryptography or keyring modules are not installed') + return None + return _decrypt_aes_cbc(ciphertext, self._v11_key) + + else: + return None + + +class MacChromeCookieDecryptor(ChromeCookieDecryptor): + def __init__(self, browser_keyring_name): + self._v10_key = None + if CRYPTO_AVAILABLE: + # values from + # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm + password = _get_mac_keyring_password(browser_keyring_name) + self._v10_key = PBKDF2HMAC(algorithm=SHA1(), length=16, + salt=b'saltysalt', iterations=1003).derive(password.encode('utf-8')) + + def decrypt(self, encrypted_value): + version = encrypted_value[:3] + ciphertext = encrypted_value[3:] + + if version == b'v10': + if self._v10_key is None: + warnings.warn('cannot decrypt cookie as the cryptography module is not installed') + return None + return _decrypt_aes_cbc(ciphertext, self._v10_key) + + else: + # other prefixes are considered 'old data' which were stored as plaintext + # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_mac.mm + return encrypted_value + + +class WindowsChromeCookieDecryptor(ChromeCookieDecryptor): + def __init__(self, browser_root): + self._v10_key = _get_windows_v10_password(browser_root) + + def decrypt(self, encrypted_value): + version = encrypted_value[:3] + ciphertext = encrypted_value[3:] + + if version == b'v10': + if self._v10_key is None: + warnings.warn('cannot decrypt cookie') + return None + + # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc + # kNonceLength + nonce_length = 96 // 8 + # boringssl + # EVP_AEAD_AES_GCM_TAG_LEN + authentication_tag_length = 16 + + raw_ciphertext = ciphertext + nonce = raw_ciphertext[:nonce_length] + ciphertext = raw_ciphertext[nonce_length:-authentication_tag_length] + authentication_tag = raw_ciphertext[-authentication_tag_length:] + + return _decrypt_aes_gcm(ciphertext, self._v10_key, nonce, authentication_tag) + + else: + # any other prefix means the data is DPAPI encrypted + # https://chromium.googlesource.com/chromium/src/+/refs/heads/main/components/os_crypt/os_crypt_win.cc + return _decrypt_windows_dpapi(encrypted_value) + + +def _get_linux_keyring_password(browser_keyring_name): + if KEYRING_AVAILABLE: + password = keyring.get_password('{} Keys'.format(browser_keyring_name), + '{} Safe Storage'.format(browser_keyring_name)) + if password is None: + # this sometimes occurs in KDE because chrome does not check hasEntry and instead + # just tries to read the value (which kwallet returns "") whereas keyring checks hasEntry + # to verify this: + # dbus-monitor "interface='org.kde.KWallet'" "type=method_return" + # while starting chrome. + # this may be a bug as the intended behaviour is to generate a random password and store + # it, but that doesn't matter here. + password = '' + return password + + +def _get_mac_keyring_password(browser_keyring_name): + if KEYRING_AVAILABLE: + return keyring.get_password('{} Safe Storage'.format(browser_keyring_name), browser_keyring_name) + else: + proc = subprocess.Popen(['security', 'find-generic-password', + '-w', # write password to stdout + '-a', browser_keyring_name, # match 'account' + '-s', '{} Safe Storage'.format(browser_keyring_name)], # match 'service' + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL) + proc.wait() + if proc.returncode == 0: + return proc.stdout.read().decode('utf-8').strip() + else: + return None + + +def _get_windows_v10_password(browser_root): + path = _find_most_recently_used_file(browser_root, 'Local State') + if path is None: + print('could not find local state file') + return None + with open(path, 'r') as f: + data = json.load(f) + try: + base64_password = data['os_crypt']['encrypted_key'] + except KeyError: + return None + encrypted_password = base64.b64decode(base64_password) + prefix = b'DPAPI' + if not encrypted_password.startswith(prefix): + return None + return _decrypt_windows_dpapi(encrypted_password[len(prefix):]) + + +def _decrypt_aes_cbc(ciphertext, key): + cipher = Cipher(algorithm=AES(key), mode=CBC(initialization_vector=b' ' * 16)) + decryptor = cipher.decryptor() + plaintext = decryptor.update(ciphertext) + decryptor.finalize() + padding_length = plaintext[-1] + try: + return plaintext[:-padding_length].decode('utf-8') + except UnicodeDecodeError: + warnings.warn('failed to decrypt cookie because UTF-8 decoding failed. Possibly the key is wrong?') + return None + + +def _decrypt_aes_gcm(ciphertext, key, nonce, authentication_tag): + cipher = Cipher(algorithm=AES(key), mode=GCM(nonce, tag=authentication_tag)) + decryptor = cipher.decryptor() + plaintext = decryptor.update(ciphertext) + decryptor.finalize() + try: + return plaintext.decode('utf-8') + except UnicodeDecodeError: + warnings.warn('failed to decrypt cookie because UTF-8 decoding failed. Possibly the key is wrong?') + return None + + +def _decrypt_windows_dpapi(ciphertext): + """ + References: + - https://docs.microsoft.com/en-us/windows/win32/api/dpapi/nf-dpapi-cryptunprotectdata + """ + class DATA_BLOB(ctypes.Structure): + _fields_ = [('cbData', DWORD), + ('pbData', ctypes.POINTER(ctypes.c_char))] + + buffer = ctypes.create_string_buffer(ciphertext) + blob_in = DATA_BLOB(ctypes.sizeof(buffer), buffer) + blob_out = DATA_BLOB() + ret = ctypes.windll.crypt32.CryptUnprotectData( + ctypes.byref(blob_in), # pDataIn + None, # ppszDataDescr: human readable description of pDataIn + None, # pOptionalEntropy: salt? + None, # pvReserved: must be NULL + None, # pPromptStruct: information about prompts to display + 0, # dwFlags + ctypes.byref(blob_out) # pDataOut + ) + if not ret: + print('failed to decrypt with DPAPI') + return None + + result = ctypes.string_at(blob_out.pbData, blob_out.cbData) + ctypes.windll.kernel32.LocalFree(blob_out.pbData) + return result + + +def _config_home(): + return os.environ.get('XDG_CONFIG_HOME', os.path.expanduser('~/.config')) + + +def _open_database_copy(database_path, tmpdir): + # cannot open sqlite databases if they are already in use (e.g. by the browser) + database_copy_path = os.path.join(tmpdir, 'temporary.sqlite') + shutil.copy(database_path, database_copy_path) + conn = sqlite3.connect(database_copy_path) + return conn.cursor() + + +def _find_most_recently_used_file(root, filename): + # if there are multiple browser profiles, take the most recently used one + paths = glob.iglob(os.path.join(root, '**', filename), recursive=True) + paths = [path for path in paths if os.path.isfile(path)] + return None if not paths else max(paths, key=lambda path: os.lstat(path).st_mtime) + + +def _merge_cookie_jars(jars): + output_jar = YoutubeDLCookieJar() + for jar in jars: + for cookie in jar: + output_jar.set_cookie(cookie) + if jar.filename is not None: + output_jar.filename = jar.filename + return output_jar + + +def load_cookies(cookie_file, browser): + cookie_jars = [] + if browser is not None: + cookie_jars.append(extract_cookies_from_browser(browser)) + + if cookie_file is not None: + cookie_file = expand_path(cookie_file) + jar = YoutubeDLCookieJar(cookie_file) + if os.access(cookie_file, os.R_OK): + jar.load(ignore_discard=True, ignore_expires=True) + cookie_jars.append(jar) + + return _merge_cookie_jars(cookie_jars) diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 0a0641bd4..457a041e2 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -17,6 +17,7 @@ from .utils import ( preferredencoding, write_string, ) +from .cookies import SUPPORTED_BROWSERS from .version import __version__ @@ -757,6 +758,10 @@ def parseOpts(overrideArguments=None): '--cookies', dest='cookiefile', metavar='FILE', help='File to read cookies from and dump cookie jar in') + filesystem.add_option( + '--cookies-from-browser', + dest='cookiesfrombrowser', metavar='BROWSER', + help='Browser to load cookies from: {}'.format(', '.join(SUPPORTED_BROWSERS))) filesystem.add_option( '--cache-dir', dest='cachedir', default=None, metavar='DIR', help='Location in the filesystem where youtube-dl can store some downloaded information permanently. By default $XDG_CACHE_HOME/youtube-dl or ~/.cache/youtube-dl . At the moment, only YouTube player files (for videos with obfuscated signatures) are cached, but that may change.')