From 8c5e9e0213e214bbfb94dea849efcccb83a86a84 Mon Sep 17 00:00:00 2001 From: Chris Putnam Date: Fri, 22 Jan 2021 00:49:35 -0600 Subject: [PATCH 1/6] [patreon] add support for patreon-hosted video embeds --- youtube_dl/extractor/patreon.py | 43 ++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py index 761a4b1de..8b5e61c52 100644 --- a/youtube_dl/extractor/patreon.py +++ b/youtube_dl/extractor/patreon.py @@ -11,8 +11,10 @@ from ..utils import ( parse_iso8601, str_or_none, try_get, + sanitized_Request, ) - +import json +import re class PatreonIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?patreon\.com/(?:creation\?hid=|posts/(?:[\w-]+-)?)(?P\d+)' @@ -65,23 +67,25 @@ class PatreonIE(InfoExtractor): 'only_matching': True, }] - # Currently Patreon exposes download URL via hidden CSS, so login is not - # needed. Keeping this commented for when this inevitably changes. - ''' def _login(self): username, password = self._get_login_info() if username is None: return login_form = { - 'redirectUrl': 'http://www.patreon.com/', - 'email': username, - 'password': password, + 'data' : { + 'type' : 'user', + 'attributes' : { + 'email' : username, + 'password' : password + }, + 'relationships' : {} + } } request = sanitized_Request( - 'https://www.patreon.com/processLogin', - compat_urllib_parse_urlencode(login_form).encode('utf-8') + 'https://www.patreon.com/api/login?include=campaign,user_location&json-api-version=1.0', + json.dumps(login_form).encode('ascii') ) login_page = self._download_webpage(request, None, note='Logging in') @@ -90,7 +94,6 @@ class PatreonIE(InfoExtractor): def _real_initialize(self): self._login() - ''' def _real_extract(self, url): video_id = self._match_id(url) @@ -146,11 +149,17 @@ class PatreonIE(InfoExtractor): if not info.get('url'): post_file = attributes['post_file'] - ext = determine_ext(post_file.get('name')) - if ext in KNOWN_EXTENSIONS: - info.update({ - 'ext': ext, - 'url': post_file['url'], - }) - + if post_file.get('name') == 'video': + # single video embed + info.update({ + 'url': post_file['url'] + }) + else: + # video is attached as a file + ext = determine_ext(post_file.get('name')) + if ext in KNOWN_EXTENSIONS: + info.update({ + 'ext': ext, + 'url': post_file['url'], + }) return info From 7f0a28e925564b150e7c8b1729bb953cb7bf2af2 Mon Sep 17 00:00:00 2001 From: Chris Putnam Date: Fri, 22 Jan 2021 00:54:47 -0600 Subject: [PATCH 2/6] code style fixes --- youtube_dl/extractor/patreon.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py index 8b5e61c52..19ec3eab7 100644 --- a/youtube_dl/extractor/patreon.py +++ b/youtube_dl/extractor/patreon.py @@ -16,6 +16,7 @@ from ..utils import ( import json import re + class PatreonIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?patreon\.com/(?:creation\?hid=|posts/(?:[\w-]+-)?)(?P\d+)' _TESTS = [{ @@ -150,16 +151,16 @@ class PatreonIE(InfoExtractor): if not info.get('url'): post_file = attributes['post_file'] if post_file.get('name') == 'video': - # single video embed - info.update({ - 'url': post_file['url'] - }) + # single video embed + info.update({ + 'url': post_file['url'] + }) else: - # video is attached as a file - ext = determine_ext(post_file.get('name')) - if ext in KNOWN_EXTENSIONS: - info.update({ - 'ext': ext, - 'url': post_file['url'], - }) + # video is attached as a file + ext = determine_ext(post_file.get('name')) + if ext in KNOWN_EXTENSIONS: + info.update({ + 'ext': ext, + 'url': post_file['url'], + }) return info From f6abfd2627e5410d2c612b976b57ff2cdc7cb3bf Mon Sep 17 00:00:00 2001 From: Chris Putnam Date: Fri, 22 Jan 2021 01:11:04 -0600 Subject: [PATCH 3/6] adding test case for paywalled patreon-hosted video --- youtube_dl/extractor/patreon.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py index 19ec3eab7..6e9a73ca2 100644 --- a/youtube_dl/extractor/patreon.py +++ b/youtube_dl/extractor/patreon.py @@ -19,6 +19,7 @@ import re class PatreonIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?patreon\.com/(?:creation\?hid=|posts/(?:[\w-]+-)?)(?P\d+)' + _NETRC_MACHINE = 'patreon' _TESTS = [{ 'url': 'http://www.patreon.com/creation?hid=743933', 'md5': 'e25505eec1053a6e6813b8ed369875cc', @@ -66,6 +67,11 @@ class PatreonIE(InfoExtractor): }, { 'url': 'https://www.patreon.com/posts/743933', 'only_matching': True, + }, { + # embedded patreon-hosted video, paywalled + 'url': 'https://www.patreon.com/posts/terps-part-1-46181905', + 'only_matching': True, + 'skip': 'Patron-only content' }] def _login(self): From 456987e0229b64d24d9dc1f113bf856ac855de37 Mon Sep 17 00:00:00 2001 From: Chris Putnam Date: Fri, 22 Jan 2021 01:15:51 -0600 Subject: [PATCH 4/6] fixing embarrassing lint issues --- youtube_dl/extractor/patreon.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py index 6e9a73ca2..50a8f10c0 100644 --- a/youtube_dl/extractor/patreon.py +++ b/youtube_dl/extractor/patreon.py @@ -12,6 +12,7 @@ from ..utils import ( str_or_none, try_get, sanitized_Request, + ExtractorError, ) import json import re @@ -80,13 +81,13 @@ class PatreonIE(InfoExtractor): return login_form = { - 'data' : { - 'type' : 'user', - 'attributes' : { - 'email' : username, - 'password' : password + 'data': { + 'type': 'user', + 'attributes': { + 'email': username, + 'password': password }, - 'relationships' : {} + 'relationships': {} } } From 271426aa5851ab2b65436eeb18f69d3fa9dde062 Mon Sep 17 00:00:00 2001 From: Chris Putnam Date: Fri, 22 Jan 2021 01:45:23 -0600 Subject: [PATCH 5/6] use mp4 extension for patreon-hosted videos --- youtube_dl/extractor/patreon.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py index 50a8f10c0..2a0d3ecf3 100644 --- a/youtube_dl/extractor/patreon.py +++ b/youtube_dl/extractor/patreon.py @@ -160,7 +160,8 @@ class PatreonIE(InfoExtractor): if post_file.get('name') == 'video': # single video embed info.update({ - 'url': post_file['url'] + 'ext': 'mp4', + 'url': post_file['url'], }) else: # video is attached as a file From 35f987487b1dfde08b29855faeaff78d2d237d65 Mon Sep 17 00:00:00 2001 From: Chris Putnam Date: Fri, 22 Jan 2021 14:33:26 -0600 Subject: [PATCH 6/6] convention fix, simplified login request --- youtube_dl/extractor/patreon.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py index 2a0d3ecf3..284a0925d 100644 --- a/youtube_dl/extractor/patreon.py +++ b/youtube_dl/extractor/patreon.py @@ -1,21 +1,21 @@ # coding: utf-8 from __future__ import unicode_literals +import json +import re + from .common import InfoExtractor from ..utils import ( clean_html, determine_ext, + ExtractorError, int_or_none, KNOWN_EXTENSIONS, mimetype2ext, parse_iso8601, str_or_none, try_get, - sanitized_Request, - ExtractorError, ) -import json -import re class PatreonIE(InfoExtractor): @@ -91,11 +91,11 @@ class PatreonIE(InfoExtractor): } } - request = sanitized_Request( - 'https://www.patreon.com/api/login?include=campaign,user_location&json-api-version=1.0', - json.dumps(login_form).encode('ascii') - ) - login_page = self._download_webpage(request, None, note='Logging in') + login_page = self._download_webpage( + 'https://www.patreon.com/api/login', + video_id=None, + note='Logging in', + data=json.dumps(login_form).encode('ascii')) if re.search(r'onLoginFailed', login_page): raise ExtractorError('Unable to login, incorrect username and/or password', expected=True)