Compare commits

..

4 Commits

Author SHA1 Message Date
Sergey M․
3234272818
[youtube:tab] PEP 8 2020-11-23 22:34:27 +07:00
Sergey M․
9d2c90354f
[youtube:favorites] Restore extractor 2020-11-23 22:33:21 +07:00
Sergey M․
316b10855a
[youtube:tab] Fix some weird typo (closes #27157) 2020-11-23 22:19:25 +07:00
Remita Amine
484fe78737 [pinterest] Add support for large collections(more than 25 pins) 2020-11-23 14:16:38 +01:00
3 changed files with 82 additions and 37 deletions

View File

@ -1491,6 +1491,7 @@ from .yourporn import YourPornIE
from .yourupload import YourUploadIE from .yourupload import YourUploadIE
from .youtube import ( from .youtube import (
YoutubeIE, YoutubeIE,
YoutubeFavouritesIE,
YoutubeHistoryIE, YoutubeHistoryIE,
YoutubeTabIE, YoutubeTabIE,
YoutubePlaylistIE, YoutubePlaylistIE,

View File

@ -1,6 +1,9 @@
# coding: utf-8 # coding: utf-8
from __future__ import unicode_literals from __future__ import unicode_literals
import json
import re
from .common import InfoExtractor from .common import InfoExtractor
from ..compat import compat_str from ..compat import compat_str
from ..utils import ( from ..utils import (
@ -16,12 +19,12 @@ from ..utils import (
class PinterestBaseIE(InfoExtractor): class PinterestBaseIE(InfoExtractor):
_VALID_URL_BASE = r'https?://(?:[^/]+\.)?pinterest\.(?:com|fr|de|ch|jp|cl|ca|it|co\.uk|nz|ru|com\.au|at|pt|co\.kr|es|com\.mx|dk|ph|th|com\.uy|co|nl|info|kr|ie|vn|com\.vn|ec|mx|in|pe|co\.at|hu|co\.in|co\.nz|id|com\.ec|com\.py|tw|be|uk|com\.bo|com\.pe)' _VALID_URL_BASE = r'https?://(?:[^/]+\.)?pinterest\.(?:com|fr|de|ch|jp|cl|ca|it|co\.uk|nz|ru|com\.au|at|pt|co\.kr|es|com\.mx|dk|ph|th|com\.uy|co|nl|info|kr|ie|vn|com\.vn|ec|mx|in|pe|co\.at|hu|co\.in|co\.nz|id|com\.ec|com\.py|tw|be|uk|com\.bo|com\.pe)'
def _extract_resource(self, webpage, video_id): def _call_api(self, resource, video_id, options):
return self._parse_json( return self._download_json(
self._search_regex( 'https://www.pinterest.com/resource/%sResource/get/' % resource,
r'<script[^>]+\bid=["\']initial-state["\'][^>]*>({.+?})</script>', video_id, 'Download %s JSON metadata' % resource, query={
webpage, 'application json'), 'data': json.dumps({'options': options})
video_id)['resourceResponses'] })['resource_response']
def _extract_video(self, data, extract_formats=True): def _extract_video(self, data, extract_formats=True):
video_id = data['id'] video_id = data['id']
@ -128,13 +131,16 @@ class PinterestIE(PinterestBaseIE):
def _real_extract(self, url): def _real_extract(self, url):
video_id = self._match_id(url) video_id = self._match_id(url)
webpage = self._download_webpage(url, video_id) data = self._call_api(
data = self._extract_resource(webpage, video_id)[0]['response']['data'] 'Pin', video_id, {
'field_set_key': 'unauth_react_main_pin',
'id': video_id,
})['data']
return self._extract_video(data) return self._extract_video(data)
class PinterestCollectionIE(PinterestBaseIE): class PinterestCollectionIE(PinterestBaseIE):
_VALID_URL = r'%s/[^/]+/(?P<id>[^/?#&]+)' % PinterestBaseIE._VALID_URL_BASE _VALID_URL = r'%s/(?P<username>[^/]+)/(?P<id>[^/?#&]+)' % PinterestBaseIE._VALID_URL_BASE
_TESTS = [{ _TESTS = [{
'url': 'https://www.pinterest.ca/mashal0407/cool-diys/', 'url': 'https://www.pinterest.ca/mashal0407/cool-diys/',
'info_dict': { 'info_dict': {
@ -142,6 +148,14 @@ class PinterestCollectionIE(PinterestBaseIE):
'title': 'cool diys', 'title': 'cool diys',
}, },
'playlist_count': 8, 'playlist_count': 8,
}, {
'url': 'https://www.pinterest.ca/fudohub/videos/',
'info_dict': {
'id': '682858430939307450',
'title': 'VIDEOS',
},
'playlist_mincount': 365,
'skip': 'Test with extract_formats=False',
}] }]
@classmethod @classmethod
@ -150,27 +164,38 @@ class PinterestCollectionIE(PinterestBaseIE):
PinterestCollectionIE, cls).suitable(url) PinterestCollectionIE, cls).suitable(url)
def _real_extract(self, url): def _real_extract(self, url):
collection_name = self._match_id(url) username, slug = re.match(self._VALID_URL, url).groups()
webpage = self._download_webpage(url, collection_name) board = self._call_api(
resource = self._extract_resource(webpage, collection_name)[1] 'Board', slug, {
'slug': slug,
'username': username
})['data']
board_id = board['id']
options = {
'board_id': board_id,
'page_size': 250,
}
bookmark = None
entries = [] entries = []
for item in resource['response']['data']: while True:
if not isinstance(item, dict) or item.get('type') != 'pin': if bookmark:
continue options['bookmarks'] = [bookmark]
video_id = item.get('id') board_feed = self._call_api('BoardFeed', board_id, options)
if video_id: for item in (board_feed.get('data') or []):
# Some pins may not be available anonymously via pin URL if not isinstance(item, dict) or item.get('type') != 'pin':
# video = self._extract_video(item, extract_formats=False) continue
# video.update({ video_id = item.get('id')
# '_type': 'url_transparent', if video_id:
# 'url': 'https://www.pinterest.com/pin/%s/' % video_id, # Some pins may not be available anonymously via pin URL
# }) # video = self._extract_video(item, extract_formats=False)
# entries.append(video) # video.update({
entries.append(self._extract_video(item)) # '_type': 'url_transparent',
title = try_get( # 'url': 'https://www.pinterest.com/pin/%s/' % video_id,
resource, lambda x: x['options']['board_title'], compat_str) # })
collection_id = try_get( # entries.append(video)
resource, lambda x: x['options']['board_id'], entries.append(self._extract_video(item))
compat_str) or collection_name bookmark = board_feed.get('bookmark')
if not bookmark:
break
return self.playlist_result( return self.playlist_result(
entries, playlist_id=collection_id, playlist_title=title) entries, playlist_id=board_id, playlist_title=board.get('name'))

View File

@ -2621,11 +2621,11 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
'url': 'https://www.youtube.com/c/CommanderVideoHq/live', 'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
'only_matching': True, 'only_matching': True,
}, },
# TODO # TODO
# { # {
# 'url': 'https://www.youtube.com/TheYoungTurks/live', # 'url': 'https://www.youtube.com/TheYoungTurks/live',
# 'only_matching': True, # 'only_matching': True,
# } # }
] ]
def _extract_channel_id(self, webpage): def _extract_channel_id(self, webpage):
@ -3013,7 +3013,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id)) self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
webpage = self._download_webpage(url, item_id) webpage = self._download_webpage(url, item_id)
identity_token = self._search_regex( identity_token = self._search_regex(
r'\bID_TOKEN["\']\s*:\s/l*["\'](.+?)["\']', webpage, r'\bID_TOKEN["\']\s*:\s*["\'](.+?)["\']', webpage,
'identity token', default=None) 'identity token', default=None)
data = self._extract_yt_initial_data(item_id, webpage) data = self._extract_yt_initial_data(item_id, webpage)
tabs = try_get( tabs = try_get(
@ -3149,6 +3149,25 @@ class YoutubeYtUserIE(InfoExtractor):
ie=YoutubeTabIE.ie_key(), video_id=user_id) ie=YoutubeTabIE.ie_key(), video_id=user_id)
class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
IE_NAME = 'youtube:favorites'
IE_DESC = 'YouTube.com favourite videos, ":ytfav" for short (requires authentication)'
_VALID_URL = r'https?://(?:www\.)?youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
_LOGIN_REQUIRED = True
_TESTS = [{
'url': ':ytfav',
'only_matching': True,
}, {
'url': ':ytfavorites',
'only_matching': True,
}]
def _real_extract(self, url):
return self.url_result(
'https://www.youtube.com/playlist?list=LL',
ie=YoutubeTabIE.ie_key())
class YoutubeSearchIE(SearchInfoExtractor, YoutubeBaseInfoExtractor): class YoutubeSearchIE(SearchInfoExtractor, YoutubeBaseInfoExtractor):
IE_DESC = 'YouTube.com searches' IE_DESC = 'YouTube.com searches'
# there doesn't appear to be a real limit, for example if you search for # there doesn't appear to be a real limit, for example if you search for