From 1abfe53cda1399d97ecb67a217b01a36f25e0f0b Mon Sep 17 00:00:00 2001 From: SsSsS <54671367+u-spec-png@users.noreply.github.com> Date: Thu, 17 Jun 2021 12:03:06 +0000 Subject: [PATCH] [newgrounds] fix extarctor and improved extractor (fix #27397) - added thumbnail extractor - added upload date - added description - added view count - fix video extraction - fix playlist extraction --- youtube_dl/extractor/newgrounds.py | 94 ++++++++++++++++++------------ 1 file changed, 58 insertions(+), 36 deletions(-) diff --git a/youtube_dl/extractor/newgrounds.py b/youtube_dl/extractor/newgrounds.py index 82e7cf522..277a96c12 100644 --- a/youtube_dl/extractor/newgrounds.py +++ b/youtube_dl/extractor/newgrounds.py @@ -5,15 +5,18 @@ import re from .common import InfoExtractor from ..utils import ( extract_attributes, - int_or_none, parse_duration, parse_filesize, + parse_count, unified_timestamp, + unified_strdate, + RegexNotFoundError, + ExtractorError, ) class NewgroundsIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?newgrounds\.com/(?:audio/listen|portal/view)/(?P[0-9]+)' + _VALID_URL = r'https?://(?:www\.)?newgrounds\.com/(?:audio/listen|portal/view)/(?P[0-9]+)(?:/format/flash)?' _TESTS = [{ 'url': 'https://www.newgrounds.com/audio/listen/549479', 'md5': 'fe6033d297591288fa1c1f780386f07a', @@ -27,15 +30,14 @@ class NewgroundsIE(InfoExtractor): 'duration': 143, }, }, { - 'url': 'https://www.newgrounds.com/portal/view/673111', - 'md5': '3394735822aab2478c31b1004fe5e5bc', + 'url': 'https://www.newgrounds.com/portal/view/297383', 'info_dict': { - 'id': '673111', + 'id': '297383', 'ext': 'mp4', - 'title': 'Dancin', - 'uploader': 'Squirrelman82', - 'timestamp': 1460256780, - 'upload_date': '20160410', + 'title': 'Metal Gear Awesome', + 'uploader': 'Egoraptor', + 'timestamp': 1140663240, + 'upload_date': '20060223', }, }, { # source format unavailable, additional mp4 formats @@ -59,32 +61,37 @@ class NewgroundsIE(InfoExtractor): webpage = self._download_webpage(url, media_id) title = self._html_search_regex( - r'([^>]+)', webpage, 'title') + r'(.+?)', webpage, 'title') - media_url = self._parse_json(self._search_regex( - r'"url"\s*:\s*("[^"]+"),', webpage, ''), media_id) + try: + media_url = self._parse_json(self._search_regex( + r'"url"\s*:\s*("[^"]+"),', webpage, ''), media_id) + except RegexNotFoundError or ExtractorError: + media_url = None + formats = [] - formats = [{ - 'url': media_url, - 'format_id': 'source', - 'quality': 1, - }] + if media_url: + formats = [{ + 'url': media_url, + 'format_id': 'source', + 'quality': 1, + }] + else: + json_data = self._download_json('https://www.newgrounds.com/portal/video/' + media_id, media_id, headers={ + 'Accept': 'application/json, text/javascript, */*; q=0.01', + 'Accept-Encoding': 'gzip, deflate, br', + 'X-Requested-With': 'XMLHttpRequest', + 'Connection': 'keep-alive', + }) - max_resolution = int_or_none(self._search_regex( - r'max_resolution["\']\s*:\s*(\d+)', webpage, 'max resolution', - default=None)) - if max_resolution: - url_base = media_url.rpartition('.')[0] - for resolution in (360, 720, 1080): - if resolution > max_resolution: - break - formats.append({ - 'url': '%s.%dp.mp4' % (url_base, resolution), - 'format_id': '%dp' % resolution, - 'height': resolution, - }) + for resolution in ('360p', '720p', '1080p'): + if resolution in json_data['sources']: + formats.append({ + 'url': json_data['sources'][resolution][0]['src'], + 'format_id': resolution, + 'height': int(resolution[:-1]), + }) - self._check_formats(formats, media_id) self._sort_formats(formats) uploader = self._html_search_regex( @@ -92,14 +99,26 @@ class NewgroundsIE(InfoExtractor): r'(?:Author|Writer)\s*]+>([^<]+)'), webpage, 'uploader', fatal=False) - timestamp = unified_timestamp(self._html_search_regex( + timestamp = self._html_search_regex( (r'
\s*Uploaded\s*
\s*
([^<]+
\s*
[^<]+)', r'
\s*Uploaded\s*
\s*
([^<]+)'), webpage, 'timestamp', - default=None)) + default=None) + + upload_date = unified_strdate(timestamp) + + timestamp = unified_timestamp(timestamp) + + thumbnail = self._og_search_thumbnail(webpage) + duration = parse_duration(self._search_regex( r'(?s)
\s*Song\s*
\s*
.+?
\s*
([^<]+)', webpage, 'duration', default=None)) + description = self._og_search_description(webpage) + + view_count = parse_count(self._html_search_regex(r'(?s)
\s*Views\s*
\s*
([\d\.,]+)
', webpage, + 'view_count', fatal=False, default=None)) + filesize_approx = parse_filesize(self._html_search_regex( r'(?s)
\s*Song\s*
\s*
(.+?)
', webpage, 'filesize', default=None)) @@ -108,7 +127,6 @@ class NewgroundsIE(InfoExtractor): if '
Song' in webpage: formats[0]['vcodec'] = 'none' - return { 'id': media_id, 'title': title, @@ -116,6 +134,10 @@ class NewgroundsIE(InfoExtractor): 'timestamp': timestamp, 'duration': duration, 'formats': formats, + 'description': description, + 'thumbnail': thumbnail, + 'view_count': view_count, + 'upload_date': upload_date, } @@ -155,14 +177,14 @@ class NewgroundsPlaylistIE(InfoExtractor): entries = [] for a, path, media_id in re.findall( - r'(]+\bhref=["\']/?((?:portal/view|audio/listen)/(\d+))[^>]+>)', + r'(]+href="https?://[^/]+/(audio/listen|portal/view)/([0-9]+)"[^>]+>)', webpage): a_class = extract_attributes(a).get('class') if a_class not in ('item-portalsubmission', 'item-audiosubmission'): continue entries.append( self.url_result( - 'https://www.newgrounds.com/%s' % path, + 'https://www.newgrounds.com/ + path + '/' + media_id, ie=NewgroundsIE.ie_key(), video_id=media_id)) return self.playlist_result(entries, playlist_id, title)