From ebf0fcd916db3d462ed9ac75bc2d04381e091ba8 Mon Sep 17 00:00:00 2001 From: dirkf Date: Tue, 22 Feb 2022 19:01:44 +0000 Subject: [PATCH] [SendtoNews] Improve _VALID_URL, fix handling API result, add/fix tests Media links now come in `configuration.sources.src` of playlist item --- youtube_dl/extractor/sendtonews.py | 133 ++++++++++++++++++++--------- 1 file changed, 94 insertions(+), 39 deletions(-) diff --git a/youtube_dl/extractor/sendtonews.py b/youtube_dl/extractor/sendtonews.py index 9d9652949..2d63918a3 100644 --- a/youtube_dl/extractor/sendtonews.py +++ b/youtube_dl/extractor/sendtonews.py @@ -5,43 +5,45 @@ import re from .common import InfoExtractor from ..utils import ( + dict_get, + ExtractorError, float_or_none, parse_iso8601, update_url_query, int_or_none, + determine_ext, determine_protocol, + strip_or_none, + try_get, unescapeHTML, + urljoin, ) class SendtoNewsIE(InfoExtractor): - _VALID_URL = r'https?://embed\.sendtonews\.com/player2/embedplayer\.php\?.*\bSC=(?P[0-9A-Za-z-]+)' - - _TEST = { + # TODO handle items with ?fk=XXXX6789&cid=1234 -> SC=XXXX6789-???????-1234 + _VALID_URL = r'https?://embed\.sendtonews\.com/(?:player\d/embed(?:player|code)\.(?:php|js)|oembed/?)\?.*\bSC=(?P[\w-]+)' + _TESTS = [{ # From http://cleveland.cbslocal.com/2016/05/16/indians-score-season-high-15-runs-in-blowout-win-over-reds-rapid-reaction/ 'url': 'http://embed.sendtonews.com/player2/embedplayer.php?SC=GxfCe0Zo7D-175909-5588&type=single&autoplay=on&sound=YES', 'info_dict': { 'id': 'GxfCe0Zo7D-175909-5588' }, - 'playlist_count': 8, - # test the first video only to prevent lengthy tests - 'playlist': [{ - 'info_dict': { - 'id': '240385', - 'ext': 'mp4', - 'title': 'Indians introduce Encarnacion', - 'description': 'Indians president of baseball operations Chris Antonetti and Edwin Encarnacion discuss the slugger\'s three-year contract with Cleveland', - 'duration': 137.898, - 'thumbnail': r're:https?://.*\.jpg$', - 'upload_date': '20170105', - 'timestamp': 1483649762, - }, - }], - 'params': { - # m3u8 download - 'skip_download': True, + 'playlist_count': 10, + }, { + 'url': 'https://embed.sendtonews.com/player4/embedplayer.php?SC=mq3wIKSb68-1206898-8402&type=single', + 'info_dict': { + 'id': '1752278', + 'ext': 'mp4', + 'title': 'Las vegas homebuilders had banner sales year in 2021, and other top stories from January 24, 2022.', + 'description': 'LAS VEGAS HOMEBUILDERS HAD BANNER SALES YEAR IN 2021., and other top stories from January 24, 2022.', + 'timestamp': 1643063702, + 'upload_date': '20220124', + 'thumbnail': r're:https?://.*\.(?:png|jpg)$', + 'categories': ['Business'], + 'tags': list, }, - } + }] _URL_TEMPLATE = '//embed.sendtonews.com/player2/embedplayer.php?SC=%s' @@ -59,17 +61,62 @@ class SendtoNewsIE(InfoExtractor): playlist_id = self._match_id(url) data_url = update_url_query( - url.replace('embedplayer.php', 'data_read.php'), - {'cmd': 'loadInitial'}) + re.sub( + r'(?Pplayer\d)?(?embed.+?|oembed/)\?', + lambda m: '/%s/data_read.php?' % ((m.group('player') or 'player4'), ), + url), + {'cmd': 'loadInitial', 'type': 'single', }) playlist_data = self._download_json(data_url, playlist_id) + playlist = try_get(playlist_data, lambda x: x['playlistData'][0], (dict, list)) or {} + if isinstance(playlist, dict): + err = playlist.get('error', 'No or invalid data returned from API') + raise ExtractorError(err) entries = [] - for video in playlist_data['playlistData'][0]: - info_dict = self._parse_jwplayer_data( - video['jwconfiguration'], - require_title=False, m3u8_id='hls', rtmp_params={'no_resume': True}) + info_dict = {} + for video in playlist: + try: + err = video.get('error') + if err and video.get('S_ID') is not None: + e = ExtractorError(err) + e.msg = err + raise e + except AttributeError: + continue + except ExtractorError as e: + self.report_warning(e.msg, playlist_id) + continue + if 'jwconfiguration' in video: + info_dict.update(self._parse_jwplayer_data( + video['jwconfiguration'], + require_title=False, m3u8_id='hls', rtmp_params={'no_resume': True})) + elif 'configuration' not in video: + continue + else: + fmt_url = urljoin( + url, + try_get(video, lambda x: x['configuration']['sources']['src'])) + if not fmt_url: + continue + video_id = strip_or_none(video.get('SM_ID') or video['configuration']['mediaid']) + title = strip_or_none(video.get('S_headLine') or video['configuration']['title']) + if not video_id or not title: + continue + ext = determine_ext(fmt_url) + if ext == 'm3u8': + formats = self._extract_m3u8_formats( + fmt_url, playlist_id, 'mp4', entry_protocol='m3u8_native', + m3u8_id='hls', fatal=False) + else: + formats = [{ + 'url': fmt_url, + 'ext': ext, + 'width': int_or_none(video.get('SM_M_VIDEO_WIDTH')), + 'height': int_or_none(video.get('SM_M_VIDEO_HEIGHT')), + }] + info_dict['formats'] = formats - for f in info_dict['formats']: + for f in info_dict.get('formats') or []: if f.get('tbr'): continue tbr = int_or_none(self._search_regex( @@ -83,23 +130,31 @@ class SendtoNewsIE(InfoExtractor): self._sort_formats(info_dict['formats'], ('tbr', 'height', 'width', 'format_id')) thumbnails = [] - if video.get('thumbnailUrl'): + for tn_id, tn in (('poster', video['configuration'].get('poster')), + ('normal', video.get('thumbnailUrl')), + ('small', video.get('smThumbnailUrl'))): + tn = urljoin(url, tn) + if not tn: + continue thumbnails.append({ - 'id': 'normal', - 'url': video['thumbnailUrl'], - }) - if video.get('smThumbnailUrl'): - thumbnails.append({ - 'id': 'small', - 'url': video['smThumbnailUrl'], + 'id': tn_id, + 'url': tn, }) info_dict.update({ - 'title': video['S_headLine'].strip(), - 'description': unescapeHTML(video.get('S_fullStory')), + 'id': video_id, + 'title': title, + 'description': unescapeHTML(dict_get(video, ('S_fullStory', 'S_shortSummary'))), 'thumbnails': thumbnails, - 'duration': float_or_none(video.get('SM_length')), + 'duration': float_or_none( + dict_get(video, ('SM_length', 'SM_M_LENGTH')) + or video['configuration'].get('duration')), 'timestamp': parse_iso8601(video.get('S_sysDate'), delimiter=' '), + 'tags': [t for t in video.get('S_tags', '').split(',') if t], + 'categories': [c for c in video.get('S_category', '').split(',') if c], }) entries.append(info_dict) + if len(entries) == 1: + entries[0]['display_id'] = playlist_id + return entries[0] return self.playlist_result(entries, playlist_id)