[twitter] try to use a Generic fallback for unknown twitter cards(closes #25982)

youtube_dl/extractor/extractors.py

@@ -1092,7 +1092,10 @@ from .spike import (
     BellatorIE,
     ParamountNetworkIE,
 )
-from .stitcher import StitcherIE
+from .stitcher import (
+    StitcherIE,
+    StitcherShowIE,
+)
 from .sport5 import Sport5IE
 from .sportbox import SportBoxIE
 from .sportdeutschland import SportDeutschlandIE

youtube_dl/extractor/stitcher.py

@@ -1,19 +1,60 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     clean_html,
     ExtractorError,
     int_or_none,
     str_or_none,
     try_get,
+    url_or_none,
 )
 
 
-class StitcherIE(InfoExtractor):
+class StitcherBaseIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?stitcher\.com/(?:podcast|show)/(?:[^/]+/)+e(?:pisode)?/(?:(?P<display_id>[^/#?&]+?)-)?(?P<id>\d+)(?:[/#?&]|$)'
+    _VALID_URL_BASE = r'https?://(?:www\.)?stitcher\.com/(?:podcast|show)/'
+
+    def _call_api(self, path, video_id, query):
+        resp = self._download_json(
+            'https://api.prod.stitcher.com/' + path,
+            video_id, query=query)
+        error_massage = try_get(resp, lambda x: x['errors'][0]['message'])
+        if error_massage:
+            raise ExtractorError(error_massage, expected=True)
+        return resp['data']
+
+    def _extract_description(self, data):
+        return clean_html(data.get('html_description') or data.get('description'))
+
+    def _extract_audio_url(self, episode):
+        return url_or_none(episode.get('audio_url') or episode.get('guid'))
+
+    def _extract_show_info(self, show):
+        return {
+            'thumbnail': show.get('image_base_url'),
+            'series': show.get('title'),
+        }
+
+    def _extract_episode(self, episode, audio_url, show_info):
+        info = {
+            'id': compat_str(episode['id']),
+            'display_id': episode.get('slug'),
+            'title': episode['title'].strip(),
+            'description': self._extract_description(episode),
+            'duration': int_or_none(episode.get('duration')),
+            'url': audio_url,
+            'vcodec': 'none',
+            'timestamp': int_or_none(episode.get('date_published')),
+            'season_number': int_or_none(episode.get('season')),
+            'season_id': str_or_none(episode.get('season_id')),
+        }
+        info.update(show_info)
+        return info
+
+
+class StitcherIE(StitcherBaseIE):
+    _VALID_URL = StitcherBaseIE._VALID_URL_BASE + r'(?:[^/]+/)+e(?:pisode)?/(?:[^/#?&]+-)?(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.stitcher.com/podcast/the-talking-machines/e/40789481?autoplay=true',
         'md5': 'e9635098e0da10b21a0e2b85585530f6',
@@ -24,8 +65,9 @@ class StitcherIE(InfoExtractor):
             'description': 'md5:547adb4081864be114ae3831b4c2b42f',
             'duration': 1604,
             'thumbnail': r're:^https?://.*\.jpg',
-            'upload_date': '20180126',
+            'upload_date': '20151008',
-            'timestamp': 1516989316,
+            'timestamp': 1444285800,
+            'series': 'Talking Machines',
         },
     }, {
         'url': 'http://www.stitcher.com/podcast/panoply/vulture-tv/e/the-rare-hourlong-comedy-plus-40846275?autoplay=true',
@@ -55,33 +97,47 @@ class StitcherIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        display_id, audio_id = re.match(self._VALID_URL, url).groups()
+        audio_id = self._match_id(url)
+        data = self._call_api(
+            'shows/episodes', audio_id, {'episode_ids': audio_id})
+        episode = data['episodes'][0]
+        audio_url = self._extract_audio_url(episode)
+        if not audio_url:
+            self.raise_login_required()
+        show = try_get(data, lambda x: x['shows'][0], dict) or {}
+        return self._extract_episode(
+            episode, audio_url, self._extract_show_info(show))
 
-        resp = self._download_json(
-            'https://api.prod.stitcher.com/episode/' + audio_id,
-            display_id or audio_id)
-        episode = try_get(resp, lambda x: x['data']['episodes'][0], dict)
-        if not episode:
-            raise ExtractorError(resp['errors'][0]['message'], expected=True)
 
-        title = episode['title'].strip()
+class StitcherShowIE(StitcherBaseIE):
-        audio_url = episode['audio_url']
+    _VALID_URL = StitcherBaseIE._VALID_URL_BASE + r'(?P<id>[^/#?&]+)/?(?:[?#&]|$)'
+    _TESTS = [{
+        'url': 'http://www.stitcher.com/podcast/the-talking-machines',
+        'info_dict': {
+            'id': 'the-talking-machines',
+            'title': 'Talking Machines',
+            'description': 'md5:831f0995e40f26c10231af39cf1ebf0b',
+        },
+        'playlist_mincount': 106,
+    }, {
+        'url': 'https://www.stitcher.com/show/the-talking-machines',
+        'only_matching': True,
+    }]
 
-        thumbnail = None
+    def _real_extract(self, url):
-        show_id = episode.get('show_id')
+        show_slug = self._match_id(url)
-        if show_id and episode.get('classic_id') != -1:
+        data = self._call_api(
-            thumbnail = 'https://stitcher-classic.imgix.net/feedimages/%s.jpg' % show_id
+            'search/show/%s/allEpisodes' % show_slug, show_slug, {'count': 10000})
+        show = try_get(data, lambda x: x['shows'][0], dict) or {}
+        show_info = self._extract_show_info(show)
 
-        return {
+        entries = []
-            'id': audio_id,
+        for episode in (data.get('episodes') or []):
-            'display_id': display_id,
+            audio_url = self._extract_audio_url(episode)
-            'title': title,
+            if not audio_url:
-            'description': clean_html(episode.get('html_description') or episode.get('description')),
+                continue
-            'duration': int_or_none(episode.get('duration')),
+            entries.append(self._extract_episode(episode, audio_url, show_info))
-            'thumbnail': thumbnail,
+
-            'url': audio_url,
+        return self.playlist_result(
-            'vcodec': 'none',
+            entries, show_slug, show.get('title'),
-            'timestamp': int_or_none(episode.get('date_created')),
+            self._extract_description(show))
-            'season_number': int_or_none(episode.get('season')),
-            'season_id': str_or_none(episode.get('season_id')),
-        }

youtube_dl/extractor/stv.py

@@ -8,13 +8,17 @@ from ..utils import (
     compat_str,
     float_or_none,
     int_or_none,
+    smuggle_url,
+    str_or_none,
+    try_get,
 )
 
 
 class STVPlayerIE(InfoExtractor):
     IE_NAME = 'stv:player'
     _VALID_URL = r'https?://player\.stv\.tv/(?P<type>episode|video)/(?P<id>[a-z0-9]{4})'
-    _TEST = {
+    _TESTS = [{
+        # shortform
         'url': 'https://player.stv.tv/video/4gwd/emmerdale/60-seconds-on-set-with-laura-norton/',
         'md5': '5adf9439c31d554f8be0707c7abe7e0a',
         'info_dict': {
@@ -27,7 +31,11 @@ class STVPlayerIE(InfoExtractor):
             'uploader_id': '1486976045',
         },
         'skip': 'this resource is unavailable outside of the UK',
-    }
+    }, {
+        # episodes
+        'url': 'https://player.stv.tv/episode/4125/jennifer-saunders-memory-lane',
+        'only_matching': True,
+    }]
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1486976045/default_default/index.html?videoId=%s'
     _PTYPE_MAP = {
         'episode': 'episodes',
@@ -36,11 +44,31 @@ class STVPlayerIE(InfoExtractor):
 
     def _real_extract(self, url):
         ptype, video_id = re.match(self._VALID_URL, url).groups()
-        resp = self._download_json(
-            'https://player.api.stv.tv/v1/%s/%s' % (self._PTYPE_MAP[ptype], video_id),
-            video_id)
 
-        result = resp['results']
+        webpage = self._download_webpage(url, video_id, fatal=False) or ''
+        props = (self._parse_json(self._search_regex(
+            r'<script[^>]+id="__NEXT_DATA__"[^>]*>({.+?})</script>',
+            webpage, 'next data', default='{}'), video_id,
+            fatal=False) or {}).get('props') or {}
+        player_api_cache = try_get(
+            props, lambda x: x['initialReduxState']['playerApiCache']) or {}
+
+        api_path, resp = None, {}
+        for k, v in player_api_cache.items():
+            if k.startswith('/episodes/') or k.startswith('/shortform/'):
+                api_path, resp = k, v
+                break
+        else:
+            episode_id = str_or_none(try_get(
+                props, lambda x: x['pageProps']['episodeId']))
+            api_path = '/%s/%s' % (self._PTYPE_MAP[ptype], episode_id or video_id)
+
+        result = resp.get('results')
+        if not result:
+            resp = self._download_json(
+                'https://player.api.stv.tv/v1' + api_path, video_id)
+            result = resp['results']
+
         video = result['video']
         video_id = compat_str(video['id'])
 
@@ -57,7 +85,7 @@ class STVPlayerIE(InfoExtractor):
         return {
             '_type': 'url_transparent',
             'id': video_id,
-            'url': self.BRIGHTCOVE_URL_TEMPLATE % video_id,
+            'url': smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % video_id, {'geo_countries': ['GB']}),
             'description': result.get('summary'),
             'duration': float_or_none(video.get('length'), 1000),
             'subtitles': subtitles,

youtube_dl/extractor/twitter.py

@@ -251,10 +251,10 @@ class TwitterIE(TwitterBaseIE):
         'info_dict': {
             'id': '700207533655363584',
             'ext': 'mp4',
-            'title': 'simon vetugo - BEAT PROD: @suhmeduh #Damndaniel',
+            'title': 'simon vertugo - BEAT PROD: @suhmeduh #Damndaniel',
             'description': 'BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ',
             'thumbnail': r're:^https?://.*\.jpg',
-            'uploader': 'simon vetugo',
+            'uploader': 'simon vertugo',
             'uploader_id': 'simonvertugo',
             'duration': 30.0,
             'timestamp': 1455777459,
@@ -312,6 +312,7 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1492000653,
             'upload_date': '20170412',
         },
+        'skip': 'Account suspended',
     }, {
         'url': 'https://twitter.com/i/web/status/910031516746514432',
         'info_dict': {
@@ -380,6 +381,14 @@ class TwitterIE(TwitterBaseIE):
         # promo_video_website card
         'url': 'https://twitter.com/GunB1g/status/1163218564784017422',
         'only_matching': True,
+    }, {
+        # promo_video_convo card
+        'url': 'https://twitter.com/poco_dandy/status/1047395834013384704',
+        'only_matching': True,
+    }, {
+        # appplayer card
+        'url': 'https://twitter.com/poco_dandy/status/1150646424461176832',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -462,7 +471,25 @@ class TwitterIE(TwitterBaseIE):
                     return try_get(o, lambda x: x[x['type'].lower() + '_value'])
 
                 card_name = card['name'].split(':')[-1]
-                if card_name in ('amplify', 'promo_video_website'):
+                if card_name == 'player':
+                    info.update({
+                        '_type': 'url',
+                        'url': get_binding_value('player_url'),
+                    })
+                elif card_name == 'periscope_broadcast':
+                    info.update({
+                        '_type': 'url',
+                        'url': get_binding_value('url') or get_binding_value('player_url'),
+                        'ie_key': PeriscopeIE.ie_key(),
+                    })
+                elif card_name == 'broadcast':
+                    info.update({
+                        '_type': 'url',
+                        'url': get_binding_value('broadcast_url'),
+                        'ie_key': TwitterBroadcastIE.ie_key(),
+                    })
+                # amplify, promo_video_website, promo_video_convo, appplayer, ...
+                else:
                     is_amplify = card_name == 'amplify'
                     vmap_url = get_binding_value('amplify_url_vmap') if is_amplify else get_binding_value('player_stream_url')
                     content_id = get_binding_value('%s_content_id' % (card_name if is_amplify else 'player'))
@@ -488,25 +515,6 @@ class TwitterIE(TwitterBaseIE):
                         'duration': int_or_none(get_binding_value(
                             'content_duration_seconds')),
                     })
-                elif card_name == 'player':
-                    info.update({
-                        '_type': 'url',
-                        'url': get_binding_value('player_url'),
-                    })
-                elif card_name == 'periscope_broadcast':
-                    info.update({
-                        '_type': 'url',
-                        'url': get_binding_value('url') or get_binding_value('player_url'),
-                        'ie_key': PeriscopeIE.ie_key(),
-                    })
-                elif card_name == 'broadcast':
-                    info.update({
-                        '_type': 'url',
-                        'url': get_binding_value('broadcast_url'),
-                        'ie_key': TwitterBroadcastIE.ie_key(),
-                    })
-                else:
-                    raise ExtractorError('Unsupported Twitter Card.')
             else:
                 expanded_url = try_get(status, lambda x: x['entities']['urls'][0]['expanded_url'])
                 if not expanded_url: