[dplay] Add support for discoveryplus.com (closes #24698)

youtube_dl/extractor/dplay.py

@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import json
 import re
 
 from .common import InfoExtractor
@@ -151,56 +152,79 @@ class DPlayIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    def _process_errors(self, e, geo_countries):
+        info = self._parse_json(e.cause.read().decode('utf-8'), None)
+        error = info['errors'][0]
+        error_code = error.get('code')
+        if error_code == 'access.denied.geoblocked':
+            self.raise_geo_restricted(countries=geo_countries)
+        elif error_code in ('access.denied.missingpackage', 'invalid.token'):
+            raise ExtractorError(
+                'This video is only available for registered users. You may want to use --cookies.', expected=True)
+        raise ExtractorError(info['errors'][0]['detail'], expected=True)
+
+    def _update_disco_api_headers(self, headers, disco_base, display_id, realm):
+        headers['Authorization'] = 'Bearer ' + self._download_json(
+            disco_base + 'token', display_id, 'Downloading token',
+            query={
+                'realm': realm,
+            })['data']['attributes']['token']
+
+    def _download_video_playback_info(self, disco_base, video_id, headers):
+        streaming = self._download_json(
+            disco_base + 'playback/videoPlaybackInfo/' + video_id,
+            video_id, headers=headers)['data']['attributes']['streaming']
+        streaming_list = []
+        for format_id, format_dict in streaming.items():
+            streaming_list.append({
+                'type': format_id,
+                'url': format_dict.get('url'),
+            })
+        return streaming_list
+
     def _get_disco_api_info(self, url, display_id, disco_host, realm, country):
         geo_countries = [country.upper()]
         self._initialize_geo_bypass({
             'countries': geo_countries,
         })
         disco_base = 'https://%s/' % disco_host
-        token = self._download_json(
-            disco_base + 'token', display_id, 'Downloading token',
-            query={
-                'realm': realm,
-            })['data']['attributes']['token']
         headers = {
             'Referer': url,
-            'Authorization': 'Bearer ' + token,
         }
-        video = self._download_json(
+        self._update_disco_api_headers(headers, disco_base, display_id, realm)
-            disco_base + 'content/videos/' + display_id, display_id,
+        try:
-            headers=headers, query={
+            video = self._download_json(
-                'fields[channel]': 'name',
+                disco_base + 'content/videos/' + display_id, display_id,
-                'fields[image]': 'height,src,width',
+                headers=headers, query={
-                'fields[show]': 'name',
+                    'fields[channel]': 'name',
-                'fields[tag]': 'name',
+                    'fields[image]': 'height,src,width',
-                'fields[video]': 'description,episodeNumber,name,publishStart,seasonNumber,videoDuration',
+                    'fields[show]': 'name',
-                'include': 'images,primaryChannel,show,tags'
+                    'fields[tag]': 'name',
-            })
+                    'fields[video]': 'description,episodeNumber,name,publishStart,seasonNumber,videoDuration',
+                    'include': 'images,primaryChannel,show,tags'
+                })
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+                self._process_errors(e, geo_countries)
+            raise
         video_id = video['data']['id']
         info = video['data']['attributes']
         title = info['name'].strip()
         formats = []
         try:
-            streaming = self._download_json(
+            streaming = self._download_video_playback_info(
-                disco_base + 'playback/videoPlaybackInfo/' + video_id,
+                disco_base, video_id, headers)
-                display_id, headers=headers)['data']['attributes']['streaming']
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                info = self._parse_json(e.cause.read().decode('utf-8'), display_id)
+                self._process_errors(e, geo_countries)
-                error = info['errors'][0]
-                error_code = error.get('code')
-                if error_code == 'access.denied.geoblocked':
-                    self.raise_geo_restricted(countries=geo_countries)
-                elif error_code == 'access.denied.missingpackage':
-                    self.raise_login_required()
-                raise ExtractorError(info['errors'][0]['detail'], expected=True)
             raise
-        for format_id, format_dict in streaming.items():
+        for format_dict in streaming:
             if not isinstance(format_dict, dict):
                 continue
             format_url = format_dict.get('url')
             if not format_url:
                 continue
+            format_id = format_dict.get('type')
             ext = determine_ext(format_url)
             if format_id == 'dash' or ext == 'mpd':
                 formats.extend(self._extract_mpd_formats(
@@ -268,3 +292,46 @@ class DPlayIE(InfoExtractor):
         host = 'disco-api.' + domain if domain[0] == 'd' else 'eu2-prod.disco-api.com'
         return self._get_disco_api_info(
             url, display_id, host, 'dplay' + country, country)
+
+
+class DiscoveryPlusIE(DPlayIE):
+    _VALID_URL = r'https?://(?:www\.)?discoveryplus\.com/video/(?P<id>[^/]+/[^/]+)'
+    _TESTS = [{
+        'url': 'https://www.discoveryplus.com/video/property-brothers-forever-home/food-and-family',
+        'info_dict': {
+            'id': '1140794',
+            'display_id': 'property-brothers-forever-home/food-and-family',
+            'ext': 'mp4',
+            'title': 'Food and Family',
+            'description': 'The brothers help a Richmond family expand their single-level home.',
+            'duration': 2583.113,
+            'timestamp': 1609304400,
+            'upload_date': '20201230',
+            'creator': 'HGTV',
+            'series': 'Property Brothers: Forever Home',
+            'season_number': 1,
+            'episode_number': 1,
+        },
+        'skip': 'Available for Premium users',
+    }]
+
+    def _update_disco_api_headers(self, headers, disco_base, display_id, realm):
+        headers['x-disco-client'] = 'WEB:UNKNOWN:dplus_us:15.0.0'
+
+    def _download_video_playback_info(self, disco_base, video_id, headers):
+        return self._download_json(
+            disco_base + 'playback/v3/videoPlaybackInfo',
+            video_id, headers=headers, data=json.dumps({
+                'deviceInfo': {
+                    'adBlocker': False,
+                },
+                'videoId': video_id,
+                'wisteriaProperties': {
+                    'platform': 'desktop',
+                },
+            }).encode('utf-8'))['data']['attributes']['streaming']
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        return self._get_disco_api_info(
+            url, display_id, 'us1-prod-direct.discoveryplus.com', 'go', 'us')

youtube_dl/extractor/extractors.py

@@ -288,7 +288,10 @@ from .douyutv import (
     DouyuShowIE,
     DouyuTVIE,
 )
-from .dplay import DPlayIE
+from .dplay import (
+    DPlayIE,
+    DiscoveryPlusIE,
+)
 from .dreisat import DreiSatIE
 from .drbonanza import DRBonanzaIE
 from .drtuber import DrTuberIE
@@ -1057,6 +1060,11 @@ from .shared import (
     VivoIE,
 )
 from .showroomlive import ShowRoomLiveIE
+from .simplecast import (
+    SimplecastIE,
+    SimplecastEpisodeIE,
+    SimplecastPodcastIE,
+)
 from .sina import SinaIE
 from .sixplay import SixPlayIE
 from .skyit import (

youtube_dl/extractor/generic.py

@@ -129,6 +129,7 @@ from .odnoklassniki import OdnoklassnikiIE
 from .kinja import KinjaEmbedIE
 from .arcpublishing import ArcPublishingIE
 from .medialaan import MedialaanIE
+from .simplecast import SimplecastIE
 
 
 class GenericIE(InfoExtractor):
@@ -2238,6 +2239,15 @@ class GenericIE(InfoExtractor):
                 'duration': 159,
             },
         },
+        {
+            # Simplecast player embed
+            'url': 'https://www.bio.org/podcast',
+            'info_dict': {
+                'id': 'podcast',
+                'title': 'I AM BIO Podcast | BIO',
+            },
+            'playlist_mincount': 52,
+        },
     ]
 
     def report_following_redirect(self, new_url):
@@ -2792,6 +2802,12 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 matches, video_id, video_title, getter=unescapeHTML, ie='FunnyOrDie')
 
+        # Look for Simplecast embeds
+        simplecast_urls = SimplecastIE._extract_urls(webpage)
+        if simplecast_urls:
+            return self.playlist_from_matches(
+                simplecast_urls, video_id, video_title)
+
         # Look for BBC iPlayer embed
         matches = re.findall(r'setPlaylist\("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)', webpage)
         if matches:

youtube_dl/extractor/simplecast.py

@@ -0,0 +1,160 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_podcast_url,
+    int_or_none,
+    parse_iso8601,
+    strip_or_none,
+    try_get,
+    urlencode_postdata,
+)
+
+
+class SimplecastBaseIE(InfoExtractor):
+    _UUID_REGEX = r'[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12}'
+    _API_BASE = 'https://api.simplecast.com/'
+
+    def _call_api(self, path_tmpl, video_id):
+        return self._download_json(
+            self._API_BASE + path_tmpl % video_id, video_id)
+
+    def _call_search_api(self, resource, resource_id, resource_url):
+        return self._download_json(
+            'https://api.simplecast.com/%ss/search' % resource, resource_id,
+            data=urlencode_postdata({'url': resource_url}))
+
+    def _parse_episode(self, episode):
+        episode_id = episode['id']
+        title = episode['title'].strip()
+        audio_file = episode.get('audio_file') or {}
+        audio_file_url = audio_file.get('url') or episode.get('audio_file_url') or episode['enclosure_url']
+
+        season = episode.get('season') or {}
+        season_href = season.get('href')
+        season_id = None
+        if season_href:
+            season_id = self._search_regex(
+                r'https?://api.simplecast.com/seasons/(%s)' % self._UUID_REGEX,
+                season_href, 'season id', default=None)
+
+        webpage_url = episode.get('episode_url')
+        channel_url = None
+        if webpage_url:
+            channel_url = self._search_regex(
+                r'(https?://[^/]+\.simplecast\.com)',
+                webpage_url, 'channel url', default=None)
+
+        return {
+            'id': episode_id,
+            'display_id': episode.get('slug'),
+            'title': title,
+            'url': clean_podcast_url(audio_file_url),
+            'webpage_url': webpage_url,
+            'channel_url': channel_url,
+            'series': try_get(episode, lambda x: x['podcast']['title']),
+            'season_number': int_or_none(season.get('number')),
+            'season_id': season_id,
+            'thumbnail': episode.get('image_url'),
+            'episode_id': episode_id,
+            'episode_number': int_or_none(episode.get('number')),
+            'description': strip_or_none(episode.get('description')),
+            'timestamp': parse_iso8601(episode.get('published_at')),
+            'duration': int_or_none(episode.get('duration')),
+            'filesize': int_or_none(audio_file.get('size') or episode.get('audio_file_size')),
+        }
+
+
+class SimplecastIE(SimplecastBaseIE):
+    IE_NAME = 'simplecast'
+    _VALID_URL = r'https?://(?:api\.simplecast\.com/episodes|player\.simplecast\.com)/(?P<id>%s)' % SimplecastBaseIE._UUID_REGEX
+    _COMMON_TEST_INFO = {
+        'display_id': 'errant-signal-chris-franklin-new-wave-video-essays',
+        'id': 'b6dc49a2-9404-4853-9aa9-9cfc097be876',
+        'ext': 'mp3',
+        'title': 'Errant Signal - Chris Franklin & New Wave Video Essays',
+        'episode_number': 1,
+        'episode_id': 'b6dc49a2-9404-4853-9aa9-9cfc097be876',
+        'description': 'md5:34752789d3d2702e2d2c975fbd14f357',
+        'season_number': 1,
+        'season_id': 'e23df0da-bae4-4531-8bbf-71364a88dc13',
+        'series': 'The RE:BIND.io Podcast',
+        'duration': 5343,
+        'timestamp': 1580979475,
+        'upload_date': '20200206',
+        'webpage_url': r're:^https?://the-re-bind-io-podcast\.simplecast\.com/episodes/errant-signal-chris-franklin-new-wave-video-essays',
+        'channel_url': r're:^https?://the-re-bind-io-podcast\.simplecast\.com$',
+    }
+    _TESTS = [{
+        'url': 'https://api.simplecast.com/episodes/b6dc49a2-9404-4853-9aa9-9cfc097be876',
+        'md5': '8c93be7be54251bf29ee97464eabe61c',
+        'info_dict': _COMMON_TEST_INFO,
+    }, {
+        'url': 'https://player.simplecast.com/b6dc49a2-9404-4853-9aa9-9cfc097be876',
+        'only_matching': True,
+    }]
+
+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(
+            r'''(?x)<iframe[^>]+src=["\']
+                (
+                    https?://(?:embed\.simplecast\.com/[0-9a-f]{8}|
+                    player\.simplecast\.com/%s
+                ))''' % SimplecastBaseIE._UUID_REGEX, webpage)
+
+    def _real_extract(self, url):
+        episode_id = self._match_id(url)
+        episode = self._call_api('episodes/%s', episode_id)
+        return self._parse_episode(episode)
+
+
+class SimplecastEpisodeIE(SimplecastBaseIE):
+    IE_NAME = 'simplecast:episode'
+    _VALID_URL = r'https?://(?!api\.)[^/]+\.simplecast\.com/episodes/(?P<id>[^/?&#]+)'
+    _TEST = {
+        'url': 'https://the-re-bind-io-podcast.simplecast.com/episodes/errant-signal-chris-franklin-new-wave-video-essays',
+        'md5': '8c93be7be54251bf29ee97464eabe61c',
+        'info_dict': SimplecastIE._COMMON_TEST_INFO,
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        episode = self._call_search_api(
+            'episode', mobj.group(1), mobj.group(0))
+        return self._parse_episode(episode)
+
+
+class SimplecastPodcastIE(SimplecastBaseIE):
+    IE_NAME = 'simplecast:podcast'
+    _VALID_URL = r'https?://(?!(?:api|cdn|embed|feeds|player)\.)(?P<id>[^/]+)\.simplecast\.com(?!/episodes/[^/?&#]+)'
+    _TESTS = [{
+        'url': 'https://the-re-bind-io-podcast.simplecast.com',
+        'playlist_mincount': 33,
+        'info_dict': {
+            'id': '07d28d26-7522-42eb-8c53-2bdcfc81c43c',
+            'title': 'The RE:BIND.io Podcast',
+        },
+    }, {
+        'url': 'https://the-re-bind-io-podcast.simplecast.com/episodes',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        subdomain = self._match_id(url)
+        site = self._call_search_api('site', subdomain, url)
+        podcast = site['podcast']
+        podcast_id = podcast['id']
+        podcast_title = podcast.get('title')
+
+        def entries():
+            episodes = self._call_api('podcasts/%s/episodes', podcast_id)
+            for episode in (episodes.get('collection') or []):
+                info = self._parse_episode(episode)
+                info['series'] = podcast_title
+                yield info
+
+        return self.playlist_result(entries(), podcast_id, podcast_title)