[nhk:program] Add support for audio programs and program clips

youtube_dl/extractor/extractors.py

@@ -725,7 +725,10 @@ from .nexx import (
     NexxEmbedIE,
 )
 from .nfl import NFLIE
-from .nhk import NhkVodIE
+from .nhk import (
+    NhkVodIE,
+    NhkVodProgramIE,
+)
 from .nhl import NHLIE
 from .nick import (
     NickIE,

youtube_dl/extractor/nhk.py

@@ -3,10 +3,88 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..utils import urljoin
 
 
-class NhkVodIE(InfoExtractor):
+class NhkBaseIE(InfoExtractor):
-    _VALID_URL = r'https?://www3\.nhk\.or\.jp/nhkworld/(?P<lang>[a-z]{2})/ondemand/(?P<type>video|audio)/(?P<id>\d{7}|[^/]+?-\d{8}-\d+)'
+    _API_URL_TEMPLATE = 'https://api.nhk.or.jp/nhkworld/%sod%slist/v7a/%s/%s/%s/all%s.json'
+    _BASE_URL_REGEX = r'https?://www3\.nhk\.or\.jp/nhkworld/(?P<lang>[a-z]{2})/ondemand'
+    _TYPE_REGEX = r'/(?P<type>video|audio)/'
+
+    def _call_api(self, m_id, lang, is_video, is_episode, is_clip):
+        return self._download_json(
+            self._API_URL_TEMPLATE % (
+                'v' if is_video else 'r',
+                'clip' if is_clip else 'esd',
+                'episode' if is_episode else 'program',
+                m_id, lang, '/all' if is_video else ''),
+            m_id, query={'apikey': 'EJfK8jdS57GqlupFgAfAAwr573q01y6k'})['data']['episodes'] or []
+
+    def _extract_episode_info(self, url, episode=None):
+        fetch_episode = episode is None
+        lang, m_type, episode_id = re.match(NhkVodIE._VALID_URL, url).groups()
+        if episode_id.isdigit():
+            episode_id = episode_id[:4] + '-' + episode_id[4:]
+
+        is_video = m_type == 'video'
+        if fetch_episode:
+            episode = self._call_api(
+                episode_id, lang, is_video, True, episode_id[:4] == '9999')[0]
+        title = episode.get('sub_title_clean') or episode['sub_title']
+
+        def get_clean_field(key):
+            return episode.get(key + '_clean') or episode.get(key)
+
+        series = get_clean_field('title')
+
+        thumbnails = []
+        for s, w, h in [('', 640, 360), ('_l', 1280, 720)]:
+            img_path = episode.get('image' + s)
+            if not img_path:
+                continue
+            thumbnails.append({
+                'id': '%dp' % h,
+                'height': h,
+                'width': w,
+                'url': 'https://www3.nhk.or.jp' + img_path,
+            })
+
+        info = {
+            'id': episode_id + '-' + lang,
+            'title': '%s - %s' % (series, title) if series and title else title,
+            'description': get_clean_field('description'),
+            'thumbnails': thumbnails,
+            'series': series,
+            'episode': title,
+        }
+        if is_video:
+            vod_id = episode['vod_id']
+            info.update({
+                '_type': 'url_transparent',
+                'ie_key': 'Piksel',
+                'url': 'https://player.piksel.com/v/refid/nhkworld/prefid/' + vod_id,
+                'id': vod_id,
+            })
+        else:
+            if fetch_episode:
+                audio_path = episode['audio']['audio']
+                info['formats'] = self._extract_m3u8_formats(
+                    'https://nhkworld-vh.akamaihd.net/i%s/master.m3u8' % audio_path,
+                    episode_id, 'm4a', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False)
+                for f in info['formats']:
+                    f['language'] = lang
+            else:
+                info.update({
+                    '_type': 'url_transparent',
+                    'ie_key': NhkVodIE.ie_key(),
+                    'url': url,
+                })
+        return info
+
+
+class NhkVodIE(NhkBaseIE):
+    _VALID_URL = r'%s%s(?P<id>\d{7}|[^/]+?-\d{8}-[0-9a-z]+)' % (NhkBaseIE._BASE_URL_REGEX, NhkBaseIE._TYPE_REGEX)
     # Content available only for a limited period of time. Visit
     # https://www3.nhk.or.jp/nhkworld/en/ondemand/ for working samples.
     _TESTS = [{
@@ -47,60 +125,54 @@ class NhkVodIE(InfoExtractor):
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/audio/j_art-20150903-1/',
         'only_matching': True,
     }]
-    _API_URL_TEMPLATE = 'https://api.nhk.or.jp/nhkworld/%sod%slist/v7a/episode/%s/%s/all%s.json'
 
     def _real_extract(self, url):
-        lang, m_type, episode_id = re.match(self._VALID_URL, url).groups()
+        return self._extract_episode_info(url)
-        if episode_id.isdigit():
-            episode_id = episode_id[:4] + '-' + episode_id[4:]
 
-        is_video = m_type == 'video'
-        episode = self._download_json(
-            self._API_URL_TEMPLATE % (
-                'v' if is_video else 'r',
-                'clip' if episode_id[:4] == '9999' else 'esd',
-                episode_id, lang, '/all' if is_video else ''),
-            episode_id, query={'apikey': 'EJfK8jdS57GqlupFgAfAAwr573q01y6k'})['data']['episodes'][0]
-        title = episode.get('sub_title_clean') or episode['sub_title']
 
-        def get_clean_field(key):
+class NhkVodProgramIE(NhkBaseIE):
-            return episode.get(key + '_clean') or episode.get(key)
+    _VALID_URL = r'%s/program%s(?P<id>[0-9a-z]+)(?:.+?\btype=(?P<episode_type>clip|(?:radio|tv)Episode))?' % (NhkBaseIE._BASE_URL_REGEX, NhkBaseIE._TYPE_REGEX)
+    _TESTS = [{
+        # video program episodes
+        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/program/video/japanrailway',
+        'info_dict': {
+            'id': 'japanrailway',
+            'title': 'Japan Railway Journal',
+        },
+        'playlist_mincount': 1,
+    }, {
+        # video program clips
+        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/program/video/japanrailway/?type=clip',
+        'info_dict': {
+            'id': 'japanrailway',
+            'title': 'Japan Railway Journal',
+        },
+        'playlist_mincount': 5,
+    }, {
+        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/program/video/10yearshayaomiyazaki/',
+        'only_matching': True,
+    }, {
+        # audio program
+        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/program/audio/listener/',
+        'only_matching': True,
+    }]
 
-        series = get_clean_field('title')
+    def _real_extract(self, url):
+        lang, m_type, program_id, episode_type = re.match(self._VALID_URL, url).groups()
 
-        thumbnails = []
+        episodes = self._call_api(
-        for s, w, h in [('', 640, 360), ('_l', 1280, 720)]:
+            program_id, lang, m_type == 'video', False, episode_type == 'clip')
-            img_path = episode.get('image' + s)
+
-            if not img_path:
+        entries = []
+        for episode in episodes:
+            episode_path = episode.get('url')
+            if not episode_path:
                 continue
-            thumbnails.append({
+            entries.append(self._extract_episode_info(
-                'id': '%dp' % h,
+                urljoin(url, episode_path), episode))
-                'height': h,
-                'width': w,
-                'url': 'https://www3.nhk.or.jp' + img_path,
-            })
 
-        info = {
+        program_title = None
-            'id': episode_id + '-' + lang,
+        if entries:
-            'title': '%s - %s' % (series, title) if series and title else title,
+            program_title = entries[0].get('series')
-            'description': get_clean_field('description'),
+
-            'thumbnails': thumbnails,
+        return self.playlist_result(entries, program_id, program_title)
-            'series': series,
-            'episode': title,
-        }
-        if is_video:
-            info.update({
-                '_type': 'url_transparent',
-                'ie_key': 'Piksel',
-                'url': 'https://player.piksel.com/v/refid/nhkworld/prefid/' + episode['vod_id'],
-            })
-        else:
-            audio = episode['audio']
-            audio_path = audio['audio']
-            info['formats'] = self._extract_m3u8_formats(
-                'https://nhkworld-vh.akamaihd.net/i%s/master.m3u8' % audio_path,
-                episode_id, 'm4a', entry_protocol='m3u8_native',
-                m3u8_id='hls', fatal=False)
-            for f in info['formats']:
-                f['language'] = lang
-        return info