[aparat] Fix extraction

closes #22285
closes #22611
closes #23348
closes #24354
closes #24591
closes #24904
closes #25418
closes #26070
closes #26350
closes #26738
closes #27563

youtube_dl/extractor/aparat.py

@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    get_element_by_id,
     int_or_none,
     merge_dicts,
     mimetype2ext,
@@ -39,23 +40,15 @@ class AparatIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id, fatal=False)
 
         if not webpage:
-            # Note: There is an easier-to-parse configuration at
-            # http://www.aparat.com/video/video/config/videohash/%video_id
-            # but the URL in there does not work
             webpage = self._download_webpage(
                 'http://www.aparat.com/video/video/embed/vt/frame/showvideo/yes/videohash/' + video_id,
                 video_id)
 
-        options = self._parse_json(
+        options = self._parse_json(self._search_regex(
-            self._search_regex(
+            r'options\s*=\s*({.+?})\s*;', webpage, 'options'), video_id)
-                r'options\s*=\s*JSON\.parse\(\s*(["\'])(?P<value>(?:(?!\1).)+)\1\s*\)',
-                webpage, 'options', group='value'),
-            video_id)
-
-        player = options['plugins']['sabaPlayerPlugin']
 
         formats = []
-        for sources in player['multiSRC']:
+        for sources in (options.get('multiSRC') or []):
             for item in sources:
                 if not isinstance(item, dict):
                     continue
@@ -85,11 +78,12 @@ class AparatIE(InfoExtractor):
         info = self._search_json_ld(webpage, video_id, default={})
 
         if not info.get('title'):
-            info['title'] = player['title']
+            info['title'] = get_element_by_id('videoTitle', webpage) or \
+                self._html_search_meta(['og:title', 'twitter:title', 'DC.Title', 'title'], webpage, fatal=True)
 
         return merge_dicts(info, {
             'id': video_id,
             'thumbnail': url_or_none(options.get('poster')),
-            'duration': int_or_none(player.get('duration')),
+            'duration': int_or_none(options.get('duration')),
             'formats': formats,
         })

youtube_dl/extractor/brightcove.py

@@ -534,14 +534,6 @@ class BrightcoveNewIE(AdobePassIE):
                         'format_id': build_format_id('rtmp'),
                     })
                 formats.append(f)
-        if not formats:
-            # for sonyliv.com DRM protected videos
-            s3_source_url = json_data.get('custom_fields', {}).get('s3sourceurl')
-            if s3_source_url:
-                formats.append({
-                    'url': s3_source_url,
-                    'format_id': 'source',
-                })
 
         errors = json_data.get('errors')
         if not formats and errors:

youtube_dl/extractor/nhk.py

@@ -90,7 +90,7 @@ class NhkVodIE(NhkBaseIE):
     _TESTS = [{
         # video clip
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999011/',
-        'md5': '256a1be14f48d960a7e61e2532d95ec3',
+        'md5': '7a90abcfe610ec22a6bfe15bd46b30ca',
         'info_dict': {
             'id': 'a95j5iza',
             'ext': 'mp4',

youtube_dl/extractor/piksel.py

@@ -6,16 +6,33 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
-    ExtractorError,
     dict_get,
+    ExtractorError,
     int_or_none,
-    unescapeHTML,
     parse_iso8601,
+    try_get,
+    unescapeHTML,
 )
 
 
 class PikselIE(InfoExtractor):
-    _VALID_URL = r'https?://player\.piksel\.com/v/(?:refid/[^/]+/prefid/)?(?P<id>[a-z0-9_]+)'
+    _VALID_URL = r'''(?x)https?://
+        (?:
+            (?:
+                player\.
+                    (?:
+                        olympusattelecom|
+                        vibebyvista
+                    )|
+                (?:api|player)\.multicastmedia|
+                (?:api-ovp|player)\.piksel
+            )\.com|
+            (?:
+                mz-edge\.stream\.co|
+                movie-s\.nhk\.or
+            )\.jp|
+            vidego\.baltimorecity\.gov
+        )/v/(?:refid/(?P<refid>[^/]+)/prefid/)?(?P<id>[\w-]+)'''
     _TESTS = [
         {
             'url': 'http://player.piksel.com/v/ums2867l',
@@ -56,46 +73,41 @@ class PikselIE(InfoExtractor):
         if mobj:
             return mobj.group('url')
 
+    def _call_api(self, app_token, resource, display_id, query, fatal=True):
+        response = (self._download_json(
+            'http://player.piksel.com/ws/ws_%s/api/%s/mode/json/apiv/5' % (resource, app_token),
+            display_id, query=query, fatal=fatal) or {}).get('response')
+        failure = try_get(response, lambda x: x['failure']['reason'])
+        if failure:
+            if fatal:
+                raise ExtractorError(failure, expected=True)
+            self.report_warning(failure)
+        return response
+
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        ref_id, display_id = re.match(self._VALID_URL, url).groups()
         webpage = self._download_webpage(url, display_id)
-        video_id = self._search_regex(
-            r'data-de-program-uuid=[\'"]([a-z0-9]+)',
-            webpage, 'program uuid', default=display_id)
         app_token = self._search_regex([
             r'clientAPI\s*:\s*"([^"]+)"',
             r'data-de-api-key\s*=\s*"([^"]+)"'
         ], webpage, 'app token')
-        response = self._download_json(
+        query = {'refid': ref_id, 'prefid': display_id} if ref_id else {'v': display_id}
-            'http://player.piksel.com/ws/ws_program/api/%s/mode/json/apiv/5' % app_token,
+        program = self._call_api(
-            video_id, query={
+            app_token, 'program', display_id, query)['WsProgramResponse']['program']
-                'v': video_id
+        video_id = program['uuid']
-            })['response']
+        video_data = program['asset']
-        failure = response.get('failure')
-        if failure:
-            raise ExtractorError(response['failure']['reason'], expected=True)
-        video_data = response['WsProgramResponse']['program']['asset']
         title = video_data['title']
+        asset_type = dict_get(video_data, ['assetType', 'asset_type'])
 
         formats = []
 
-        m3u8_url = dict_get(video_data, [
+        def process_asset_file(asset_file):
-            'm3u8iPadURL',
+            if not asset_file:
-            'ipadM3u8Url',
+                return
-            'm3u8AndroidURL',
-            'm3u8iPhoneURL',
-            'iphoneM3u8Url'])
-        if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', 'm3u8_native',
-                m3u8_id='hls', fatal=False))
-
-        asset_type = dict_get(video_data, ['assetType', 'asset_type'])
-        for asset_file in video_data.get('assetFiles', []):
             # TODO: extract rtmp formats
             http_url = asset_file.get('http_url')
             if not http_url:
-                continue
+                return
             tbr = None
             vbr = int_or_none(asset_file.get('videoBitrate'), 1024)
             abr = int_or_none(asset_file.get('audioBitrate'), 1024)
@@ -118,6 +130,43 @@ class PikselIE(InfoExtractor):
                 'filesize': int_or_none(asset_file.get('filesize')),
                 'tbr': tbr,
             })
+
+        def process_asset_files(asset_files):
+            for asset_file in (asset_files or []):
+                process_asset_file(asset_file)
+
+        process_asset_files(video_data.get('assetFiles'))
+        process_asset_file(video_data.get('referenceFile'))
+        if not formats:
+            asset_id = video_data.get('assetid') or program.get('assetid')
+            if asset_id:
+                process_asset_files(try_get(self._call_api(
+                    app_token, 'asset_file', display_id, {
+                        'assetid': asset_id,
+                    }, False), lambda x: x['WsAssetFileResponse']['AssetFiles']))
+
+        m3u8_url = dict_get(video_data, [
+            'm3u8iPadURL',
+            'ipadM3u8Url',
+            'm3u8AndroidURL',
+            'm3u8iPhoneURL',
+            'iphoneM3u8Url'])
+        if m3u8_url:
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native',
+                m3u8_id='hls', fatal=False))
+
+        smil_url = dict_get(video_data, ['httpSmil', 'hdSmil', 'rtmpSmil'])
+        if smil_url:
+            transform_source = None
+            if ref_id == 'nhkworld':
+                # TODO: figure out if this is something to be fixed in urljoin,
+                # _parse_smil_formats or keep it here
+                transform_source = lambda x: x.replace('src="/', 'src="').replace('/media"', '/media/"')
+            formats.extend(self._extract_smil_formats(
+                re.sub(r'/od/[^/]+/', '/od/http/', smil_url), video_id,
+                transform_source=transform_source, fatal=False))
+
         self._sort_formats(formats)
 
         subtitles = {}