[aljazeera] fix extraction(closes #20911)(closes #27779)

youtube_dl/extractor/aljazeera.py

@@ -1,13 +1,16 @@
 from __future__ import unicode_literals
 
+import json
+import re
+
 from .common import InfoExtractor
 
 
 class AlJazeeraIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?aljazeera\.com/(?:programmes|video)/.*?/(?P<id>[^/]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?aljazeera\.com/(?P<type>program/[^/]+|(?:feature|video)s)/\d{4}/\d{1,2}/\d{1,2}/(?P<id>[^/?&#]+)'
 
     _TESTS = [{
-        'url': 'http://www.aljazeera.com/programmes/the-slum/2014/08/deliverance-201482883754237240.html',
+        'url': 'https://www.aljazeera.com/program/episode/2014/9/19/deliverance',
         'info_dict': {
             'id': '3792260579001',
             'ext': 'mp4',
@@ -20,14 +23,34 @@ class AlJazeeraIE(InfoExtractor):
         'add_ie': ['BrightcoveNew'],
         'skip': 'Not accessible from Travis CI server',
     }, {
-        'url': 'http://www.aljazeera.com/video/news/2017/05/sierra-leone-709-carat-diamond-auctioned-170511100111930.html',
+        'url': 'https://www.aljazeera.com/videos/2017/5/11/sierra-leone-709-carat-diamond-to-be-auctioned-off',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.aljazeera.com/features/2017/8/21/transforming-pakistans-buses-into-art',
         'only_matching': True,
     }]
-    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/665003303001/default_default/index.html?videoId=%s'
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
-        program_name = self._match_id(url)
-        webpage = self._download_webpage(url, program_name)
-        brightcove_id = self._search_regex(
-            r'RenderPagesVideo\(\'(.+?)\'', webpage, 'brightcove id')
-        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)
+        post_type, name = re.match(self._VALID_URL, url).groups()
+        post_type = {
+            'features': 'post',
+            'program': 'episode',
+            'videos': 'video',
+        }[post_type.split('/')[0]]
+        video = self._download_json(
+            'https://www.aljazeera.com/graphql', name, query={
+                'operationName': 'SingleArticleQuery',
+                'variables': json.dumps({
+                    'name': name,
+                    'postType': post_type,
+                }),
+            }, headers={
+                'wp-site': 'aje',
+            })['data']['article']['video']
+        video_id = video['id']
+        account_id = video.get('accountId') or '665003303001'
+        player_id = video.get('playerId') or 'BkeSH5BDb'
+        return self.url_result(
+            self.BRIGHTCOVE_URL_TEMPLATE % (account_id, player_id, video_id),
+            'BrightcoveNew', video_id)

youtube_dl/extractor/extractors.py

@@ -651,6 +651,11 @@ from .microsoftvirtualacademy import (
     MicrosoftVirtualAcademyIE,
     MicrosoftVirtualAcademyCourseIE,
 )
+from .minds import (
+    MindsIE,
+    MindsChannelIE,
+    MindsGroupIE,
+)
 from .ministrygrid import MinistryGridIE
 from .minoto import MinotoIE
 from .miomio import MioMioIE

youtube_dl/extractor/minds.py

@@ -0,0 +1,196 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    clean_html,
+    int_or_none,
+    str_or_none,
+    strip_or_none,
+)
+
+
+class MindsBaseIE(InfoExtractor):
+    _VALID_URL_BASE = r'https?://(?:www\.)?minds\.com/'
+
+    def _call_api(self, path, video_id, resource, query=None):
+        api_url = 'https://www.minds.com/api/' + path
+        token = self._get_cookies(api_url).get('XSRF-TOKEN')
+        return self._download_json(
+            api_url, video_id, 'Downloading %s JSON metadata' % resource, headers={
+                'Referer': 'https://www.minds.com/',
+                'X-XSRF-TOKEN': token.value if token else '',
+            }, query=query)
+
+
+class MindsIE(MindsBaseIE):
+    IE_NAME = 'minds'
+    _VALID_URL = MindsBaseIE._VALID_URL_BASE + r'(?:media|newsfeed|archive/view)/(?P<id>[0-9]+)'
+    _TESTS = [{
+        'url': 'https://www.minds.com/media/100000000000086822',
+        'md5': '215a658184a419764852239d4970b045',
+        'info_dict': {
+            'id': '100000000000086822',
+            'ext': 'mp4',
+            'title': 'Minds intro sequence',
+            'thumbnail': r're:https?://.+\.png',
+            'uploader_id': 'ottman',
+            'upload_date': '20130524',
+            'timestamp': 1369404826,
+            'uploader': 'Bill Ottman',
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'tags': ['animation'],
+            'comment_count': int,
+            'license': 'attribution-cc',
+        },
+    }, {
+        # entity.type == 'activity' and empty title
+        'url': 'https://www.minds.com/newsfeed/798025111988506624',
+        'md5': 'b2733a74af78d7fd3f541c4cbbaa5950',
+        'info_dict': {
+            'id': '798022190320226304',
+            'ext': 'mp4',
+            'title': '798022190320226304',
+            'uploader': 'ColinFlaherty',
+            'upload_date': '20180111',
+            'timestamp': 1515639316,
+            'uploader_id': 'ColinFlaherty',
+        },
+    }, {
+        'url': 'https://www.minds.com/archive/view/715172106794442752',
+        'only_matching': True,
+    }, {
+        # youtube perma_url
+        'url': 'https://www.minds.com/newsfeed/1197131838022602752',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        entity_id = self._match_id(url)
+        entity = self._call_api(
+            'v1/entities/entity/' + entity_id, entity_id, 'entity')['entity']
+        if entity.get('type') == 'activity':
+            if entity.get('custom_type') == 'video':
+                video_id = entity['entity_guid']
+            else:
+                return self.url_result(entity['perma_url'])
+        else:
+            assert(entity['subtype'] == 'video')
+            video_id = entity_id
+        # 1080p and webm formats available only on the sources array
+        video = self._call_api(
+            'v2/media/video/' + video_id, video_id, 'video')
+
+        formats = []
+        for source in (video.get('sources') or []):
+            src = source.get('src')
+            if not src:
+                continue
+            formats.append({
+                'format_id': source.get('label'),
+                'height': int_or_none(source.get('size')),
+                'url': src,
+            })
+        self._sort_formats(formats)
+
+        entity = video.get('entity') or entity
+        owner = entity.get('ownerObj') or {}
+        uploader_id = owner.get('username')
+
+        tags = entity.get('tags')
+        if tags and isinstance(tags, compat_str):
+            tags = [tags]
+
+        thumbnail = None
+        poster = video.get('poster') or entity.get('thumbnail_src')
+        if poster:
+            urlh = self._request_webpage(poster, video_id, fatal=False)
+            if urlh:
+                thumbnail = urlh.geturl()
+
+        return {
+            'id': video_id,
+            'title': entity.get('title') or video_id,
+            'formats': formats,
+            'description': clean_html(entity.get('description')) or None,
+            'license': str_or_none(entity.get('license')),
+            'timestamp': int_or_none(entity.get('time_created')),
+            'uploader': strip_or_none(owner.get('name')),
+            'uploader_id': uploader_id,
+            'uploader_url': 'https://www.minds.com/' + uploader_id if uploader_id else None,
+            'view_count': int_or_none(entity.get('play:count')),
+            'like_count': int_or_none(entity.get('thumbs:up:count')),
+            'dislike_count': int_or_none(entity.get('thumbs:down:count')),
+            'tags': tags,
+            'comment_count': int_or_none(entity.get('comments:count')),
+            'thumbnail': thumbnail,
+        }
+
+
+class MindsFeedBaseIE(MindsBaseIE):
+    _PAGE_SIZE = 150
+
+    def _entries(self, feed_id):
+        query = {'limit': self._PAGE_SIZE, 'sync': 1}
+        i = 1
+        while True:
+            data = self._call_api(
+                'v2/feeds/container/%s/videos' % feed_id,
+                feed_id, 'page %s' % i, query)
+            entities = data.get('entities') or []
+            for entity in entities:
+                guid = entity.get('guid')
+                if not guid:
+                    continue
+                yield self.url_result(
+                    'https://www.minds.com/newsfeed/' + guid,
+                    MindsIE.ie_key(), guid)
+            query['from_timestamp'] = data['load-next']
+            if not (query['from_timestamp'] and len(entities) == self._PAGE_SIZE):
+                break
+            i += 1
+
+    def _real_extract(self, url):
+        feed_id = self._match_id(url)
+        feed = self._call_api(
+            'v1/%s/%s' % (self._FEED_PATH, feed_id),
+            feed_id, self._FEED_TYPE)[self._FEED_TYPE]
+
+        return self.playlist_result(
+            self._entries(feed['guid']), feed_id,
+            strip_or_none(feed.get('name')),
+            feed.get('briefdescription'))
+
+
+class MindsChannelIE(MindsFeedBaseIE):
+    _FEED_TYPE = 'channel'
+    IE_NAME = 'minds:' + _FEED_TYPE
+    _VALID_URL = MindsBaseIE._VALID_URL_BASE + r'(?!(?:newsfeed|media|api|archive|groups)/)(?P<id>[^/?&#]+)'
+    _FEED_PATH = 'channel'
+    _TEST = {
+        'url': 'https://www.minds.com/ottman',
+        'info_dict': {
+            'id': 'ottman',
+            'title': 'Bill Ottman',
+            'description': 'Co-creator & CEO @minds',
+        },
+        'playlist_mincount': 54,
+    }
+
+
+class MindsGroupIE(MindsFeedBaseIE):
+    _FEED_TYPE = 'group'
+    IE_NAME = 'minds:' + _FEED_TYPE
+    _VALID_URL = MindsBaseIE._VALID_URL_BASE + r'groups/profile/(?P<id>[0-9]+)'
+    _FEED_PATH = 'groups/group'
+    _TEST = {
+        'url': 'https://www.minds.com/groups/profile/785582576369672204/feed/videos',
+        'info_dict': {
+            'id': '785582576369672204',
+            'title': 'Cooking Videos',
+        },
+        'playlist_mincount': 1,
+    }