[facebook] add support for Relay post pages(closes #26935)

This reverts commit dc65041c22.

youtube_dl/extractor/facebook.py

@@ -26,6 +26,7 @@ from ..utils import (
     sanitized_Request,
     try_get,
     urlencode_postdata,
+    urljoin,
 )
 
 
@@ -41,7 +42,8 @@ class FacebookIE(InfoExtractor):
                                 photo\.php|
                                 video\.php|
                                 video/embed|
-                                story\.php
+                                story\.php|
+                                watch/?
                             )\?(?:.*?)(?:v|video_id|story_fbid)=|
                             [^/]+/videos/(?:[^/]+/)?|
                             [^/]+/posts/|
@@ -72,6 +74,7 @@ class FacebookIE(InfoExtractor):
         },
         'skip': 'Requires logging in',
     }, {
+        # data.video
         'url': 'https://www.facebook.com/video.php?v=274175099429670',
         'info_dict': {
             'id': '274175099429670',
@@ -133,6 +136,7 @@ class FacebookIE(InfoExtractor):
         },
     }, {
         # have 1080P, but only up to 720p in swf params
+        # data.video.story.attachments[].media
         'url': 'https://www.facebook.com/cnn/videos/10155529876156509/',
         'md5': '9571fae53d4165bbbadb17a94651dcdc',
         'info_dict': {
@@ -147,6 +151,7 @@ class FacebookIE(InfoExtractor):
         },
     }, {
         # bigPipe.onPageletArrive ... onPageletArrive pagelet_group_mall
+        # data.node.comet_sections.content.story.attachments[].style_type_renderer.attachment.media
         'url': 'https://www.facebook.com/yaroslav.korpan/videos/1417995061575415/',
         'info_dict': {
             'id': '1417995061575415',
@@ -174,6 +179,7 @@ class FacebookIE(InfoExtractor):
             'skip_download': True,
         },
     }, {
+        # data.node.comet_sections.content.story.attachments[].style_type_renderer.attachment.media
         'url': 'https://www.facebook.com/groups/1024490957622648/permalink/1396382447100162/',
         'info_dict': {
             'id': '1396382447100162',
@@ -193,18 +199,23 @@ class FacebookIE(InfoExtractor):
         'url': 'https://www.facebook.com/amogood/videos/1618742068337349/?fref=nf',
         'only_matching': True,
     }, {
+        # data.mediaset.currMedia.edges
         'url': 'https://www.facebook.com/ChristyClarkForBC/videos/vb.22819070941/10153870694020942/?type=2&theater',
         'only_matching': True,
     }, {
+        # data.video.story.attachments[].media
         'url': 'facebook:544765982287235',
         'only_matching': True,
     }, {
+        # data.node.comet_sections.content.story.attachments[].style_type_renderer.attachment.media
         'url': 'https://www.facebook.com/groups/164828000315060/permalink/764967300301124/',
         'only_matching': True,
     }, {
+        # data.video.creation_story.attachments[].media
         'url': 'https://zh-hk.facebook.com/peoplespower/videos/1135894589806027/',
         'only_matching': True,
     }, {
+        # data.video
         'url': 'https://www.facebookcorewwwi.onion/video.php?v=274175099429670',
         'only_matching': True,
     }, {
@@ -212,6 +223,7 @@ class FacebookIE(InfoExtractor):
         'url': 'https://www.facebook.com/onlycleverentertainment/videos/1947995502095005/',
         'only_matching': True,
     }, {
+        # data.video
         'url': 'https://www.facebook.com/WatchESLOne/videos/359649331226507/',
         'info_dict': {
             'id': '359649331226507',
@@ -222,7 +234,39 @@ class FacebookIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # data.node.comet_sections.content.story.attachments[].style_type_renderer.attachment.all_subattachments.nodes[].media
+        'url': 'https://www.facebook.com/100033620354545/videos/106560053808006/',
+        'info_dict': {
+            'id': '106560053808006',
+        },
+        'playlist_count': 2,
+    }, {
+        # data.video.story.attachments[].media
+        'url': 'https://www.facebook.com/watch/?v=647537299265662',
+        'only_matching': True,
+    }, {
+        # data.node.comet_sections.content.story.attachments[].style_type_renderer.attachment.all_subattachments.nodes[].media
+        'url': 'https://www.facebook.com/PankajShahLondon/posts/10157667649866271',
+        'info_dict': {
+            'id': '10157667649866271',
+        },
+        'playlist_count': 3,
+    }, {
+        # data.nodes[].comet_sections.content.story.attachments[].style_type_renderer.attachment.media
+        'url': 'https://m.facebook.com/Alliance.Police.Department/posts/4048563708499330',
+        'info_dict': {
+            'id': '117576630041613',
+            'ext': 'mp4',
+            # TODO: title can be extracted from video page
+            'title': 'Facebook video #117576630041613',
+            'uploader_id': '189393014416438',
+            'upload_date': '20201123',
+            'timestamp': 1606162592,
+        },
+        'skip': 'Requires logging in',
     }]
+    _SUPPORTED_PAGLETS_REGEX = r'(?:pagelet_group_mall|permalink_video_pagelet|hyperfeed_story_id_[0-9a-f]+)'
 
     @staticmethod
     def _extract_urls(webpage):
@@ -305,18 +349,20 @@ class FacebookIE(InfoExtractor):
     def _real_initialize(self):
         self._login()
 
-    def _extract_from_url(self, url, video_id, fatal_if_no_video=True):
+    def _extract_from_url(self, url, video_id):
         webpage = self._download_webpage(
             url.replace('://m.facebook.com/', '://www.facebook.com/'), video_id)
 
         video_data = None
 
         def extract_video_data(instances):
+            video_data = []
             for item in instances:
                 if item[1][0] == 'VideoConfig':
                     video_item = item[2][0]
                     if video_item.get('video_id'):
-                        return video_item['videoData']
+                        video_data.append(video_item['videoData'])
+            return video_data
 
         server_js_data = self._parse_json(self._search_regex(
             r'handleServerJS\(({.+})(?:\);|,")', webpage,
@@ -330,18 +376,24 @@ class FacebookIE(InfoExtractor):
                 return extract_video_data(try_get(
                     js_data, lambda x: x['jsmods']['instances'], list) or [])
 
-        formats = []
+        def extract_dash_manifest(video, formats):
-
-        def extract_dash_manifest(video):
             dash_manifest = video.get('dash_manifest')
             if dash_manifest:
                 formats.extend(self._parse_mpd_formats(
                     compat_etree_fromstring(compat_urllib_parse_unquote_plus(dash_manifest))))
 
+        def process_formats(formats):
+            # Downloads with browser's User-Agent are rate limited. Working around
+            # with non-browser User-Agent.
+            for f in formats:
+                f.setdefault('http_headers', {})['User-Agent'] = 'facebookexternalhit/1.1'
+
+            self._sort_formats(formats)
+
         if not video_data:
             server_js_data = self._parse_json(self._search_regex([
-                r'bigPipe\.onPageletArrive\(({.+?})\)\s*;\s*}\s*\)\s*,\s*["\']onPageletArrive\s+(?:pagelet_group_mall|permalink_video_pagelet|hyperfeed_story_id_\d+)',
+                r'bigPipe\.onPageletArrive\(({.+?})\)\s*;\s*}\s*\)\s*,\s*["\']onPageletArrive\s+' + self._SUPPORTED_PAGLETS_REGEX,
-                r'bigPipe\.onPageletArrive\(({.*?id\s*:\s*"permalink_video_pagelet".*?})\);'
+                r'bigPipe\.onPageletArrive\(({.*?id\s*:\s*"%s".*?})\);' % self._SUPPORTED_PAGLETS_REGEX
             ], webpage, 'js data', default='{}'), video_id, js_to_json, False)
             video_data = extract_from_jsmods_instances(server_js_data)
 
@@ -351,7 +403,10 @@ class FacebookIE(InfoExtractor):
                 webpage, 'graphql data', default='{}'), video_id, fatal=False) or {}
             for require in (graphql_data.get('require') or []):
                 if require[0] == 'RelayPrefetchedStreamCache':
+                    entries = []
+
                     def parse_graphql_video(video):
+                        formats = []
                         q = qualities(['sd', 'hd'])
                         for (suffix, format_id) in [('', 'sd'), ('_quality_hd', 'hd')]:
                             playable_url = video.get('playable_url' + suffix)
@@ -362,8 +417,8 @@ class FacebookIE(InfoExtractor):
                                 'quality': q(format_id),
                                 'url': playable_url,
                             })
-                        extract_dash_manifest(video)
+                        extract_dash_manifest(video, formats)
-                        self._sort_formats(formats)
+                        process_formats(formats)
                         v_id = video.get('videoId') or video.get('id') or video_id
                         info = {
                             'id': v_id,
@@ -382,27 +437,46 @@ class FacebookIE(InfoExtractor):
                             })
                         else:
                             info['title'] = description or 'Facebook video #%s' % v_id
-                        return webpage, info
+                        entries.append(info)
 
-                    data = try_get(require, lambda x: x[3][1]['__bbox']['result']['data'], dict) or {}
+                    def parse_attachment(attachment, key='media'):
-
+                        media = attachment.get(key) or {}
-                    attachments = try_get(data, [
-                        lambda x: x['video']['story']['attachments'],
-                        lambda x: x['video']['creation_story']['attachments'],
-                        lambda x: x['node']['comet_sections']['content']['story']['attachments']
-                    ], list) or []
-                    for attachment in attachments:
-                        media = attachment.get('media') or try_get(attachment, lambda x: x['style_type_renderer']['attachment']['media'], dict) or {}
                         if media.get('__typename') == 'Video':
                             return parse_graphql_video(media)
 
+                    data = try_get(require, lambda x: x[3][1]['__bbox']['result']['data'], dict) or {}
+
+                    nodes = data.get('nodes') or []
+                    node = data.get('node') or {}
+                    if not nodes and node:
+                        nodes.append(node)
+                    for node in nodes:
+                        attachments = try_get(node, lambda x: x['comet_sections']['content']['story']['attachments'], list) or []
+                        for attachment in attachments:
+                            attachment = try_get(attachment, lambda x: x['style_type_renderer']['attachment'], dict)
+                            ns = try_get(attachment, lambda x: x['all_subattachments']['nodes'], list) or []
+                            for n in ns:
+                                parse_attachment(n)
+                            parse_attachment(attachment)
+
+                    edges = try_get(data, lambda x: x['mediaset']['currMedia']['edges'], list) or []
+                    for edge in edges:
+                        parse_attachment(edge, key='node')
+
                     video = data.get('video') or {}
                     if video:
-                        return parse_graphql_video(video)
+                        attachments = try_get(video, [
+                            lambda x: x['story']['attachments'],
+                            lambda x: x['creation_story']['attachments']
+                        ], list) or []
+                        for attachment in attachments:
+                            parse_attachment(attachment)
+                        if not entries:
+                            parse_graphql_video(video)
+
+                    return self.playlist_result(entries, video_id)
 
         if not video_data:
-            if not fatal_if_no_video:
-                return webpage, False
             m_msg = re.search(r'class="[^"]*uiInterstitialContent[^"]*"><div>(.*?)</div>', webpage)
             if m_msg is not None:
                 raise ExtractorError(
@@ -440,6 +514,18 @@ class FacebookIE(InfoExtractor):
         if not video_data:
             raise ExtractorError('Cannot parse data')
 
+        if len(video_data) > 1:
+            entries = []
+            for v in video_data:
+                video_url = v[0].get('video_url')
+                if not video_url:
+                    continue
+                entries.append(self.url_result(urljoin(
+                    url, video_url), self.ie_key(), v[0].get('video_id')))
+            return self.playlist_result(entries, video_id)
+        video_data = video_data[0]
+
+        formats = []
         subtitles = {}
         for f in video_data:
             format_id = f['stream_type']
@@ -459,19 +545,14 @@ class FacebookIE(InfoExtractor):
                             'url': src,
                             'preference': preference,
                         })
-            extract_dash_manifest(f[0])
+            extract_dash_manifest(f[0], formats)
             subtitles_src = f[0].get('subtitles_src')
             if subtitles_src:
                 subtitles.setdefault('en', []).append({'url': subtitles_src})
         if not formats:
             raise ExtractorError('Cannot find video formats')
 
-        # Downloads with browser's User-Agent are rate limited. Working around
+        process_formats(formats)
-        # with non-browser User-Agent.
-        for f in formats:
-            f.setdefault('http_headers', {})['User-Agent'] = 'facebookexternalhit/1.1'
-
-        self._sort_formats(formats)
 
         video_title = self._html_search_regex(
             r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>([^<]*)</h2>', webpage,
@@ -511,35 +592,13 @@ class FacebookIE(InfoExtractor):
             'subtitles': subtitles,
         }
 
-        return webpage, info_dict
+        return info_dict
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         real_url = self._VIDEO_PAGE_TEMPLATE % video_id if url.startswith('facebook:') else url
-        webpage, info_dict = self._extract_from_url(real_url, video_id, fatal_if_no_video=False)
+        return self._extract_from_url(real_url, video_id)
-
-        if info_dict:
-            return info_dict
-
-        if '/posts/' in url:
-            video_id_json = self._search_regex(
-                r'(["\'])video_ids\1\s*:\s*(?P<ids>\[.+?\])', webpage, 'video ids', group='ids',
-                default='')
-            if video_id_json:
-                entries = [
-                    self.url_result('facebook:%s' % vid, FacebookIE.ie_key())
-                    for vid in self._parse_json(video_id_json, video_id)]
-                return self.playlist_result(entries, video_id)
-
-            # Single Video?
-            video_id = self._search_regex(r'video_id:\s*"([0-9]+)"', webpage, 'single video id')
-            return self.url_result('facebook:%s' % video_id, FacebookIE.ie_key())
-        else:
-            _, info_dict = self._extract_from_url(
-                self._VIDEO_PAGE_TEMPLATE % video_id,
-                video_id, fatal_if_no_video=True)
-            return info_dict
 
 
 class FacebookPluginsVideoIE(InfoExtractor):