[XVideos] Refactor XVideosPlaylistIE and XVideosRelatedIE with a base class

youtube_dl/extractor/xvideos.py

@@ -274,112 +274,203 @@ class XVideosIE(InfoExtractor):
             'channel_url': uploader_url,
         } if '/channels/' in (uploader_url or '') else {})
 
-class XVideosPlaylistIE(InfoExtractor):
-    _VALID_URL = r'''(?x)
-                    https?://
-                        (?:[^/]+\.)?xvideos2?\.com/
-                          (?:c(?:/[sm]:[^/]+)*|
-                             profiles|
-                             favorite)/
-                            (?P<id>[^#?/]+)
-                  '''
-    _TESTS = []
 
+class XVideosPlaylistBaseIE(InfoExtractor):
     def _extract_videos_from_json_list(self, json_list, path='video'):
-        return (
+        return traverse_obj(json_list, (
-            'https://www.xvideos.com/%s%d' % (path, x.get('id'), )
+            Ellipsis, 'id', T(int_or_none),
-            for x in json_list if isinstance(x, dict))
+            T(lambda x: self.url_result('https://www.xvideos.com/%s%d' % (path, x)))))
 
     def _get_playlist_url(self, url, playlist_id):
         """URL of first playlist page"""
-        id_match = re.match(self._VALID_URL, url).groupdict()
-        video_sort = id_match.get('sort')
-        if video_sort:
-            url, _ = compat_urlparse.urldefrag(url)
-            if url.endswith('/'):
-                url = url[:-1]
-            url = '%s/%s' % (url, video_sort.replace('-', '/'))
         return url
 
+    def _get_playlist_id(self, playlist_id, **kwargs):
+        pnum = kwargs.get('pnum')
+        return join_nonempty(playlist_id, pnum, delim='/')
+
+    def _can_be_paginated(self, playlist_id):
+        return False
+
+    def _get_title(self, page, playlist_id, **kwargs):
+        """title of playlist"""
+        title = (
+            self._og_search_title(page, default=None)
+            or self._html_search_regex(
+                r'<title\b[^>]*>([^<]+?)(?:\s+-\s+XVIDEOS\.COM)?</title>',
+                page, 'title', default=None)
+            or 'XVideos playlist %s' % playlist_id)
+        pnum = kwargs.get('pnum')
+        pnum = ('p%s' % pnum) if pnum is not None else (
+            'all' if self._can_be_paginated(playlist_id) else None)
+        if pnum:
+            title = '%s (%s)' % (title, pnum)
+        return title
+
+    def _get_description(self, page, playlist_id):
+        return None
+
     def _get_next_page(self, url, num, page):
         '''URL of num'th continuation page of url'''
         if page.startswith('{'):
             url, sub = re.subn(r'(/)(\d{1,7})($|[#?/])', r'\g<1>%d\3' % (num, ), url)
             if sub == 0:
-                url += '/%d' % (num, )
+                url += '/%d' % num
             return url
-        next_page = self._search_regex(
+        return traverse_obj(
-            r'''(?s)(<a\s[^>]*?\bclass\s*=\s*(?P<q>'|").*?\bnext-page\b.*?(?P=q)[^>]*?>)''',
+            self._search_regex(
-            page, 'next page', default=None)
+                r'''(?s)(<a\s[^>]*?\bclass\s*=\s*(?P<q>'|")[^>]*?\bnext-page\b.*?(?P=q)[^>]*>)''',
-        if next_page:
+                page, 'next page', default=None),
-            next_page = extract_attributes(next_page)
+            (T(extract_attributes), 'href', T(lambda u: urljoin(url, u)))) or False
-            next_page = next_page.get('href')
-            if next_page:
-                return urljoin(url, next_page)
-        return False
 
     def _extract_videos(self, url, playlist_id, num, page):
-        """Get iterable videos plus stop flag"""
+        """Get iterable video entries plus stop flag"""
-        return ((
+        return (
-            'https://www.xvideos.com/video' + x.group('video_id')
+            traverse_obj(
-            for x in re.finditer(r'''<div\s[^>]*?id\s*=\s*(\'|")video_(?P<video_id>[0-9]+)\1''', page)),
+                re.finditer(
+                    r'''<div\s[^>]*?id\s*=\s*(\'|")video_(?P<video_id>[0-9]+)\1''', page),
+                (Ellipsis, 'video_id', T(lambda x: self.url_result('xvideos:' + x, ie=XVideosIE.ie_key())))),
             None)
 
     def _real_extract(self, url):
-        id_match = re.match(self._VALID_URL, url).groupdict()
+        mobj = self._match_valid_url(url)
-
+        playlist_id = mobj.group('id')
-        playlist_id = id_match['id']
+        pnum = mobj.groupdict().get('pnum')
-        if url.endswith(playlist_id):
+        webpage = self._download_webpage(url, playlist_id, fatal=False) or ''
-            url += '/0'
         next_page = self._get_playlist_url(url, playlist_id)
-        matches = []
+        playlist_id = self._get_playlist_id(playlist_id, pnum=pnum, url=url)
+
+        def entries(url, webpage):
+            next_page = url
+            ids = set()
             for count in itertools.count(0):
+                if not webpage:
                     webpage = self._download_webpage(
                         next_page,
                         '%s (+%d)' % (playlist_id, count) if count > 0 else playlist_id)
 
                 vids, stop = self._extract_videos(next_page, playlist_id, count, webpage)
 
-            if vids:
+                for from_ in vids:
-                matches.append(vids)
+                    h_id = hash(from_['url'])
+                    if h_id not in ids:
+                        yield from_
+                        ids.add(h_id)
 
-            if stop:
+                if stop or pnum is not None:
                     break
                 next_page = self._get_next_page(next_page, count + 1, webpage)
                 if not next_page:
                     break
+                webpage = None
 
-        return self.playlist_from_matches(
+        playlist_title = self._get_title(webpage, playlist_id, pnum=pnum)
-            itertools.chain.from_iterable(matches), playlist_id)
+        # text may have a final + as an expand widget
+        description = remove_end(self._get_description(webpage, playlist_id), '+')
+
+        return merge_dicts(self.playlist_result(
+            LazyList(entries(next_page, webpage if next_page == url else None)),
+            playlist_id, playlist_title), {
+                'description': description,
+        })
 
 
-class XVideosRelatedIE(XVideosPlaylistIE):
+class XVideosRelatedIE(XVideosPlaylistBaseIE):
+    IE_DESC = 'Related videos/playlists in the respective tabs of a video page'
     _VALID_URL = XVideosIE._VALID_URL + r'(?:/[^/]+)*?\#_related-(?P<related>videos|playlists)'
 
-    _TESTS = []
+    _TESTS = [{
+        'url': 'https://www.xvideos.com/video46474569/solo_girl#_related-videos',
+        'info_dict': {
+            'id': '46474569/related/videos',
+            'title': 'solo girl (related videos)',
+        },
+        'playlist_mincount': 40,
+    }, {
+        'url': 'https://www.xvideos.com/video46474569/solo_girl#_related-playlists',
+        'info_dict': {
+            'id': '46474569/related/playlists',
+            'title': 'solo girl (related playlists)',
+        },
+        'playlist_mincount': 20,
+    }]
+
+    def _get_playlist_id(self, playlist_id, **kwargs):
+        url = kwargs.get('url')
+        return '/related/'.join((
+            playlist_id,
+            self._match_valid_url(url).group('related')))
+
+    def _get_title(self, page, playlist_id, **kwargs):
+        return '%s (%s)' % (
+            super(XVideosRelatedIE, self)._get_title(page, playlist_id),
+            playlist_id.split('/', 1)[-1].replace('/', ' '))
 
     def _extract_videos(self, url, playlist_id, num, page):
-        id_match = re.match(self._VALID_URL, url).groupdict()
+        related = playlist_id.rsplit('/', 1)[-1]
-        related = id_match.get('related')
         if not related:
             return super(XVideosRelatedIE, self)._extract_videos(url, playlist_id, num, page)
 
         if related == 'videos':
             related_json = self._search_regex(
-                r'(?s)videos_related\s*=\s*(\[.*?])\s*;',
+                r'(?s)videos?_related\s*=\s*(\[.*?])\s*;',
                 page, 'related', default='[]')
             related_json = self._parse_json(related_json, playlist_id, fatal=False) or []
             return (self._extract_videos_from_json_list(related_json), True)
         # playlists
         related_json = self._download_json(
-            'https://www.xvideos.com/video-playlists/' + playlist_id, playlist_id, fatal=False)
+            'https://www.xvideos.com/video-playlists/' + playlist_id.split('/', 1)[0], playlist_id, fatal=False)
         return (
             self._extract_videos_from_json_list(
-                try_get(related_json, lambda x: x['playlists'], list) or [],
+                traverse_obj(related_json, ('playlists', Ellipsis)),
                 path='favorite/'),
             True)
 
 
+class XVideosPlaylistIE(XVideosPlaylistBaseIE):
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:[^/]+\.)?xvideos\d*\.com/
+                          (?P<id>gay|shemale|best(?:/\d{4}-\d{2})|(?P<fav>favorite)/\d+)(?:(?(fav)[\w-]+/|)/(?P<pnum>\d+))?
+                  '''
+    _TESTS = [{
+        'url': 'https://www.xvideos.com/best/2023-08/4',
+        'info_dict': {
+            'id': 'best/2023-08/4',
+            'title': 'Playlist best (2023-08, p4)',
+        },
+        'playlist_count': 27,
+    }, {
+        'url': 'https://www.xvideos.com/favorite/84800989/mental_health',
+        'info_dict': {
+            'id': 'favorite/84800989',
+            'title': 'Playlist favorite/84800989',
+        },
+        'playlist_count': 5,
+    }]
+
+    def _can_be_paginated(self, playlist_id):
+        return True
+
+    def _get_playlist_url(self, url, playlist_id):
+        if url.endswith(playlist_id):
+            url += '/0'
+        return super(XVideosPlaylistIE, self)._get_playlist_url(url, playlist_id)
+
+    def _get_title(self, page, playlist_id, **kwargs):
+        pl_id = playlist_id.split('/')
+        if pl_id[0] == 'favorite':
+            pl_id[0] = '/'.join(pl_id[:2])
+            del pl_id[1]
+        pnum = int_or_none(pl_id[-1])
+        if pnum is not None:
+            pl_id[-1] = ' p%d' % pnum
+        title = 'Playlist ' + pl_id[0]
+        if len(pl_id) > 1:
+            title = '%s (%s)' % (title, ','.join(pl_id[1:]))
+        return title
+
+
 class XVideosChannelIE(XVideosPlaylistIE):
     _VALID_URL = r'''(?x)
                     https?://