Merge a2e33ecb485665ffb1233f05bcce972571a38cb1 into 3eb8d22ddb8982ca4fb56bb7a8d6517538bf14c6

added age_limit to test
changed info thumbnail check

youtube_dl/extractor/extractors.py

@@ -1327,7 +1327,10 @@ from .trovo import (
 )
 from .trunews import TruNewsIE
 from .trutv import TruTVIE
-from .tube8 import Tube8IE
+from .tube8 import (
+    Tube8IE,
+    Tube8ListIE,
+)
 from .tubitv import TubiTvIE
 from .tumblr import TumblrIE
 from .tunein import (

youtube_dl/extractor/tube8.py

@@ -1,86 +1,275 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
+import itertools
 import re
+from time import sleep
 
+from .common import InfoExtractor
 from ..utils import (
+    clean_html,
+    ExtractorError,
+    get_element_by_class,
+    get_element_by_id,
     int_or_none,
-    str_to_int,
+    merge_dicts,
+    parse_qs,
+    strip_or_none,
+    T,
+    traverse_obj,
+    url_or_none,
+    urljoin,
 )
-from .keezmovies import KeezMoviesIE
 
 
-class Tube8IE(KeezMoviesIE):
-    _VALID_URL = r'https?://(?:www\.)?tube8\.com/(?:[^/]+/)+(?P<display_id>[^/]+)/(?P<id>\d+)'
+class Tube8IE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?tube8\.com/porn-video/(?P<id>\d+)'
     _TESTS = [{
-        'url': 'http://www.tube8.com/teen/kasia-music-video/229795/',
-        'md5': '65e20c48e6abff62ed0c3965fff13a39',
+        'url': 'https://www.tube8.com/porn-video/189530841/',
+        'md5': '532408f59e89a32027d873af6289c85a',
         'info_dict': {
-            'id': '229795',
-            'display_id': 'kasia-music-video',
+            'id': '189530841',
             'ext': 'mp4',
-            'description': 'hot teen Kasia grinding',
-            'uploader': 'unknown',
-            'title': 'Kasia music video',
+            'title': 'Found dildo. She let it cum in her tight ass to keep the secret',
+            'description': 'Cowgirl:479',
+            'uploader': 'MaryKrylova',
+            'thumbnail': r're:https?://.*\.jpg$',
             'age_limit': 18,
-            'duration': 230,
-            'categories': ['Teen'],
-            'tags': ['dancing'],
-        },
-    }, {
-        'url': 'http://www.tube8.com/shemale/teen/blonde-cd-gets-kidnapped-by-two-blacks-and-punished-for-being-a-slutty-girl/19569151/',
-        'only_matching': True,
+        }
     }]
 
-    @staticmethod
-    def _extract_urls(webpage):
-        return re.findall(
-            r'<iframe[^>]+\bsrc=["\']((?:https?:)?//(?:www\.)?tube8\.com/embed/(?:[^/]+/)+\d+)',
-            webpage)
+    _EMBED_REGEX = r'<iframe [^>]*\bsrc=["\'](?P<url>(?:https?:)?//(?:www\.)?tube8\.com/embed/(?:[^/]+/)*\d+)'
+
+    @classmethod
+    def _extract_urls(cls, webpage):
+        return [m.group('url') for m in re.finditer(cls._EMBED_REGEX, webpage)]
 
     def _real_extract(self, url):
-        webpage, info = self._extract_info(url)
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
 
-        if not info['title']:
-            info['title'] = self._html_search_regex(
-                r'videoTitle\s*=\s*"([^"]+)', webpage, 'title')
+        title = self._html_search_regex(
+            r'<title(?:\s[^>]*)?>([^<]+)(?:\|\s*Tube8\s*)?</title>',
+            webpage, 'title', default=None)
+        if title == 'Removed Video':
+            raise ExtractorError('Video has been removed', expected=True)
 
-        description = self._html_search_regex(
-            r'(?s)Description:</dt>\s*<dd>(.+?)</dd>', webpage, 'description', fatal=False)
-        uploader = self._html_search_regex(
-            r'<span class="username">\s*(.+?)\s*<',
-            webpage, 'uploader', fatal=False)
+        age_verify_msg = self._search_regex(
+            r'(your elected officials in \w+(?:\s+\w+){,2} are requiring us to verify your age before allowing you access to our website)',
+            webpage, 'age verification message', default=None)
+        if age_verify_msg:
+            self.raise_geo_restricted('%s said: "%s"' % (self.IE_NAME, age_verify_msg))
 
-        like_count = int_or_none(self._search_regex(
-            r'rupVar\s*=\s*"(\d+)"', webpage, 'like count', fatal=False))
-        dislike_count = int_or_none(self._search_regex(
-            r'rdownVar\s*=\s*"(\d+)"', webpage, 'dislike count', fatal=False))
-        view_count = str_to_int(self._search_regex(
-            r'Views:\s*</dt>\s*<dd>([\d,\.]+)',
-            webpage, 'view count', fatal=False))
-        comment_count = str_to_int(self._search_regex(
-            r'<span id="allCommentsCount">(\d+)</span>',
-            webpage, 'comment count', fatal=False))
+        playervars = self._search_json(
+            r'\bplayervars\s*:', webpage, 'playervars', video_id)
 
-        category = self._search_regex(
-            r'Category:\s*</dt>\s*<dd>\s*<a[^>]+href=[^>]+>([^<]+)',
-            webpage, 'category', fatal=False)
-        categories = [category] if category else None
+        info = self._search_json_ld(webpage, video_id, expected_type='VideoObject', default={})
+        for k in ('url', 'description'):
+            info.pop(k, None)
 
-        tags_str = self._search_regex(
-            r'(?s)Tags:\s*</dt>\s*<dd>(.+?)</(?!a)',
-            webpage, 'tags', fatal=False)
-        tags = [t for t in re.findall(
-            r'<a[^>]+href=[^>]+>([^<]+)', tags_str)] if tags_str else None
+        info['title'] = (
+            traverse_obj(playervars, ('video_title', T(strip_or_none)))
+            or info.get('title') or title or self._og_search_title(webpage))
 
-        info.update({
-            'description': description,
-            'uploader': uploader,
-            'view_count': view_count,
-            'like_count': like_count,
-            'dislike_count': dislike_count,
-            'comment_count': comment_count,
-            'categories': categories,
-            'tags': tags,
-        })
+        # Borrowed from youporn extractor
+        def get_fmt(x):
+            v_url = url_or_none(x.get('videoUrl'))
+            if v_url:
+                x['videoUrl'] = v_url
+                return (x['format'], x)
 
-        return info
+        defs_by_format = dict(traverse_obj(playervars, (
+            'mediaDefinitions', lambda _, v: v.get('format'), T(get_fmt))))
+
+        def get_format_data(f):
+            if f not in defs_by_format:
+                return []
+            return self._download_json(
+                defs_by_format[f]['videoUrl'], video_id, '{0}-formats'.format(f))
+
+        formats = traverse_obj(get_format_data('mp4'), (
+            lambda _, v: v.get('videoUrl'), {
+                'url': ('videoUrl', T(url_or_none)),
+                'height': ('quality', T(int_or_none)),
+            }))
+
+        info['formats'] = []
+        for fmt in formats:
+            if not fmt.get('url'):
+                continue
+            height, fmt['tbr'] = map(int_or_none, self._search_regex(
+                r'(?i)(?P<height>\d{3,4})p_(?P<bitrate>\d+)k_\d+', fmt['url'],
+                'media details', group=('height', 'bitrate'),
+                default=(None, None)))
+            if fmt.get('height') is None:
+                fmt['height'] = height
+            else:
+                height = fmt['height']
+            fmt['format_id'] = '%dp' % height if height else 'mp4'
+            info['formats'].append(fmt)
+
+        self._sort_formats(info['formats'])
+
+        return merge_dicts({
+            'id': video_id,
+            'uploader': clean_html(get_element_by_class('submitByLink', webpage)),
+            'age_limit': self._rta_search(webpage),
+        }, info, traverse_obj(playervars, {
+            'thumbnail': ('image_url', T(url_or_none)),
+            'duration': ('video_duration', T(int_or_none)),
+            'description': 'actionTags',
+        }))
+
+
+# Currently only user channels
+class Tube8ListIE(InfoExtractor):
+    _PLAYLIST_TITLEBAR_RE = r'\s+[Vv]ideos\s*$'
+    _PAGE_RETRY_COUNT = 0  # ie, no retry
+    _PAGE_RETRY_DELAY = 2  # seconds
+
+    _VALID_URL = r'https?://(?:www\.)?tube8\.com/user-videos/(?P<id>\d+)/(?P<author>[^/#?]+)'
+    _TESTS = [{
+        'url': 'https://www.tube8.com/user-videos/195075441/MaryKrylova/',
+        'info_dict': {
+            'id': '195075441',
+        },
+        'playlist_mincount': 29,
+    }, {
+        'url': 'https://www.tube8.com/user-videos/195048331/FoxyElf/',
+        'info_dict': {
+            'id': '195048331',
+        },
+        'playlist_mincount': 80,
+    }]
+
+    # Borrowed from youporn extractor
+    @classmethod
+    def _get_title_from_slug(cls, title_slug):
+        return re.sub(r'[_-]', ' ', title_slug)
+
+    # Borrowed from youporn extractor
+    def _get_next_url(self, url, pl_id, html):
+        return urljoin(url, self._search_regex(
+            r'''<a\s[^>]*?(?<!-)\bhref\s*=\s*("|')(?P<url>(?:(?!\1)[^>])+)\1''',
+            get_element_by_id('next', html) or '', 'next page',
+            group='url', default=None))
+
+    # Borrowed from youporn extractor
+    def _entries(self, url, pl_id, html=None, page_num=None):
+
+        # separates page sections
+        PLAYLIST_SECTION_RE = (
+            r'''<div\s[^>]*(?<!-)\bclass\s*=\s*('|")(?:[\w$-]+\s+|\s)*?title-bar(?:\s+[\w$-]+|\s)*\1[^>]*>'''
+        )
+        # contains video link
+        VIDEO_URL_RE = r'''(?x)
+            <div\s[^>]*(?<!-)\bdata-video-id\s*=\s*('|")\d+\1[^>]*>\s*
+            (?:<div\b[\s\S]+?</div>\s*)*
+            <a\s[^>]*(?<!-)\bhref\s*=\s*('|")(?P<url>(?:(?!\2)[^>])+)\2
+        '''
+
+        def yield_pages(url, html=html, page_num=page_num):
+            fatal = not html
+            for pnum in itertools.count(start=page_num or 1):
+                if not html:
+                    html = self._download_webpage(
+                        url, pl_id, note='Downloading page %d' % pnum,
+                        fatal=fatal)
+                if not html:
+                    break
+                fatal = False
+                yield (url, html, pnum)
+                # explicit page: extract just that page
+                if page_num is not None:
+                    break
+                next_url = self._get_next_url(url, pl_id, html)
+                if not next_url or next_url == url:
+                    break
+                url, html = next_url, None
+
+        def retry_page(msg, tries_left, page_data):
+            if tries_left <= 0:
+                return
+            self.report_warning(msg, pl_id)
+            sleep(self._PAGE_RETRY_DELAY)
+            return next(
+                yield_pages(page_data[0], page_num=page_data[2]), None)
+
+        def yield_entries(html):
+            for frag in re.split(PLAYLIST_SECTION_RE, html):
+                if not frag:
+                    continue
+                t_text = get_element_by_class('title-text', frag or '')
+                if not (t_text and re.search(self._PLAYLIST_TITLEBAR_RE, t_text)):
+                    continue
+                for m in re.finditer(VIDEO_URL_RE, frag):
+                    video_url = urljoin(url, m.group('url'))
+                    if video_url:
+                        yield self.url_result(video_url)
+
+        last_first_url = None
+        for page_data in yield_pages(url, html=html, page_num=page_num):
+            # page_data: url, html, page_num
+            first_url = None
+            tries_left = self._PAGE_RETRY_COUNT + 1
+            while tries_left > 0:
+                tries_left -= 1
+                for from_ in yield_entries(page_data[1]):
+                    # may get the same page twice instead of empty page
+                    # or (site bug) intead of actual next page
+                    if not first_url:
+                        first_url = from_['url']
+                        if first_url == last_first_url:
+                            # sometimes (/porntags/) the site serves the previous page
+                            # instead but may provide the correct page after a delay
+                            page_data = retry_page(
+                                'Retrying duplicate page...', tries_left, page_data)
+                            if page_data:
+                                first_url = None
+                                break
+                            continue
+                    yield from_
+                else:
+                    if not first_url and 'no-result-paragarph1' in page_data[1]:
+                        page_data = retry_page(
+                            'Retrying empty page...', tries_left, page_data)
+                        if page_data:
+                            continue
+                    else:
+                        # success/failure
+                        break
+            # may get an infinite (?) sequence of empty pages
+            if not first_url:
+                break
+            last_first_url = first_url
+
+    # Borrowed from youporn extractor
+    def _real_extract(self, url, html=None):
+        # exceptionally, id may be None
+        m_dict = self._match_valid_url(url).groupdict()
+        pl_id, page_type, sort = (m_dict.get(k) for k in ('id', 'type', 'sort'))
+
+        qs = dict(traverse_obj(parse_qs(url), (
+            T(dict.items), lambda _, k_v: k_v[1],
+            T(lambda k_v: (k_v[0], k_v[0][-1])))))
+
+        base_id = pl_id or 'Tube8'
+        title = self._get_title_from_slug(base_id)
+        if page_type:
+            title = '%s %s' % (page_type.capitalize(), title)
+        base_id = [base_id.lower()]
+        if sort is None:
+            title += ' videos'
+        else:
+            title = '%s videos by %s' % (title, re.sub(r'[_-]', ' ', sort))
+            base_id.append(sort)
+        if qs:
+            ps = ['%s=%s' % item for item in sorted(qs.items())]
+            title += ' (%s)' % ','.join(ps)
+            base_id.extend(ps)
+        pl_id = '/'.join(base_id)
+
+        return self.playlist_result(
+            self._entries(url, pl_id, html=html,
+                          page_num=int_or_none(qs.get('page'))),
+            playlist_id=pl_id)

youtube_dl/jsinterp.py

@@ -686,6 +686,8 @@ class JSInterpreter(object):
             raise self.Exception('Cannot get index {idx!r:.100}'.format(**locals()), expr=repr(obj), cause=e)
 
     def _dump(self, obj, namespace):
+        if obj is JS_Undefined:
+            return 'undefined'
         try:
             return json.dumps(obj)
         except TypeError: