release 2018.04.25

[ci skip]

.github/ISSUE_TEMPLATE.md

@@ -6,8 +6,8 @@
 
 ---
 
-### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.04.16*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
-- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.04.16**
+### Make sure you are using the *latest* version: run `youtube-dl --version` and ensure your version is *2018.04.25*. If it's not, read [this FAQ entry](https://github.com/rg3/youtube-dl/blob/master/README.md#how-do-i-update-youtube-dl) and update. Issues with outdated version will be rejected.
+- [ ] I've **verified** and **I assure** that I'm running youtube-dl **2018.04.25**
 
 ### Before submitting an *issue* make sure you have:
 - [ ] At least skimmed through the [README](https://github.com/rg3/youtube-dl/blob/master/README.md), **most notably** the [FAQ](https://github.com/rg3/youtube-dl#faq) and [BUGS](https://github.com/rg3/youtube-dl#bugs) sections
@@ -36,7 +36,7 @@ Add the `-v` flag to **your command line** you run youtube-dl with (`youtube-dl
 [debug] User config: []
 [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
 [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
-[debug] youtube-dl version 2018.04.16
+[debug] youtube-dl version 2018.04.25
 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
 [debug] Proxy map: {}

AUTHORS

@@ -236,3 +236,6 @@ Lei Wang
 Petr Novák
 Leonardo Taccari
 Martin Weinelt
+Surya Oktafendri
+TingPing
+Alexandre Macabies

ChangeLog

@@ -1,3 +1,32 @@
+version 2018.04.25
+
+Core
+* [utils] Fix match_str for boolean meta fields
++ [Makefile] Add support for pandoc 2 and disable smart extension (#16251)
+* [YoutubeDL] Fix typo in media extension compatibility checker (#16215)
+
+Extractors
++ [openload] Recognize IPv6 stream URLs (#16136, #16137, #16205, #16246,
+  #16250)
++ [twitch] Extract is_live according to status (#16259)
+* [pornflip] Relax URL regular expression (#16258)
+- [etonline] Remove extractor (#16256)
+* [breakcom] Fix extraction (#16254)
++ [youtube] Add ability to authenticate with cookies
+* [youtube:feed] Implement lazy playlist extraction (#10184)
++ [svt] Add support for TV channel live streams (#15279, #15809)
+* [ccma] Fix video extraction (#15931)
+* [rentv] Fix extraction (#15227)
++ [nick] Add support for nickjr.nl (#16230)
+* [extremetube] Fix metadata extraction
++ [keezmovies] Add support for generic embeds (#16134, #16154)
+* [nexx] Extract new azure URLs (#16223)
+* [cbssports] Fix extraction (#16217)
+* [kaltura] Improve embeds detection (#16201)
+* [instagram:user] Fix extraction (#16119)
+* [cbs] Skip DRM asset types (#16104)
+
+
 version 2018.04.16
 
 Extractors

Makefile

@@ -14,6 +14,9 @@ PYTHON ?= /usr/bin/env python
 # set SYSCONFDIR to /etc if PREFIX=/usr or PREFIX=/usr/local
 SYSCONFDIR = $(shell if [ $(PREFIX) = /usr -o $(PREFIX) = /usr/local ]; then echo /etc; else echo $(PREFIX)/etc; fi)
 
+# set markdown input format to "markdown-smart" for pandoc version 2 and to "markdown" for pandoc prior to version 2
+MARKDOWN = $(shell if [ `pandoc -v | head -n1 | cut -d" " -f2 | head -c1` = "2" ]; then echo markdown-smart; else echo markdown; fi)
+
 install: youtube-dl youtube-dl.1 youtube-dl.bash-completion youtube-dl.zsh youtube-dl.fish
 	install -d $(DESTDIR)$(BINDIR)
 	install -m 755 youtube-dl $(DESTDIR)$(BINDIR)
@@ -82,11 +85,11 @@ supportedsites:
 	$(PYTHON) devscripts/make_supportedsites.py docs/supportedsites.md
 
 README.txt: README.md
-	pandoc -f markdown -t plain README.md -o README.txt
+	pandoc -f $(MARKDOWN) -t plain README.md -o README.txt
 
 youtube-dl.1: README.md
 	$(PYTHON) devscripts/prepare_manpage.py youtube-dl.1.temp.md
-	pandoc -s -f markdown -t man youtube-dl.1.temp.md -o youtube-dl.1
+	pandoc -s -f $(MARKDOWN) -t man youtube-dl.1.temp.md -o youtube-dl.1
 	rm -f youtube-dl.1.temp.md
 
 youtube-dl.bash-completion: youtube_dl/*.py youtube_dl/*/*.py devscripts/bash-completion.in

docs/supportedsites.md

@@ -257,7 +257,6 @@
  - **ESPN**
  - **ESPNArticle**
  - **EsriVideo**
- - **ETOnline**
  - **Europa**
  - **EveryonesMixtape**
  - **ExpoTV**

test/test_subtitles.py

@@ -232,7 +232,7 @@ class TestNPOSubtitles(BaseTestSubtitles):
 
 
 class TestMTVSubtitles(BaseTestSubtitles):
-    url = 'http://www.cc.com/video-clips/kllhuv/stand-up-greg-fitzsimmons--uncensored---too-good-of-a-mother'
+    url = 'http://www.cc.com/video-clips/p63lk0/adam-devine-s-house-party-chasing-white-swans'
     IE = ComedyCentralIE
 
     def getInfoDict(self):
@@ -243,7 +243,7 @@ class TestMTVSubtitles(BaseTestSubtitles):
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['en']))
-        self.assertEqual(md5(subtitles['en']), 'b9f6ca22a6acf597ec76f61749765e65')
+        self.assertEqual(md5(subtitles['en']), '78206b8d8a0cfa9da64dc026eea48961')
 
 
 class TestNRKSubtitles(BaseTestSubtitles):

test/test_utils.py

@@ -1072,6 +1072,18 @@ ffmpeg version 2.4.4 Copyright (c) 2000-2014 the FFmpeg ...'''), '2.4.4')
         self.assertFalse(match_str(
             'like_count > 100 & dislike_count <? 50 & description',
             {'like_count': 190, 'dislike_count': 10}))
+        self.assertTrue(match_str('is_live', {'is_live': True}))
+        self.assertFalse(match_str('is_live', {'is_live': False}))
+        self.assertFalse(match_str('is_live', {'is_live': None}))
+        self.assertFalse(match_str('is_live', {}))
+        self.assertFalse(match_str('!is_live', {'is_live': True}))
+        self.assertTrue(match_str('!is_live', {'is_live': False}))
+        self.assertTrue(match_str('!is_live', {'is_live': None}))
+        self.assertTrue(match_str('!is_live', {}))
+        self.assertTrue(match_str('title', {'title': 'abc'}))
+        self.assertTrue(match_str('title', {'title': ''}))
+        self.assertFalse(match_str('!title', {'title': 'abc'}))
+        self.assertFalse(match_str('!title', {'title': ''}))
 
     def test_parse_dfxp_time_expr(self):
         self.assertEqual(parse_dfxp_time_expr(None), None)

test/test_youtube_lists.py

@@ -61,7 +61,7 @@ class TestYoutubeLists(unittest.TestCase):
         dl = FakeYDL()
         dl.params['extract_flat'] = True
         ie = YoutubePlaylistIE(dl)
-        result = ie.extract('https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
+        result = ie.extract('https://www.youtube.com/playlist?list=PL-KKIb8rvtMSrAO9YFbeM6UQrAqoFTUWv')
         self.assertIsPlaylist(result)
         for entry in result['entries']:
             self.assertTrue(entry.get('title'))

youtube_dl/YoutubeDL.py

@@ -1853,7 +1853,7 @@ class YoutubeDL(object):
                     def compatible_formats(formats):
                         video, audio = formats
                         # Check extension
-                        video_ext, audio_ext = audio.get('ext'), video.get('ext')
+                        video_ext, audio_ext = video.get('ext'), audio.get('ext')
                         if video_ext and audio_ext:
                             COMPATIBLE_EXTS = (
                                 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'ismv', 'isma'),

youtube_dl/extractor/breakcom.py

@@ -3,15 +3,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from .youtube import YoutubeIE
 from ..compat import compat_str
-from ..utils import (
-    int_or_none,
-    parse_age_limit,
-)
+from ..utils import int_or_none
 
 
 class BreakIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?P<site>break|screenjunkies)\.com/video/(?P<display_id>[^/]+?)(?:-(?P<id>\d+))?(?:[/?#&]|$)'
+    _VALID_URL = r'https?://(?:www\.)?break\.com/video/(?P<display_id>[^/]+?)(?:-(?P<id>\d+))?(?:[/?#&]|$)'
     _TESTS = [{
         'url': 'http://www.break.com/video/when-girls-act-like-guys-2468056',
         'info_dict': {
@@ -19,125 +17,73 @@ class BreakIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'When Girls Act Like D-Bags',
             'age_limit': 13,
+        },
+    }, {
+        # youtube embed
+        'url': 'http://www.break.com/video/someone-forgot-boat-brakes-work',
+        'info_dict': {
+            'id': 'RrrDLdeL2HQ',
+            'ext': 'mp4',
+            'title': 'Whale Watching Boat Crashing Into San Diego Dock',
+            'description': 'md5:afc1b2772f0a8468be51dd80eb021069',
+            'upload_date': '20160331',
+            'uploader': 'Steve Holden',
+            'uploader_id': 'sdholden07',
+        },
+        'params': {
+            'skip_download': True,
         }
-    }, {
-        'url': 'http://www.screenjunkies.com/video/best-quentin-tarantino-movie-2841915',
-        'md5': '5c2b686bec3d43de42bde9ec047536b0',
-        'info_dict': {
-            'id': '2841915',
-            'display_id': 'best-quentin-tarantino-movie',
-            'ext': 'mp4',
-            'title': 'Best Quentin Tarantino Movie',
-            'thumbnail': r're:^https?://.*\.jpg',
-            'duration': 3671,
-            'age_limit': 13,
-            'tags': list,
-        },
-    }, {
-        'url': 'http://www.screenjunkies.com/video/honest-trailers-the-dark-knight',
-        'info_dict': {
-            'id': '2348808',
-            'display_id': 'honest-trailers-the-dark-knight',
-            'ext': 'mp4',
-            'title': 'Honest Trailers - The Dark Knight',
-            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
-            'age_limit': 10,
-            'tags': list,
-        },
-    }, {
-        # requires subscription but worked around
-        'url': 'http://www.screenjunkies.com/video/knocking-dead-ep-1-the-show-so-far-3003285',
-        'info_dict': {
-            'id': '3003285',
-            'display_id': 'knocking-dead-ep-1-the-show-so-far',
-            'ext': 'mp4',
-            'title': 'State of The Dead Recap: Knocking Dead Pilot',
-            'thumbnail': r're:^https?://.*\.jpg',
-            'duration': 3307,
-            'age_limit': 13,
-            'tags': list,
-        },
     }, {
         'url': 'http://www.break.com/video/ugc/baby-flex-2773063',
         'only_matching': True,
     }]
 
-    _DEFAULT_BITRATES = (48, 150, 320, 496, 864, 2240, 3264)
-
     def _real_extract(self, url):
-        site, display_id, video_id = re.match(self._VALID_URL, url).groups()
+        display_id, video_id = re.match(self._VALID_URL, url).groups()
 
-        if not video_id:
-            webpage = self._download_webpage(url, display_id)
-            video_id = self._search_regex(
-                (r'src=["\']/embed/(\d+)', r'data-video-content-id=["\'](\d+)'),
-                webpage, 'video id')
+        webpage = self._download_webpage(url, display_id)
 
-        webpage = self._download_webpage(
-            'http://www.%s.com/embed/%s' % (site, video_id),
-            display_id, 'Downloading video embed page')
-        embed_vars = self._parse_json(
+        youtube_url = YoutubeIE._extract_url(webpage)
+        if youtube_url:
+            return self.url_result(youtube_url, ie=YoutubeIE.ie_key())
+
+        content = self._parse_json(
             self._search_regex(
-                r'(?s)embedVars\s*=\s*({.+?})\s*</script>', webpage, 'embed vars'),
+                r'(?s)content["\']\s*:\s*(\[.+?\])\s*[,\n]', webpage,
+                'content'),
             display_id)
 
-        youtube_id = embed_vars.get('youtubeId')
-        if youtube_id:
-            return self.url_result(youtube_id, 'Youtube')
-
-        title = embed_vars['contentName']
-
         formats = []
-        bitrates = []
-        for f in embed_vars.get('media', []):
-            if not f.get('uri') or f.get('mediaPurpose') != 'play':
+        for video in content:
+            video_url = video.get('url')
+            if not video_url or not isinstance(video_url, compat_str):
                 continue
-            bitrate = int_or_none(f.get('bitRate'))
-            if bitrate:
-                bitrates.append(bitrate)
+            bitrate = int_or_none(self._search_regex(
+                r'(\d+)_kbps', video_url, 'tbr', default=None))
             formats.append({
-                'url': f['uri'],
+                'url': video_url,
                 'format_id': 'http-%d' % bitrate if bitrate else 'http',
-                'width': int_or_none(f.get('width')),
-                'height': int_or_none(f.get('height')),
                 'tbr': bitrate,
-                'format': 'mp4',
             })
-
-        if not bitrates:
-            # When subscriptionLevel > 0, i.e. plus subscription is required
-            # media list will be empty. However, hds and hls uris are still
-            # available. We can grab them assuming bitrates to be default.
-            bitrates = self._DEFAULT_BITRATES
-
-        auth_token = embed_vars.get('AuthToken')
-
-        def construct_manifest_url(base_url, ext):
-            pieces = [base_url]
-            pieces.extend([compat_str(b) for b in bitrates])
-            pieces.append('_kbps.mp4.%s?%s' % (ext, auth_token))
-            return ','.join(pieces)
-
-        if bitrates and auth_token:
-            hds_url = embed_vars.get('hdsUri')
-            if hds_url:
-                formats.extend(self._extract_f4m_formats(
-                    construct_manifest_url(hds_url, 'f4m'),
-                    display_id, f4m_id='hds', fatal=False))
-            hls_url = embed_vars.get('hlsUri')
-            if hls_url:
-                formats.extend(self._extract_m3u8_formats(
-                    construct_manifest_url(hls_url, 'm3u8'),
-                    display_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False))
         self._sort_formats(formats)
 
+        title = self._search_regex(
+            (r'title["\']\s*:\s*(["\'])(?P<value>(?:(?!\1).)+)\1',
+             r'<h1[^>]*>(?P<value>[^<]+)'), webpage, 'title', group='value')
+
+        def get(key, name):
+            return int_or_none(self._search_regex(
+                r'%s["\']\s*:\s*["\'](\d+)' % key, webpage, name,
+                default=None))
+
+        age_limit = get('ratings', 'age limit')
+        video_id = video_id or get('pid', 'video id') or display_id
+
         return {
             'id': video_id,
             'display_id': display_id,
             'title': title,
-            'thumbnail': embed_vars.get('thumbUri'),
-            'duration': int_or_none(embed_vars.get('videoLengthInSeconds')) or None,
-            'age_limit': parse_age_limit(embed_vars.get('audienceRating')),
-            'tags': embed_vars.get('tags', '').split(','),
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'age_limit': age_limit,
             'formats': formats,
         }

youtube_dl/extractor/cbs.py

@@ -65,7 +65,7 @@ class CBSIE(CBSBaseIE):
         last_e = None
         for item in items_data.findall('.//item'):
             asset_type = xpath_text(item, 'assetType')
-            if not asset_type or asset_type in asset_types:
+            if not asset_type or asset_type in asset_types or asset_type in ('HLS_FPS', 'DASH_CENC'):
                 continue
             asset_types.append(asset_type)
             query = {

youtube_dl/extractor/cbssports.py

@@ -4,28 +4,35 @@ from .cbs import CBSBaseIE
 
 
 class CBSSportsIE(CBSBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?cbssports\.com/video/player/[^/]+/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?cbssports\.com/[^/]+/(?:video|news)/(?P<id>[^/?#&]+)'
 
     _TESTS = [{
-        'url': 'http://www.cbssports.com/video/player/videos/708337219968/0/ben-simmons-the-next-lebron?-not-so-fast',
+        'url': 'https://www.cbssports.com/nba/video/donovan-mitchell-flashes-star-potential-in-game-2-victory-over-thunder/',
         'info_dict': {
-            'id': '708337219968',
+            'id': '1214315075735',
             'ext': 'mp4',
-            'title': 'Ben Simmons the next LeBron? Not so fast',
-            'description': 'md5:854294f627921baba1f4b9a990d87197',
-            'timestamp': 1466293740,
-            'upload_date': '20160618',
+            'title': 'Donovan Mitchell flashes star potential in Game 2 victory over Thunder',
+            'description': 'md5:df6f48622612c2d6bd2e295ddef58def',
+            'timestamp': 1524111457,
+            'upload_date': '20180419',
             'uploader': 'CBSI-NEW',
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         }
+    }, {
+        'url': 'https://www.cbssports.com/nba/news/nba-playoffs-2018-watch-76ers-vs-heat-game-3-series-schedule-tv-channel-online-stream/',
+        'only_matching': True,
     }]
 
     def _extract_video_info(self, filter_query, video_id):
         return self._extract_feed_info('dJ5BDC', 'VxxJg8Ymh8sE', filter_query, video_id)
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        video_id = self._search_regex(
+            [r'(?:=|%26)pcid%3D(\d+)', r'embedVideo(?:Container)?_(\d+)'],
+            webpage, 'video id')
         return self._extract_video_info('byId=%s' % video_id, video_id)

youtube_dl/extractor/ccma.py

@@ -4,11 +4,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
+    clean_html,
     int_or_none,
     parse_duration,
     parse_iso8601,
-    clean_html,
+    parse_resolution,
 )
 
 
@@ -40,34 +42,42 @@ class CCMAIE(InfoExtractor):
 
     def _real_extract(self, url):
         media_type, media_id = re.match(self._VALID_URL, url).groups()
-        media_data = {}
-        formats = []
-        profiles = ['pc'] if media_type == 'audio' else ['mobil', 'pc']
-        for i, profile in enumerate(profiles):
-            md = self._download_json('http://dinamics.ccma.cat/pvideo/media.jsp', media_id, query={
+
+        media = self._download_json(
+            'http://dinamics.ccma.cat/pvideo/media.jsp', media_id, query={
                 'media': media_type,
                 'idint': media_id,
-                'profile': profile,
-            }, fatal=False)
-            if md:
-                media_data = md
-                media_url = media_data.get('media', {}).get('url')
-                if media_url:
-                    formats.append({
-                        'format_id': profile,
-                        'url': media_url,
-                        'quality': i,
-                    })
+            })
+
+        formats = []
+        media_url = media['media']['url']
+        if isinstance(media_url, list):
+            for format_ in media_url:
+                format_url = format_.get('file')
+                if not format_url or not isinstance(format_url, compat_str):
+                    continue
+                label = format_.get('label')
+                f = parse_resolution(label)
+                f.update({
+                    'url': format_url,
+                    'format_id': label,
+                })
+                formats.append(f)
+        else:
+            formats.append({
+                'url': media_url,
+                'vcodec': 'none' if media_type == 'audio' else None,
+            })
         self._sort_formats(formats)
 
-        informacio = media_data['informacio']
+        informacio = media['informacio']
         title = informacio['titol']
         durada = informacio.get('durada', {})
         duration = int_or_none(durada.get('milisegons'), 1000) or parse_duration(durada.get('text'))
         timestamp = parse_iso8601(informacio.get('data_emissio', {}).get('utc'))
 
         subtitles = {}
-        subtitols = media_data.get('subtitols', {})
+        subtitols = media.get('subtitols', {})
         if subtitols:
             sub_url = subtitols.get('url')
             if sub_url:
@@ -77,7 +87,7 @@ class CCMAIE(InfoExtractor):
                     })
 
         thumbnails = []
-        imatges = media_data.get('imatges', {})
+        imatges = media.get('imatges', {})
         if imatges:
             thumbnail_url = imatges.get('url')
             if thumbnail_url:

youtube_dl/extractor/etonline.py

@@ -1,39 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-
-
-class ETOnlineIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?etonline\.com/(?:[^/]+/)*(?P<id>[^/?#&]+)'
-    _TESTS = [{
-        'url': 'http://www.etonline.com/tv/211130_dove_cameron_liv_and_maddie_emotional_episode_series_finale/',
-        'info_dict': {
-            'id': '211130_dove_cameron_liv_and_maddie_emotional_episode_series_finale',
-            'title': 'md5:a21ec7d3872ed98335cbd2a046f34ee6',
-            'description': 'md5:8b94484063f463cca709617c79618ccd',
-        },
-        'playlist_count': 2,
-    }, {
-        'url': 'http://www.etonline.com/media/video/here_are_the_stars_who_love_bringing_their_moms_as_dates_to_the_oscars-211359/',
-        'only_matching': True,
-    }]
-    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1242911076001/default_default/index.html?videoId=ref:%s'
-
-    def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, playlist_id)
-
-        entries = [
-            self.url_result(
-                self.BRIGHTCOVE_URL_TEMPLATE % video_id, 'BrightcoveNew', video_id)
-            for video_id in re.findall(
-                r'site\.brightcove\s*\([^,]+,\s*["\'](title_\d+)', webpage)]
-
-        return self.playlist_result(
-            entries, playlist_id,
-            self._og_search_title(webpage, fatal=False),
-            self._og_search_description(webpage))

youtube_dl/extractor/extractors.py

@@ -326,7 +326,6 @@ from .espn import (
     FiveThirtyEightIE,
 )
 from .esri import EsriVideoIE
-from .etonline import ETOnlineIE
 from .europa import EuropaIE
 from .everyonesmixtape import EveryonesMixtapeIE
 from .expotv import ExpoTVIE

youtube_dl/extractor/extremetube.py

@@ -8,12 +8,12 @@ class ExtremeTubeIE(KeezMoviesIE):
     _VALID_URL = r'https?://(?:www\.)?extremetube\.com/(?:[^/]+/)?video/(?P<id>[^/#?&]+)'
     _TESTS = [{
         'url': 'http://www.extremetube.com/video/music-video-14-british-euro-brit-european-cumshots-swallow-652431',
-        'md5': '1fb9228f5e3332ec8c057d6ac36f33e0',
+        'md5': '92feaafa4b58e82f261e5419f39c60cb',
         'info_dict': {
             'id': 'music-video-14-british-euro-brit-european-cumshots-swallow-652431',
             'ext': 'mp4',
             'title': 'Music Video 14 british euro brit european cumshots swallow',
-            'uploader': 'unknown',
+            'uploader': 'anonim',
             'view_count': int,
             'age_limit': 18,
         }
@@ -36,10 +36,10 @@ class ExtremeTubeIE(KeezMoviesIE):
                 r'<h1[^>]+title="([^"]+)"[^>]*>', webpage, 'title')
 
         uploader = self._html_search_regex(
-            r'Uploaded by:\s*</strong>\s*(.+?)\s*</div>',
+            r'Uploaded by:\s*</[^>]+>\s*<a[^>]+>(.+?)</a>',
             webpage, 'uploader', fatal=False)
         view_count = str_to_int(self._search_regex(
-            r'Views:\s*</strong>\s*<span>([\d,\.]+)</span>',
+            r'Views:\s*</[^>]+>\s*<[^>]+>([\d,\.]+)</',
             webpage, 'view count', fatal=False))
 
         info.update({

youtube_dl/extractor/generic.py

@@ -1220,7 +1220,7 @@ class GenericIE(InfoExtractor):
                 'title': '35871',
                 'timestamp': 1355743100,
                 'upload_date': '20121217',
-                'uploader_id': 'batchUser',
+                'uploader_id': 'cplapp@learn360.com',
             },
             'add_ie': ['Kaltura'],
         },
@@ -1271,6 +1271,22 @@ class GenericIE(InfoExtractor):
             },
             'add_ie': ['Kaltura'],
         },
+        {
+            # meta twitter:player
+            'url': 'http://thechive.com/2017/12/08/all-i-want-for-christmas-is-more-twerk/',
+            'info_dict': {
+                'id': '0_01b42zps',
+                'ext': 'mp4',
+                'title': 'Main Twerk (Video)',
+                'upload_date': '20171208',
+                'uploader_id': 'sebastian.salinas@thechive.com',
+                'timestamp': 1512713057,
+            },
+            'params': {
+                'skip_download': True,
+            },
+            'add_ie': ['Kaltura'],
+        },
         # referrer protected EaglePlatform embed
         {
             'url': 'https://tvrain.ru/lite/teleshow/kak_vse_nachinalos/namin-418921/',

youtube_dl/extractor/instagram.py

@@ -6,11 +6,16 @@ import json
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..compat import (
+    compat_str,
+    compat_HTTPError,
+)
 from ..utils import (
+    ExtractorError,
     get_element_by_attribute,
     int_or_none,
     lowercase_escape,
+    std_headers,
     try_get,
 )
 
@@ -239,6 +244,8 @@ class InstagramUserIE(InfoExtractor):
         }
     }
 
+    _gis_tmpl = None
+
     def _entries(self, data):
         def get_count(suffix):
             return int_or_none(try_get(
@@ -254,19 +261,39 @@ class InstagramUserIE(InfoExtractor):
         for page_num in itertools.count(1):
             variables = json.dumps({
                 'id': uploader_id,
-                'first': 100,
+                'first': 12,
                 'after': cursor,
             })
-            s = '%s:%s:%s' % (rhx_gis, csrf_token, variables)
-            media = self._download_json(
-                'https://www.instagram.com/graphql/query/', uploader_id,
-                'Downloading JSON page %d' % page_num, headers={
-                    'X-Requested-With': 'XMLHttpRequest',
-                    'X-Instagram-GIS': hashlib.md5(s.encode('utf-8')).hexdigest(),
-                }, query={
-                    'query_hash': '472f257a40c653c64c666ce877d59d2b',
-                    'variables': variables,
-                })['data']['user']['edge_owner_to_timeline_media']
+
+            if self._gis_tmpl:
+                gis_tmpls = [self._gis_tmpl]
+            else:
+                gis_tmpls = [
+                    '%s' % rhx_gis,
+                    '',
+                    '%s:%s' % (rhx_gis, csrf_token),
+                    '%s:%s:%s' % (rhx_gis, csrf_token, std_headers['User-Agent']),
+                ]
+
+            for gis_tmpl in gis_tmpls:
+                try:
+                    media = self._download_json(
+                        'https://www.instagram.com/graphql/query/', uploader_id,
+                        'Downloading JSON page %d' % page_num, headers={
+                            'X-Requested-With': 'XMLHttpRequest',
+                            'X-Instagram-GIS': hashlib.md5(
+                                ('%s:%s' % (gis_tmpl, variables)).encode('utf-8')).hexdigest(),
+                        }, query={
+                            'query_hash': '42323d64886122307be10013ad2dcc44',
+                            'variables': variables,
+                        })['data']['user']['edge_owner_to_timeline_media']
+                    self._gis_tmpl = gis_tmpl
+                    break
+                except ExtractorError as e:
+                    if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                        if gis_tmpl != gis_tmpls[-1]:
+                            continue
+                    raise
 
             edges = media.get('edges')
             if not edges or not isinstance(edges, list):

youtube_dl/extractor/kaltura.py

@@ -135,10 +135,10 @@ class KalturaIE(InfoExtractor):
                 ''', webpage) or
             re.search(
                 r'''(?xs)
-                    <iframe[^>]+src=(?P<q1>["'])
-                      (?:https?:)?//(?:www\.)?kaltura\.com/(?:(?!(?P=q1)).)*\b(?:p|partner_id)/(?P<partner_id>\d+)
+                    <(?:iframe[^>]+src|meta[^>]+\bcontent)=(?P<q1>["'])
+                      (?:https?:)?//(?:(?:www|cdnapi)\.)?kaltura\.com/(?:(?!(?P=q1)).)*\b(?:p|partner_id)/(?P<partner_id>\d+)
                       (?:(?!(?P=q1)).)*
-                      [?&]entry_id=(?P<id>(?:(?!(?P=q1))[^&])+)
+                      [?&;]entry_id=(?P<id>(?:(?!(?P=q1))[^&])+)
                     (?P=q1)
                 ''', webpage)
         )

youtube_dl/extractor/keezmovies.py

@@ -20,23 +20,23 @@ from ..utils import (
 class KeezMoviesIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?keezmovies\.com/video/(?:(?P<display_id>[^/]+)-)?(?P<id>\d+)'
     _TESTS = [{
-        'url': 'http://www.keezmovies.com/video/petite-asian-lady-mai-playing-in-bathtub-1214711',
-        'md5': '1c1e75d22ffa53320f45eeb07bc4cdc0',
+        'url': 'https://www.keezmovies.com/video/arab-wife-want-it-so-bad-i-see-she-thirsty-and-has-tiny-money-18070681',
+        'md5': '2ac69cdb882055f71d82db4311732a1a',
         'info_dict': {
-            'id': '1214711',
-            'display_id': 'petite-asian-lady-mai-playing-in-bathtub',
+            'id': '18070681',
+            'display_id': 'arab-wife-want-it-so-bad-i-see-she-thirsty-and-has-tiny-money',
             'ext': 'mp4',
-            'title': 'Petite Asian Lady Mai Playing In Bathtub',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'title': 'Arab wife want it so bad I see she thirsty and has tiny money.',
+            'thumbnail': None,
             'view_count': int,
             'age_limit': 18,
         }
     }, {
-        'url': 'http://www.keezmovies.com/video/1214711',
+        'url': 'http://www.keezmovies.com/video/18070681',
         'only_matching': True,
     }]
 
-    def _extract_info(self, url):
+    def _extract_info(self, url, fatal=True):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('id')
         display_id = (mobj.group('display_id')
@@ -55,7 +55,7 @@ class KeezMoviesIE(InfoExtractor):
         encrypted = False
 
         def extract_format(format_url, height=None):
-            if not isinstance(format_url, compat_str) or not format_url.startswith('http'):
+            if not isinstance(format_url, compat_str) or not format_url.startswith(('http', '//')):
                 return
             if format_url in format_urls:
                 return
@@ -105,7 +105,11 @@ class KeezMoviesIE(InfoExtractor):
                 raise ExtractorError(
                     'Video %s is no longer available' % video_id, expected=True)
 
-        self._sort_formats(formats)
+        try:
+            self._sort_formats(formats)
+        except ExtractorError:
+            if fatal:
+                raise
 
         if not title:
             title = self._html_search_regex(
@@ -122,7 +126,9 @@ class KeezMoviesIE(InfoExtractor):
         }
 
     def _real_extract(self, url):
-        webpage, info = self._extract_info(url)
+        webpage, info = self._extract_info(url, fatal=False)
+        if not info['formats']:
+            return self.url_result(url, 'Generic')
         info['view_count'] = str_to_int(self._search_regex(
             r'<b>([\d,.]+)</b> Views?', webpage, 'view count', fatal=False))
         return info

youtube_dl/extractor/mofosex.py

@@ -12,7 +12,7 @@ class MofosexIE(KeezMoviesIE):
     _VALID_URL = r'https?://(?:www\.)?mofosex\.com/videos/(?P<id>\d+)/(?P<display_id>[^/?#&.]+)\.html'
     _TESTS = [{
         'url': 'http://www.mofosex.com/videos/318131/amateur-teen-playing-and-masturbating-318131.html',
-        'md5': '39a15853632b7b2e5679f92f69b78e91',
+        'md5': '558fcdafbb63a87c019218d6e49daf8a',
         'info_dict': {
             'id': '318131',
             'display_id': 'amateur-teen-playing-and-masturbating-318131',

youtube_dl/extractor/nexx.py

@@ -230,15 +230,18 @@ class NexxIE(InfoExtractor):
 
         azure_locator = stream_data['azureLocator']
 
-        AZURE_URL = 'http://nx%s%02d.akamaized.net/'
-
-        def get_cdn_shield_base(shield_type='', prefix='-p'):
+        def get_cdn_shield_base(shield_type='', static=False):
             for secure in ('', 's'):
                 cdn_shield = stream_data.get('cdnShield%sHTTP%s' % (shield_type, secure.upper()))
                 if cdn_shield:
                     return 'http%s://%s' % (secure, cdn_shield)
             else:
-                return AZURE_URL % (prefix, int(stream_data['azureAccount'].replace('nexxplayplus', '')))
+                if 'fb' in stream_data['azureAccount']:
+                    prefix = 'df' if static else 'f'
+                else:
+                    prefix = 'd' if static else 'p'
+                account = int(stream_data['azureAccount'].replace('nexxplayplus', '').replace('nexxplayfb', ''))
+                return 'http://nx-%s%02d.akamaized.net/' % (prefix, account)
 
         azure_stream_base = get_cdn_shield_base()
         is_ml = ',' in language
@@ -260,7 +263,7 @@ class NexxIE(InfoExtractor):
         formats.extend(self._extract_ism_formats(
             azure_manifest_url % '', video_id, ism_id='%s-mss' % cdn, fatal=False))
 
-        azure_progressive_base = get_cdn_shield_base('Prog', '-d')
+        azure_progressive_base = get_cdn_shield_base('Prog', True)
         azure_file_distribution = stream_data.get('azureFileDistribution')
         if azure_file_distribution:
             fds = azure_file_distribution.split(',')

youtube_dl/extractor/nick.py

@@ -81,13 +81,23 @@ class NickIE(MTVServicesInfoExtractor):
 
 class NickBrIE(MTVServicesInfoExtractor):
     IE_NAME = 'nickelodeon:br'
-    _VALID_URL = r'https?://(?P<domain>(?:www\.)?nickjr|mundonick\.uol)\.com\.br/(?:programas/)?[^/]+/videos/(?:episodios/)?(?P<id>[^/?#.]+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?:
+                            (?P<domain>(?:www\.)?nickjr|mundonick\.uol)\.com\.br|
+                            (?:www\.)?nickjr\.nl
+                        )
+                        /(?:programas/)?[^/]+/videos/(?:episodios/)?(?P<id>[^/?\#.]+)
+                    '''
     _TESTS = [{
         'url': 'http://www.nickjr.com.br/patrulha-canina/videos/210-labirinto-de-pipoca/',
         'only_matching': True,
     }, {
         'url': 'http://mundonick.uol.com.br/programas/the-loud-house/videos/muitas-irmas/7ljo9j',
         'only_matching': True,
+    }, {
+        'url': 'http://www.nickjr.nl/paw-patrol/videos/311-ge-wol-dig-om-terug-te-zijn/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

youtube_dl/extractor/openload.py

@@ -340,7 +340,10 @@ class OpenloadIE(InfoExtractor):
                       get_element_by_id('streamurj', webpage) or
                       self._search_regex(
                           (r'>\s*([\w-]+~\d{10,}~\d+\.\d+\.0\.0~[\w-]+)\s*<',
-                           r'>\s*([\w~-]+~\d+\.\d+\.\d+\.\d+~[\w~-]+)'), webpage,
+                           r'>\s*([\w~-]+~\d+\.\d+\.\d+\.\d+~[\w~-]+)',
+                           r'>\s*([\w-]+~\d{10,}~(?:[a-f\d]+:){2}:~[\w-]+)\s*<',
+                           r'>\s*([\w~-]+~[a-f0-9:]+~[\w~-]+)\s*<',
+                           r'>\s*([\w~-]+~[a-f0-9:]+~[\w~-]+)'), webpage,
                           'stream URL'))
 
         video_url = 'https://openload.co/stream/%s?mime=true' % decoded_id

youtube_dl/extractor/pornflip.py

@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class PornFlipIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?pornflip\.com/(?:v|embed)/(?P<id>[0-9A-Za-z-]{11})'
+    _VALID_URL = r'https?://(?:www\.)?pornflip\.com/(?:v|embed)/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://www.pornflip.com/v/wz7DfNhMmep',
         'md5': '98c46639849145ae1fd77af532a9278c',
@@ -40,6 +40,9 @@ class PornFlipIE(InfoExtractor):
     }, {
         'url': 'https://www.pornflip.com/embed/EkRD6-vS2-s',
         'only_matching': True,
+    }, {
+        'url': 'https://www.pornflip.com/v/NG9q6Pb_iK8',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

youtube_dl/extractor/rentv.py

@@ -3,6 +3,10 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..compat import compat_str
+from ..utils import (
+    determine_ext,
+    int_or_none,
+)
 
 
 class RENTVIE(InfoExtractor):
@@ -13,7 +17,9 @@ class RENTVIE(InfoExtractor):
         'info_dict': {
             'id': '118577',
             'ext': 'mp4',
-            'title': 'Документальный спецпроект: "Промывка мозгов. Технологии XXI века"'
+            'title': 'Документальный спецпроект: "Промывка мозгов. Технологии XXI века"',
+            'timestamp': 1472230800,
+            'upload_date': '20160826',
         }
     }, {
         'url': 'http://ren.tv/player/118577',
@@ -26,9 +32,33 @@ class RENTVIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage('http://ren.tv/player/' + video_id, video_id)
-        jw_config = self._parse_json(self._search_regex(
-            r'config\s*=\s*({.+});', webpage, 'jw config'), video_id)
-        return self._parse_jwplayer_data(jw_config, video_id, m3u8_id='hls')
+        config = self._parse_json(self._search_regex(
+            r'config\s*=\s*({.+})\s*;', webpage, 'config'), video_id)
+        title = config['title']
+        formats = []
+        for video in config['src']:
+            src = video.get('src')
+            if not src or not isinstance(src, compat_str):
+                continue
+            ext = determine_ext(src)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    src, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
+            else:
+                formats.append({
+                    'url': src,
+                })
+        self._sort_formats(formats)
+        return {
+            'id': video_id,
+            'title': title,
+            'description': config.get('description'),
+            'thumbnail': config.get('image'),
+            'duration': int_or_none(config.get('duration')),
+            'timestamp': int_or_none(config.get('date')),
+            'formats': formats,
+        }
 
 
 class RENTVArticleIE(InfoExtractor):

youtube_dl/extractor/svt.py

@@ -22,6 +22,8 @@ class SVTBaseIE(InfoExtractor):
     _GEO_COUNTRIES = ['SE']
 
     def _extract_video(self, video_info, video_id):
+        is_live = dict_get(video_info, ('live', 'simulcast'), default=False)
+        m3u8_protocol = 'm3u8' if is_live else 'm3u8_native'
         formats = []
         for vr in video_info['videoReferences']:
             player_type = vr.get('playerType') or vr.get('format')
@@ -30,7 +32,7 @@ class SVTBaseIE(InfoExtractor):
             if ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
                     vurl, video_id,
-                    ext='mp4', entry_protocol='m3u8_native',
+                    ext='mp4', entry_protocol=m3u8_protocol,
                     m3u8_id=player_type, fatal=False))
             elif ext == 'f4m':
                 formats.extend(self._extract_f4m_formats(
@@ -90,6 +92,7 @@ class SVTBaseIE(InfoExtractor):
             'season_number': season_number,
             'episode': episode,
             'episode_number': episode_number,
+            'is_live': is_live,
         }
 
 
@@ -134,7 +137,7 @@ class SVTPlayBaseIE(SVTBaseIE):
 
 class SVTPlayIE(SVTPlayBaseIE):
     IE_DESC = 'SVT Play and Öppet arkiv'
-    _VALID_URL = r'https?://(?:www\.)?(?:svtplay|oppetarkiv)\.se/(?:video|klipp)/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:svtplay|oppetarkiv)\.se/(?:video|klipp|kanaler)/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://www.svtplay.se/video/5996901/flygplan-till-haile-selassie/flygplan-till-haile-selassie-2',
         'md5': '2b6704fe4a28801e1a098bbf3c5ac611',
@@ -158,6 +161,9 @@ class SVTPlayIE(SVTPlayBaseIE):
     }, {
         'url': 'http://www.svtplay.se/klipp/9023742/stopptid-om-bjorn-borg',
         'only_matching': True,
+    }, {
+        'url': 'https://www.svtplay.se/kanaler/svt1',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -173,6 +179,10 @@ class SVTPlayIE(SVTPlayBaseIE):
 
         thumbnail = self._og_search_thumbnail(webpage)
 
+        def adjust_title(info):
+            if info['is_live']:
+                info['title'] = self._live_title(info['title'])
+
         if data:
             video_info = try_get(
                 data, lambda x: x['context']['dispatcher']['stores']['VideoTitlePageStore']['data']['video'],
@@ -183,6 +193,7 @@ class SVTPlayIE(SVTPlayBaseIE):
                     'title': data['context']['dispatcher']['stores']['MetaStore']['title'],
                     'thumbnail': thumbnail,
                 })
+                adjust_title(info_dict)
                 return info_dict
 
         video_id = self._search_regex(
@@ -198,6 +209,7 @@ class SVTPlayIE(SVTPlayBaseIE):
                 info_dict['title'] = re.sub(
                     r'\s*\|\s*.+?$', '',
                     info_dict.get('episode') or self._og_search_title(webpage))
+            adjust_title(info_dict)
             return info_dict
 
 

youtube_dl/extractor/twitch.py

@@ -168,6 +168,13 @@ class TwitchItemBaseIE(TwitchBaseIE):
         return self.playlist_result(entries, info['id'], info['title'])
 
     def _extract_info(self, info):
+        status = info.get('status')
+        if status == 'recording':
+            is_live = True
+        elif status == 'recorded':
+            is_live = False
+        else:
+            is_live = None
         return {
             'id': info['_id'],
             'title': info.get('title') or 'Untitled Broadcast',
@@ -178,6 +185,7 @@ class TwitchItemBaseIE(TwitchBaseIE):
             'uploader_id': info.get('channel', {}).get('name'),
             'timestamp': parse_iso8601(info.get('recorded_at')),
             'view_count': int_or_none(info.get('views')),
+            'is_live': is_live,
         }
 
     def _real_extract(self, url):

youtube_dl/extractor/youtube.py

@@ -87,7 +87,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         (username, password) = self._get_login_info()
         # No authentication to be performed
         if username is None:
-            if self._LOGIN_REQUIRED:
+            if self._LOGIN_REQUIRED and self._downloader.params.get('cookiefile') is None:
                 raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
             return True
 
@@ -2699,10 +2699,7 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
     def _real_initialize(self):
         self._login()
 
-    def _real_extract(self, url):
-        page = self._download_webpage(
-            'https://www.youtube.com/feed/%s' % self._FEED_NAME, self._PLAYLIST_TITLE)
-
+    def _entries(self, page):
         # The extraction process is the same as for playlists, but the regex
         # for the video ids doesn't contain an index
         ids = []
@@ -2713,12 +2710,15 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
             # 'recommended' feed has infinite 'load more' and each new portion spins
             # the same videos in (sometimes) slightly different order, so we'll check
             # for unicity and break when portion has no new videos
-            new_ids = filter(lambda video_id: video_id not in ids, orderedSet(matches))
+            new_ids = list(filter(lambda video_id: video_id not in ids, orderedSet(matches)))
             if not new_ids:
                 break
 
             ids.extend(new_ids)
 
+            for entry in self._ids_to_results(new_ids):
+                yield entry
+
             mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
             if not mobj:
                 break
@@ -2730,8 +2730,12 @@ class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
             content_html = more['content_html']
             more_widget_html = more['load_more_widget_html']
 
+    def _real_extract(self, url):
+        page = self._download_webpage(
+            'https://www.youtube.com/feed/%s' % self._FEED_NAME,
+            self._PLAYLIST_TITLE)
         return self.playlist_result(
-            self._ids_to_results(ids), playlist_title=self._PLAYLIST_TITLE)
+            self._entries(page), playlist_title=self._PLAYLIST_TITLE)
 
 
 class YoutubeWatchLaterIE(YoutubePlaylistIE):

youtube_dl/utils.py

@@ -2574,8 +2574,8 @@ def _match_one(filter_part, dct):
         return op(actual_value, comparison_value)
 
     UNARY_OPERATORS = {
-        '': lambda v: v is not None,
-        '!': lambda v: v is None,
+        '': lambda v: (v is True) if isinstance(v, bool) else (v is not None),
+        '!': lambda v: (v is False) if isinstance(v, bool) else (v is None),
     }
     operator_rex = re.compile(r'''(?x)\s*
         (?P<op>%s)\s*(?P<key>[a-z_]+)

youtube_dl/version.py

@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2018.04.16'
+__version__ = '2018.04.25'